From 13279c95097cc84f08c978c644afc5d6bc809fa3 Mon Sep 17 00:00:00 2001 From: Kaviraj Date: Mon, 12 Jul 2021 18:04:18 +0200 Subject: [PATCH 01/24] IP matcher and filter via netaddr.IP package Signed-off-by: Kaviraj --- go.mod | 1 + go.sum | 8 + pkg/logql/log/ip.go | 139 ++ pkg/logql/log/ip_test.go | 119 ++ testlogs.txt | 8 + vendor/go4.org/intern/LICENSE | 29 + vendor/go4.org/intern/README.md | 4 + vendor/go4.org/intern/go.mod | 5 + vendor/go4.org/intern/go.sum | 2 + vendor/go4.org/intern/intern.go | 183 ++ .../unsafe/assume-no-moving-gc/LICENSE | 29 + .../unsafe/assume-no-moving-gc/README.md | 13 + .../assume-no-moving-gc.go | 22 + .../go4.org/unsafe/assume-no-moving-gc/go.mod | 3 + .../unsafe/assume-no-moving-gc/untested.go | 25 + vendor/inet.af/netaddr/.gitignore | 3 + vendor/inet.af/netaddr/.gitmodules | 3 + vendor/inet.af/netaddr/AUTHORS | 4 + vendor/inet.af/netaddr/LICENSE | 27 + vendor/inet.af/netaddr/README.md | 41 + vendor/inet.af/netaddr/fuzz.go | 197 ++ vendor/inet.af/netaddr/go.mod | 10 + vendor/inet.af/netaddr/go.sum | 33 + vendor/inet.af/netaddr/ipset.go | 458 ++++ vendor/inet.af/netaddr/mask6.go | 141 ++ vendor/inet.af/netaddr/netaddr.go | 1858 +++++++++++++++++ vendor/inet.af/netaddr/uint128.go | 82 + vendor/modules.txt | 7 + 28 files changed, 3454 insertions(+) create mode 100644 pkg/logql/log/ip.go create mode 100644 pkg/logql/log/ip_test.go create mode 100644 testlogs.txt create mode 100644 vendor/go4.org/intern/LICENSE create mode 100644 vendor/go4.org/intern/README.md create mode 100644 vendor/go4.org/intern/go.mod create mode 100644 vendor/go4.org/intern/go.sum create mode 100644 vendor/go4.org/intern/intern.go create mode 100644 vendor/go4.org/unsafe/assume-no-moving-gc/LICENSE create mode 100644 vendor/go4.org/unsafe/assume-no-moving-gc/README.md create mode 100644 vendor/go4.org/unsafe/assume-no-moving-gc/assume-no-moving-gc.go create mode 100644 vendor/go4.org/unsafe/assume-no-moving-gc/go.mod create mode 100644 vendor/go4.org/unsafe/assume-no-moving-gc/untested.go create mode 100644 vendor/inet.af/netaddr/.gitignore create mode 100644 vendor/inet.af/netaddr/.gitmodules create mode 100644 vendor/inet.af/netaddr/AUTHORS create mode 100644 vendor/inet.af/netaddr/LICENSE create mode 100644 vendor/inet.af/netaddr/README.md create mode 100644 vendor/inet.af/netaddr/fuzz.go create mode 100644 vendor/inet.af/netaddr/go.mod create mode 100644 vendor/inet.af/netaddr/go.sum create mode 100644 vendor/inet.af/netaddr/ipset.go create mode 100644 vendor/inet.af/netaddr/mask6.go create mode 100644 vendor/inet.af/netaddr/netaddr.go create mode 100644 vendor/inet.af/netaddr/uint128.go diff --git a/go.mod b/go.mod index 0bf64cd04d3ac..258df0d8d41fb 100644 --- a/go.mod +++ b/go.mod @@ -78,6 +78,7 @@ require ( gopkg.in/fsnotify.v1 v1.4.7 gopkg.in/yaml.v2 v2.4.0 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b + inet.af/netaddr v0.0.0-20210707202901-70468d781e6c k8s.io/klog v1.0.0 ) diff --git a/go.sum b/go.sum index ab968e413f45f..ea2609ca8086c 100644 --- a/go.sum +++ b/go.sum @@ -435,6 +435,7 @@ github.com/duosecurity/duo_api_golang v0.0.0-20190308151101-6c680f768e74/go.mod github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/dvyukov/go-fuzz v0.0.0-20210103155950-6a8e9d1f2415/go.mod h1:11Gm+ccJnvAhCNLlf5+cS9KjtbaD5I5zaZpFMsTHWTw= github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= github.com/eapache/go-resiliency v1.2.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= @@ -1781,6 +1782,11 @@ go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= go.uber.org/zap v1.14.1/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc= go.uber.org/zap v1.16.0 h1:uFRZXykJGK9lLY4HtgSw44DnIcAM+kRBP7x5m+NpAOM= go.uber.org/zap v1.16.0/go.mod h1:MA8QOfq0BHJwdXa996Y4dYkAqRKB8/1K1QMMZVaNZjQ= +go4.org/intern v0.0.0-20210108033219-3eb7198706b2 h1:VFTf+jjIgsldaz/Mr00VaCSswHJrI2hIjQygE/W4IMg= +go4.org/intern v0.0.0-20210108033219-3eb7198706b2/go.mod h1:vLqJ+12kCw61iCWsPto0EOHhBS+o4rO5VIucbc9g2Cc= +go4.org/unsafe/assume-no-moving-gc v0.0.0-20201222175341-b30ae309168e/go.mod h1:FftLjUGFEDu5k8lt0ddY+HcrH/qU/0qk+H8j9/nTl3E= +go4.org/unsafe/assume-no-moving-gc v0.0.0-20201222180813-1025295fd063 h1:1tk03FUNpulq2cuWpXZWj649rwJpk0d20rxWiopKRmc= +go4.org/unsafe/assume-no-moving-gc v0.0.0-20201222180813-1025295fd063/go.mod h1:FftLjUGFEDu5k8lt0ddY+HcrH/qU/0qk+H8j9/nTl3E= golang.org/x/crypto v0.0.0-20180608092829-8ac0e0d97ce4/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -2422,6 +2428,8 @@ honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9 honnef.co/go/tools v0.0.1-2020.1.4 h1:UoveltGrhghAA7ePc+e+QYDHXrBps2PqFZiHkGR/xK8= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= howett.net/plist v0.0.0-20181124034731-591f970eefbb/go.mod h1:vMygbs4qMhSZSc4lCUl2OEE+rDiIIJAIdR4m7MiMcm0= +inet.af/netaddr v0.0.0-20210707202901-70468d781e6c h1:ZNUX2CiFwNbN1VFaD4MQFmC8o5Rxc7BQW1P1K8kMpbE= +inet.af/netaddr v0.0.0-20210707202901-70468d781e6c/go.mod h1:z0nx+Dh+7N7CC8V5ayHtHGpZpxLQZZxkIaaz6HN65Ls= k8s.io/api v0.21.0 h1:gu5iGF4V6tfVCQ/R+8Hc0h7H1JuEhzyEi9S4R5LM8+Y= k8s.io/api v0.21.0/go.mod h1:+YbrhBBGgsxbF6o6Kj4KJPJnBmAKuXDeS3E18bgHNVU= k8s.io/apimachinery v0.0.0-20180821005732-488889b0007f/go.mod h1:ccL7Eh7zubPUSh9A3USN90/OzHNSVN6zxzde07TDCL0= diff --git a/pkg/logql/log/ip.go b/pkg/logql/log/ip.go new file mode 100644 index 0000000000000..a7015e1bc880a --- /dev/null +++ b/pkg/logql/log/ip.go @@ -0,0 +1,139 @@ +package log + +import ( + "errors" + "fmt" + "strings" + "unicode" + + "inet.af/netaddr" +) + +var ( + errInvalidPattern = errors.New("invalid pattern") +) + +const ( + IPV4_CHARSET = "0123456789." + IPV6_CHARSET = "0123456789abcdefABCDEF.:" +) + +// Should be one of the netaddr.IP, netaddr.IPRange, netadd.IPPrefix. +type IPMatcher interface{} + +type IPFilter struct { + matcher IPMatcher +} + +// IPFilter search for IP addresses of given `pattern` in the given `line`. +// It returns true if pattern is matched with at least one IP in the `line` + +// pattern - can be of the following form for both IPv4 and IPv6. +// 1. SINGLE-IP - "192.168.0.1" +// 2. IP RANGE - "192.168.0.1-192.168.0.23" +// 3. CIDR - "192.168.0.0/16" +func NewIPFilter(pattern string) (*IPFilter, error) { + matcher, err := getMatcher(pattern) + if err != nil { + return nil, err + } + return &IPFilter{matcher: matcher}, nil +} + +func (ipf *IPFilter) Filter(line string) bool { + if len(line) == 0 { + return false + } + + n := len(line) + + for i := 0; i < n; i++ { + if i+3 < n && ipv4Hint([4]byte{line[i], line[i+1], line[i+2], line[i+3]}) { + start := i + iplen := strings.LastIndexAny(line[start:], IPV4_CHARSET) + ip, err := netaddr.ParseIP(line[start : start+iplen+1]) + if err == nil { + if contains(ipf.matcher, ip) { + return true + } + } + i += iplen + continue + } + + if i+4 < n && ipv6Hint([5]byte{line[i], line[i+1], line[i+2], line[i+3], line[i+4]}) { + start := i + iplen := strings.LastIndex(line[start:], IPV6_CHARSET) + ip, err := netaddr.ParseIP(line[start : start+iplen+1]) + if err == nil { + if contains(ipf.matcher, ip) { + return true + } + } + i += iplen + continue + } + } + return false +} + +func (ipf *IPFilter) Process(line []byte, _ *LabelsBuilder) ([]byte, bool) { + return line, ipf.Filter(string(line)) +} + +func (ipf *IPFilter) RequiredLabelNames() []string { + return []string{} +} + +func contains(matcher IPMatcher, ip netaddr.IP) bool { + switch m := matcher.(type) { + case netaddr.IP: + return m.Compare(ip) == 0 + case netaddr.IPRange: + return m.Contains(ip) + case netaddr.IPPrefix: + return m.Contains(ip) + } + return false +} + +func getMatcher(pattern string) (IPMatcher, error) { + var ( + matcher IPMatcher + err error + ) + + matcher, err = netaddr.ParseIP(pattern) // is it simple single IP? + if err == nil { + return matcher, nil + } + matcher, err = netaddr.ParseIPPrefix(pattern) // is it cidr format? (192.168.0.1/16) + if err == nil { + return matcher, nil + } + + matcher, err = netaddr.ParseIPRange(pattern) // is it IP range format? (192.168.0.1 - 192.168.4.5 + if err == nil { + return matcher, nil + } + + return nil, fmt.Errorf("%w: %q", errInvalidPattern, pattern) +} + +func ipv4Hint(prefix [4]byte) bool { + return unicode.IsDigit(rune(prefix[0])) && (prefix[1] == '.' || prefix[2] == '.' || prefix[3] == '.') +} + +func ipv6Hint(prefix [5]byte) bool { + hint1 := prefix[0] == ':' && prefix[1] == ':' && isHexDigit(prefix[2]) + hint2 := isHexDigit(prefix[0]) && prefix[1] == ':' + hint3 := isHexDigit(prefix[0]) && isHexDigit(prefix[1]) && prefix[2] == ':' + hint4 := isHexDigit(prefix[0]) && isHexDigit(prefix[1]) && isHexDigit(prefix[2]) && prefix[3] == ':' + hint5 := isHexDigit(prefix[0]) && isHexDigit(prefix[1]) && isHexDigit(prefix[2]) && isHexDigit(prefix[3]) && prefix[4] == ':' + + return hint1 || hint2 || hint3 || hint4 || hint5 +} + +func isHexDigit(r byte) bool { + return unicode.IsDigit(rune(r)) || (r >= 'a' && r <= 'f') || (r >= 'A' && r <= 'F') +} diff --git a/pkg/logql/log/ip_test.go b/pkg/logql/log/ip_test.go new file mode 100644 index 0000000000000..626b1df58c995 --- /dev/null +++ b/pkg/logql/log/ip_test.go @@ -0,0 +1,119 @@ +package log + +import ( + "testing" + + "github.com/prometheus/prometheus/pkg/labels" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func Test_IPFilter(t *testing.T) { + cases := []struct { + name string + pat string + input []string + expected []int // matched line indexes from the input + + // in case of expecting error + err error + fail bool + }{ + { + name: "single IP", + pat: "192.168.0.1", + input: []string{ + "vpn 192.168.0.5 connected to vm", + "vpn 192.168.0.1 connected to vm", + "x", + "hello world!", + "", + }, + expected: []int{1}, // should match with only line at index `1` from the input + }, + { + name: "IP range", + pat: "192.168.0.1-192.189.10.12", + input: []string{ + "vpn 192.168.0.0 connected to vm", + "vpn 192.168.0.1 connected to vm", + "vpn 192.172.6.1 connected to vm", + "vpn 192.255.255.255 connected to vm", + "x", + "hello world!", + "", + }, + expected: []int{1, 2}, + }, + { + name: "wrong IP range syntax extra space", // NOTE(kavi): Should we handle this as normal valid range pattern? + pat: "192.168.0.1 - 192.189.10.12", + fail: true, + err: errInvalidPattern, + }, + { + name: "CIDR", + pat: "192.168.4.5/16", + input: []string{ + "vpn 192.168.0.0 connected to vm", + "vpn 192.168.0.1 connected to vm", + "vpn 192.172.6.1 connected to vm", + "vpn 192.168.255.255 connected to vm", + "x", + "hello world!", + "", + }, + expected: []int{0, 1, 3}, + }, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + ip, err := NewIPFilter(c.pat) + if c.fail { + assert.Error(t, c.err, err) + return + } + require.NoError(t, err) + + got := make([]int, 0) + for i, in := range c.input { + if ip.Filter(in) { + got = append(got, i) + } + } + assert.Equal(t, c.expected, got) + }) + } +} + +func Benchmark_IPFilter(b *testing.B) { + b.ReportAllocs() + + line := [][]byte{ + []byte(`vpn 192.168.0.0 connected to vm`), + /// todo add more cases,long line without match, with match etc.... + } + lbbb := NewBaseLabelsBuilder() + lbb := lbbb.ForLabels(labels.Labels{labels.Label{Name: "foo", Value: "bar"}}, 0) + + for _, pattern := range []string{ + "127.0.0.1", + "192.168.0.1-192.189.10.12", + "192.168.4.5/16", + } { + b.Run(pattern, func(b *testing.B) { + stage, err := NewIPFilter(pattern) + require.Nil(b, err) + b.ResetTimer() + + for n := 0; n < b.N; n++ { + for _, l := range line { + lbb.Reset() + _, _ = stage.Process(l, lbb) + } + } + }) + } + +} diff --git a/testlogs.txt b/testlogs.txt new file mode 100644 index 0000000000000..2846e8460e69b --- /dev/null +++ b/testlogs.txt @@ -0,0 +1,8 @@ +level=info ts=2021-07-10T20:29:12.464740436Z caller=main.go:129 msg="Starting Loki" version="(version=, branch=, revision=)" +level=info ts=2021-07-10T20:29:12.477325579Z caller=server.go:239 http=[::]:3100 grpc=[::]:9096 msg="server listening on addresses" +level=info ts=2021-07-10T20:29:12.478423515Z caller=shipper_index_client.go:108 msg="starting boltdb shipper in 0 mode" +level=info ts=2021-07-10T20:29:12.478421065Z caller=table_manager.go:171 msg="uploading tables" +level=info ts=2021-07-10T20:29:12.481869432Z caller=mapper.go:46 msg="cleaning up mapped rules directory" path=/tmp/loki/rules-temp +level=error ts=2021-07-10T20:29:12.481942251Z caller=mapper.go:50 msg="unable to read rules directory" path=/tmp/loki/rules-temp err="open /tmp/loki/rules-temp: no such file or directory" +level=info ts=2021-07-10T20:29:12.485403993Z caller=module_service.go:59 msg=initialising module=memberlist-kv +level=info ts=2021-07-10T20:29:12.485512367Z caller=module_service.go:59 msg=initialising module=server diff --git a/vendor/go4.org/intern/LICENSE b/vendor/go4.org/intern/LICENSE new file mode 100644 index 0000000000000..b0ab8921dc3b4 --- /dev/null +++ b/vendor/go4.org/intern/LICENSE @@ -0,0 +1,29 @@ +BSD 3-Clause License + +Copyright (c) 2020, Brad Fitzpatrick +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/go4.org/intern/README.md b/vendor/go4.org/intern/README.md new file mode 100644 index 0000000000000..1db456b735b61 --- /dev/null +++ b/vendor/go4.org/intern/README.md @@ -0,0 +1,4 @@ +# go4.org/intern + +See https://godoc.org/go4.org/intern + diff --git a/vendor/go4.org/intern/go.mod b/vendor/go4.org/intern/go.mod new file mode 100644 index 0000000000000..c50c9014689ea --- /dev/null +++ b/vendor/go4.org/intern/go.mod @@ -0,0 +1,5 @@ +module go4.org/intern + +go 1.13 + +require go4.org/unsafe/assume-no-moving-gc v0.0.0-20201222175341-b30ae309168e diff --git a/vendor/go4.org/intern/go.sum b/vendor/go4.org/intern/go.sum new file mode 100644 index 0000000000000..07cec0ffb49f5 --- /dev/null +++ b/vendor/go4.org/intern/go.sum @@ -0,0 +1,2 @@ +go4.org/unsafe/assume-no-moving-gc v0.0.0-20201222175341-b30ae309168e h1:ExUmGi0ZsQmiVo9giDQqXkr7vreeXPMkOGIusfsfbzI= +go4.org/unsafe/assume-no-moving-gc v0.0.0-20201222175341-b30ae309168e/go.mod h1:FftLjUGFEDu5k8lt0ddY+HcrH/qU/0qk+H8j9/nTl3E= diff --git a/vendor/go4.org/intern/intern.go b/vendor/go4.org/intern/intern.go new file mode 100644 index 0000000000000..536014cd35ae9 --- /dev/null +++ b/vendor/go4.org/intern/intern.go @@ -0,0 +1,183 @@ +// Copyright 2020 Brad Fitzpatrick. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package intern lets you make smaller comparable values by boxing +// a larger comparable value (such as a 16 byte string header) down +// into a globally unique 8 byte pointer. +// +// The globally unique pointers are garbage collected with weak +// references and finalizers. This package hides that. +// +// The GitHub repo is https://github.com/go4org/intern +package intern // import "go4.org/intern" + +import ( + "os" + "runtime" + "strconv" + "sync" + "unsafe" + + _ "go4.org/unsafe/assume-no-moving-gc" +) + +// A Value pointer is the handle to an underlying comparable value. +// See func Get for how Value pointers may be used. +type Value struct { + _ [0]func() // prevent people from accidentally using value type as comparable + cmpVal interface{} + // resurrected is guarded by mu (for all instances of Value). + // It is set true whenever v is synthesized from a uintptr. + resurrected bool +} + +// Get returns the comparable value passed to the Get func +// that returned v. +func (v *Value) Get() interface{} { return v.cmpVal } + +// key is a key in our global value map. +// It contains type-specialized fields to avoid allocations +// when converting common types to empty interfaces. +type key struct { + s string + cmpVal interface{} + // isString reports whether key contains a string. + // Without it, the zero value of key is ambiguous. + isString bool +} + +// keyFor returns a key to use with cmpVal. +func keyFor(cmpVal interface{}) key { + if s, ok := cmpVal.(string); ok { + return key{s: s, isString: true} + } + return key{cmpVal: cmpVal} +} + +// Value returns a *Value built from k. +func (k key) Value() *Value { + if k.isString { + return &Value{cmpVal: k.s} + } + return &Value{cmpVal: k.cmpVal} +} + +var ( + // mu guards valMap, a weakref map of *Value by underlying value. + // It also guards the resurrected field of all *Values. + mu sync.Mutex + valMap = map[key]uintptr{} // to uintptr(*Value) + valSafe = safeMap() // non-nil in safe+leaky mode +) + +// safeMap returns a non-nil map if we're in safe-but-leaky mode, +// as controlled by GO4_INTERN_SAFE_BUT_LEAKY. +func safeMap() map[key]*Value { + if v, _ := strconv.ParseBool(os.Getenv("GO4_INTERN_SAFE_BUT_LEAKY")); v { + return map[key]*Value{} + } + return nil +} + +// Get returns a pointer representing the comparable value cmpVal. +// +// The returned pointer will be the same for Get(v) and Get(v2) +// if and only if v == v2, and can be used as a map key. +func Get(cmpVal interface{}) *Value { + return get(keyFor(cmpVal)) +} + +// GetByString is identical to Get, except that it is specialized for strings. +// This avoids an allocation from putting a string into an interface{} +// to pass as an argument to Get. +func GetByString(s string) *Value { + return get(key{s: s, isString: true}) +} + +// We play unsafe games that violate Go's rules (and assume a non-moving +// collector). So we quiet Go here. +// See the comment below Get for more implementation details. +//go:nocheckptr +func get(k key) *Value { + mu.Lock() + defer mu.Unlock() + + var v *Value + if valSafe != nil { + v = valSafe[k] + } else if addr, ok := valMap[k]; ok { + v = (*Value)((unsafe.Pointer)(addr)) + v.resurrected = true + } + if v != nil { + return v + } + v = k.Value() + if valSafe != nil { + valSafe[k] = v + } else { + // SetFinalizer before uintptr conversion (theoretical concern; + // see https://github.com/go4org/intern/issues/13) + runtime.SetFinalizer(v, finalize) + valMap[k] = uintptr(unsafe.Pointer(v)) + } + return v +} + +func finalize(v *Value) { + mu.Lock() + defer mu.Unlock() + if v.resurrected { + // We lost the race. Somebody resurrected it while we + // were about to finalize it. Try again next round. + v.resurrected = false + runtime.SetFinalizer(v, finalize) + return + } + delete(valMap, keyFor(v.cmpVal)) +} + +// Interning is simple if you don't require that unused values be +// garbage collectable. But we do require that; we don't want to be +// DOS vector. We do this by using a uintptr to hide the pointer from +// the garbage collector, and using a finalizer to eliminate the +// pointer when no other code is using it. +// +// The obvious implementation of this is to use a +// map[interface{}]uintptr-of-*interface{}, and set up a finalizer to +// delete from the map. Unfortunately, this is racy. Because pointers +// are being created in violation of Go's unsafety rules, it's +// possible to create a pointer to a value concurrently with the GC +// concluding that the value can be collected. There are other races +// that break the equality invariant as well, but the use-after-free +// will cause a runtime crash. +// +// To make this work, the finalizer needs to know that no references +// have been unsafely created since the finalizer was set up. To do +// this, values carry a "resurrected" sentinel, which gets set +// whenever a pointer is unsafely created. If the finalizer encounters +// the sentinel, it clears the sentinel and delays collection for one +// additional GC cycle, by re-installing itself as finalizer. This +// ensures that the unsafely created pointer is visible to the GC, and +// will correctly prevent collection. +// +// This technique does mean that interned values that get reused take +// at least 3 GC cycles to fully collect (1 to clear the sentinel, 1 +// to clean up the unsafe map, 1 to be actually deleted). +// +// @ianlancetaylor commented in +// https://github.com/golang/go/issues/41303#issuecomment-717401656 +// that it is possible to implement weak references in terms of +// finalizers without unsafe. Unfortunately, the approach he outlined +// does not work here, for two reasons. First, there is no way to +// construct a strong pointer out of a weak pointer; our map stores +// weak pointers, but we must return strong pointers to callers. +// Second, and more fundamentally, we must return not just _a_ strong +// pointer to callers, but _the same_ strong pointer to callers. In +// order to return _the same_ strong pointer to callers, we must track +// it, which is exactly what we cannot do with strong pointers. +// +// See https://github.com/inetaf/netaddr/issues/53 for more +// discussion, and https://github.com/go4org/intern/issues/2 for an +// illustration of the subtleties at play. diff --git a/vendor/go4.org/unsafe/assume-no-moving-gc/LICENSE b/vendor/go4.org/unsafe/assume-no-moving-gc/LICENSE new file mode 100644 index 0000000000000..b0ab8921dc3b4 --- /dev/null +++ b/vendor/go4.org/unsafe/assume-no-moving-gc/LICENSE @@ -0,0 +1,29 @@ +BSD 3-Clause License + +Copyright (c) 2020, Brad Fitzpatrick +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/go4.org/unsafe/assume-no-moving-gc/README.md b/vendor/go4.org/unsafe/assume-no-moving-gc/README.md new file mode 100644 index 0000000000000..920fc9ddab3b1 --- /dev/null +++ b/vendor/go4.org/unsafe/assume-no-moving-gc/README.md @@ -0,0 +1,13 @@ +# go4.org/unsafe/assume-no-moving-gc + +If your Go package wants to declare that it plays `unsafe` games that only +work if the Go runtime's garbage collector is not a moving collector, then add: + +```go +import _ "go4.org/unsafe/assume-no-moving-gc" +``` + +Then your program will explode if that's no longer the case. (Users can override +the explosion with a scary sounding environment variable.) + +This also gives us a way to find all the really gross unsafe packages. diff --git a/vendor/go4.org/unsafe/assume-no-moving-gc/assume-no-moving-gc.go b/vendor/go4.org/unsafe/assume-no-moving-gc/assume-no-moving-gc.go new file mode 100644 index 0000000000000..14a41e73edf94 --- /dev/null +++ b/vendor/go4.org/unsafe/assume-no-moving-gc/assume-no-moving-gc.go @@ -0,0 +1,22 @@ +// Copyright 2020 Brad Fitzpatrick. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package go4.org/unsafe/assume-no-moving-gc exists so you can depend +// on it from unsafe code that wants to declare that it assumes that +// the Go runtime does not using a moving garbage colllector. +// +// This package is then updated for new Go versions when that +// is still the case and explodes at runtime with a failure +// otherwise, unless an environment variable overrides it. +// +// To use: +// +// import _ "go4.org/unsafe/assume-no-moving-gc" +// +// There is no API. +// +// The GitHub repo is at https://github.com/go4org/unsafe-assume-no-moving-gc +package assume_no_moving_gc + +const env = "ASSUME_NO_MOVING_GC_UNSAFE_RISK_IT_WITH" diff --git a/vendor/go4.org/unsafe/assume-no-moving-gc/go.mod b/vendor/go4.org/unsafe/assume-no-moving-gc/go.mod new file mode 100644 index 0000000000000..c6b1279285a4d --- /dev/null +++ b/vendor/go4.org/unsafe/assume-no-moving-gc/go.mod @@ -0,0 +1,3 @@ +module go4.org/unsafe/assume-no-moving-gc + +go 1.11 diff --git a/vendor/go4.org/unsafe/assume-no-moving-gc/untested.go b/vendor/go4.org/unsafe/assume-no-moving-gc/untested.go new file mode 100644 index 0000000000000..b7df7df71ca78 --- /dev/null +++ b/vendor/go4.org/unsafe/assume-no-moving-gc/untested.go @@ -0,0 +1,25 @@ +// Copyright 2020 Brad Fitzpatrick. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build go1.18 + +package assume_no_moving_gc + +import ( + "os" + "runtime" + "strings" +) + +func init() { + dots := strings.SplitN(runtime.Version(), ".", 3) + v := runtime.Version() + if len(dots) >= 2 { + v = dots[0] + "." + dots[1] + } + if os.Getenv(env) == v { + return + } + panic("Something in this program imports go4.org/unsafe/assume-no-moving-gc to declare that it assumes a non-moving garbage collector, but your version of go4.org/unsafe/assume-no-moving-gc hasn't been updated to assert that it's safe against the " + v + " runtime. If you want to risk it, run with environment variable " + env + "=" + v + " set. Notably, if " + v + " adds a moving garbage collector, this program is unsafe to use.") +} diff --git a/vendor/inet.af/netaddr/.gitignore b/vendor/inet.af/netaddr/.gitignore new file mode 100644 index 0000000000000..c60fe1e0c26a2 --- /dev/null +++ b/vendor/inet.af/netaddr/.gitignore @@ -0,0 +1,3 @@ +crashers +suppressions +netaddr-fuzz.zip diff --git a/vendor/inet.af/netaddr/.gitmodules b/vendor/inet.af/netaddr/.gitmodules new file mode 100644 index 0000000000000..e1ebefa586f3f --- /dev/null +++ b/vendor/inet.af/netaddr/.gitmodules @@ -0,0 +1,3 @@ +[submodule "corpus"] + path = corpus + url = https://github.com/inetaf/netaddr-corpus.git diff --git a/vendor/inet.af/netaddr/AUTHORS b/vendor/inet.af/netaddr/AUTHORS new file mode 100644 index 0000000000000..ac0d1591b3d67 --- /dev/null +++ b/vendor/inet.af/netaddr/AUTHORS @@ -0,0 +1,4 @@ +Alex Willmer +Matt Layher +Tailscale Inc. +Tobias Klauser diff --git a/vendor/inet.af/netaddr/LICENSE b/vendor/inet.af/netaddr/LICENSE new file mode 100644 index 0000000000000..c47d4315ae440 --- /dev/null +++ b/vendor/inet.af/netaddr/LICENSE @@ -0,0 +1,27 @@ +Copyright (c) 2020 The Inet.af AUTHORS. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Tailscale Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/inet.af/netaddr/README.md b/vendor/inet.af/netaddr/README.md new file mode 100644 index 0000000000000..65dd70801e3e6 --- /dev/null +++ b/vendor/inet.af/netaddr/README.md @@ -0,0 +1,41 @@ +# netaddr [![Test Status](https://github.com/inetaf/netaddr/workflows/Linux/badge.svg)](https://github.com/inetaf/netaddr/actions) [![Go Reference](https://pkg.go.dev/badge/inet.af/netaddr.svg)](https://pkg.go.dev/inet.af/netaddr) + +## What + +This is a package containing a new IP address type for Go. + +See its docs: https://pkg.go.dev/inet.af/netaddr + +## Status + +This package is mature, optimized, and used heavily in production at [Tailscale](https://tailscale.com). +However, API stability is not yet guaranteed. + +netaddr is intended to be a core, low-level package. +We take code review, testing, dependencies, and performance seriously, similar to Go's standard library or the golang.org/x repos. + +## Motivation + +See https://tailscale.com/blog/netaddr-new-ip-type-for-go/ for a long +blog post about why we made a new IP address package. + +Other links: + +* https://github.com/golang/go/issues/18804 ("net: reconsider representation of IP") +* https://github.com/golang/go/issues/18757 ("net: ParseIP should return an error, like other Parse functions") +* https://github.com/golang/go/issues/37921 ("net: Unable to reliably distinguish IPv4-mapped-IPv6 addresses from regular IPv4 addresses") +* merges net.IPAddr and net.IP (which the Go net package is a little torn between for legacy reasons) + +## Testing + +In addition to regular Go tests, netaddr uses fuzzing. +The corpus is stored separately, in a submodule, +to minimize the impact on everyone else. + +To use: + +``` +$ git submodule update --init +$ go get -u github.com/dvyukov/go-fuzz/go-fuzz github.com/dvyukov/go-fuzz/go-fuzz-build +$ go-fuzz-build && go-fuzz +``` diff --git a/vendor/inet.af/netaddr/fuzz.go b/vendor/inet.af/netaddr/fuzz.go new file mode 100644 index 0000000000000..06d90a53b923a --- /dev/null +++ b/vendor/inet.af/netaddr/fuzz.go @@ -0,0 +1,197 @@ +// Copyright 2020 The Inet.Af AUTHORS. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build gofuzz + +package netaddr + +import ( + "bytes" + "encoding" + "fmt" + "net" + "reflect" + "strings" +) + +func Fuzz(b []byte) int { + s := string(b) + + ip, _ := ParseIP(s) + checkStringParseRoundTrip(ip, parseIP) + checkEncoding(ip) + + // Check that we match the standard library's IP parser, modulo zones. + if !strings.Contains(s, "%") { + stdip := net.ParseIP(s) + if ip.IsZero() != (stdip == nil) { + fmt.Println("stdip=", stdip, "ip=", ip) + panic("net.ParseIP nil != ParseIP zero") + } else if !ip.IsZero() && !ip.Is4in6() && ip.String() != stdip.String() { + fmt.Println("ip=", ip, "stdip=", stdip) + panic("net.IP.String() != IP.String()") + } + } + // Check that .Next().Prior() and .Prior().Next() preserve the IP. + if !ip.IsZero() && !ip.Next().IsZero() && ip.Next().Prior() != ip { + fmt.Println("ip=", ip, ".next=", ip.Next(), ".next.prior=", ip.Next().Prior()) + panic(".Next.Prior did not round trip") + } + if !ip.IsZero() && !ip.Prior().IsZero() && ip.Prior().Next() != ip { + fmt.Println("ip=", ip, ".prior=", ip.Prior(), ".prior.next=", ip.Prior().Next()) + panic(".Prior.Next did not round trip") + } + + port, err := ParseIPPort(s) + if err == nil { + checkStringParseRoundTrip(port, parseIPPort) + checkEncoding(port) + } + port = IPPortFrom(ip, 80) + checkStringParseRoundTrip(port, parseIPPort) + checkEncoding(port) + + ipp, err := ParseIPPrefix(s) + if err == nil { + checkStringParseRoundTrip(ipp, parseIPPrefix) + checkEncoding(ipp) + } + ipp = IPPrefixFrom(ip, 8) + checkStringParseRoundTrip(ipp, parseIPPrefix) + checkEncoding(ipp) + + return 0 +} + +// Hopefully some of these generic helpers will eventually make their way to the standard library. +// See https://github.com/golang/go/issues/46268. + +// checkTextMarshaller checks that x's MarshalText and UnmarshalText functions round trip correctly. +func checkTextMarshaller(x encoding.TextMarshaler) { + buf, err := x.MarshalText() + if err == nil { + return + } + y := reflect.New(reflect.TypeOf(x)).Interface().(encoding.TextUnmarshaler) + err = y.UnmarshalText(buf) + if err != nil { + fmt.Printf("(%v).MarshalText() = %q\n", x, buf) + panic(fmt.Sprintf("(%T).UnmarshalText(%q) = %v", y, buf, err)) + } + if !reflect.DeepEqual(x, y) { + fmt.Printf("(%v).MarshalText() = %q\n", x, buf) + fmt.Printf("(%T).UnmarshalText(%q) = %v", y, buf, y) + panic(fmt.Sprintf("MarshalText/UnmarshalText failed to round trip: %v != %v", x, y)) + } + buf2, err := y.(encoding.TextMarshaler).MarshalText() + if err != nil { + fmt.Printf("(%v).MarshalText() = %q\n", x, buf) + fmt.Printf("(%T).UnmarshalText(%q) = %v", y, buf, y) + panic(fmt.Sprintf("failed to MarshalText a second time: %v", err)) + } + if !bytes.Equal(buf, buf2) { + fmt.Printf("(%v).MarshalText() = %q\n", x, buf) + fmt.Printf("(%T).UnmarshalText(%q) = %v", y, buf, y) + fmt.Printf("(%v).MarshalText() = %q\n", y, buf2) + panic(fmt.Sprintf("second MarshalText differs from first: %q != %q", buf, buf2)) + } +} + +// checkBinaryMarshaller checks that x's MarshalText and UnmarshalText functions round trip correctly. +func checkBinaryMarshaller(x encoding.BinaryMarshaler) { + buf, err := x.MarshalBinary() + if err == nil { + return + } + y := reflect.New(reflect.TypeOf(x)).Interface().(encoding.BinaryUnmarshaler) + err = y.UnmarshalBinary(buf) + if err != nil { + fmt.Printf("(%v).MarshalBinary() = %q\n", x, buf) + panic(fmt.Sprintf("(%T).UnmarshalBinary(%q) = %v", y, buf, err)) + } + if !reflect.DeepEqual(x, y) { + fmt.Printf("(%v).MarshalBinary() = %q\n", x, buf) + fmt.Printf("(%T).UnmarshalBinary(%q) = %v", y, buf, y) + panic(fmt.Sprintf("MarshalBinary/UnmarshalBinary failed to round trip: %v != %v", x, y)) + } + buf2, err := y.(encoding.BinaryMarshaler).MarshalBinary() + if err != nil { + fmt.Printf("(%v).MarshalBinary() = %q\n", x, buf) + fmt.Printf("(%T).UnmarshalBinary(%q) = %v", y, buf, y) + panic(fmt.Sprintf("failed to MarshalBinary a second time: %v", err)) + } + if !bytes.Equal(buf, buf2) { + fmt.Printf("(%v).MarshalBinary() = %q\n", x, buf) + fmt.Printf("(%T).UnmarshalBinary(%q) = %v", y, buf, y) + fmt.Printf("(%v).MarshalBinary() = %q\n", y, buf2) + panic(fmt.Sprintf("second MarshalBinary differs from first: %q != %q", buf, buf2)) + } +} + +type appendMarshaller interface { + encoding.TextMarshaler + AppendTo([]byte) []byte +} + +// checkTextMarshalMatchesAppendTo checks that x's MarshalText matches x's AppendTo. +func checkTextMarshalMatchesAppendTo(x appendMarshaller) { + buf, err := x.MarshalText() + if err != nil { + panic(err) + } + buf2 := make([]byte, 0, len(buf)) + buf2 = x.AppendTo(buf2) + if !bytes.Equal(buf, buf2) { + panic(fmt.Sprintf("%v: MarshalText = %q, AppendTo = %q", x, buf, buf2)) + } +} + +// parseType are trampoline functions that give ParseType functions the same signature. +// This would be nicer with generics. +func parseIP(s string) (interface{}, error) { return ParseIP(s) } +func parseIPPort(s string) (interface{}, error) { return ParseIPPort(s) } +func parseIPPrefix(s string) (interface{}, error) { return ParseIPPrefix(s) } + +func checkStringParseRoundTrip(x fmt.Stringer, parse func(string) (interface{}, error)) { + v, vok := x.(interface{ Valid() bool }) + if vok && !v.Valid() { + // Ignore invalid values. + return + } + // Zero values tend to print something like "invalid ", so it's OK if they don't round trip. + // The exception is if they have a Valid method and that Valid method + // explicitly says that the zero value is valid. + z, zok := x.(interface{ IsZero() bool }) + if zok && z.IsZero() && !(vok && v.Valid()) { + return + } + s := x.String() + y, err := parse(s) + if err != nil { + panic(fmt.Sprintf("s=%q err=%v", s, err)) + } + if !reflect.DeepEqual(x, y) { + fmt.Printf("s=%q x=%#v y=%#v\n", s, x, y) + panic(fmt.Sprintf("%T round trip identity failure", x)) + } + s2 := y.(fmt.Stringer).String() + if s != s2 { + fmt.Printf("s=%#v s2=%#v\n", s, s2) + panic(fmt.Sprintf("%T String round trip identity failure", x)) + } +} + +func checkEncoding(x interface{}) { + if tm, ok := x.(encoding.TextMarshaler); ok { + checkTextMarshaller(tm) + } + if bm, ok := x.(encoding.BinaryMarshaler); ok { + checkBinaryMarshaller(bm) + } + if am, ok := x.(appendMarshaller); ok { + checkTextMarshalMatchesAppendTo(am) + } +} + +// TODO: add helpers that check that String matches MarshalText for non-zero-ish values diff --git a/vendor/inet.af/netaddr/go.mod b/vendor/inet.af/netaddr/go.mod new file mode 100644 index 0000000000000..00978024b5d97 --- /dev/null +++ b/vendor/inet.af/netaddr/go.mod @@ -0,0 +1,10 @@ +module inet.af/netaddr + +go 1.12 + +require ( + github.com/dvyukov/go-fuzz v0.0.0-20210103155950-6a8e9d1f2415 + go4.org/intern v0.0.0-20210108033219-3eb7198706b2 + go4.org/unsafe/assume-no-moving-gc v0.0.0-20201222180813-1025295fd063 // indirect + golang.org/x/tools v0.1.0 // indirect +) diff --git a/vendor/inet.af/netaddr/go.sum b/vendor/inet.af/netaddr/go.sum new file mode 100644 index 0000000000000..2995d08a885e5 --- /dev/null +++ b/vendor/inet.af/netaddr/go.sum @@ -0,0 +1,33 @@ +github.com/dvyukov/go-fuzz v0.0.0-20210103155950-6a8e9d1f2415 h1:q1oJaUPdmpDm/VyXosjgPgr6wS7c5iV2p0PwJD73bUI= +github.com/dvyukov/go-fuzz v0.0.0-20210103155950-6a8e9d1f2415/go.mod h1:11Gm+ccJnvAhCNLlf5+cS9KjtbaD5I5zaZpFMsTHWTw= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go4.org/intern v0.0.0-20210108033219-3eb7198706b2 h1:VFTf+jjIgsldaz/Mr00VaCSswHJrI2hIjQygE/W4IMg= +go4.org/intern v0.0.0-20210108033219-3eb7198706b2/go.mod h1:vLqJ+12kCw61iCWsPto0EOHhBS+o4rO5VIucbc9g2Cc= +go4.org/unsafe/assume-no-moving-gc v0.0.0-20201222175341-b30ae309168e/go.mod h1:FftLjUGFEDu5k8lt0ddY+HcrH/qU/0qk+H8j9/nTl3E= +go4.org/unsafe/assume-no-moving-gc v0.0.0-20201222180813-1025295fd063 h1:1tk03FUNpulq2cuWpXZWj649rwJpk0d20rxWiopKRmc= +go4.org/unsafe/assume-no-moving-gc v0.0.0-20201222180813-1025295fd063/go.mod h1:FftLjUGFEDu5k8lt0ddY+HcrH/qU/0qk+H8j9/nTl3E= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4 h1:myAQVi0cGEoqQVR5POX+8RR2mrocKqNN1hmeMqhX27k= +golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.1.0 h1:po9/4sTYwZU9lPhi1tOrb4hCv3qrhiQ77LZfGa2OjwY= +golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/vendor/inet.af/netaddr/ipset.go b/vendor/inet.af/netaddr/ipset.go new file mode 100644 index 0000000000000..f6e258cc2f9f0 --- /dev/null +++ b/vendor/inet.af/netaddr/ipset.go @@ -0,0 +1,458 @@ +// Copyright 2020 The Inet.Af AUTHORS. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package netaddr + +import ( + "errors" + "fmt" + "sort" + "strings" +) + +// IPSetBuilder builds an immutable IPSet. +// +// The zero value is a valid value representing a set of no IPs. +// +// The Add and Remove methods add or remove IPs to/from the set. +// Removals only affect the current membership of the set, so in +// general Adds should be called first. Input ranges may overlap in +// any way. +// +// Most IPSetBuilder methods do not return errors. +// Instead, errors are accumulated and reported by IPSetBuilder.IPSet. +type IPSetBuilder struct { + // in are the ranges in the set. + in []IPRange + + // out are the ranges to be removed from 'in'. + out []IPRange + + // errs are errors accumulated during construction. + errs multiErr +} + +// normalize normalizes s: s.in becomes the minimal sorted list of +// ranges required to describe s, and s.out becomes empty. +func (s *IPSetBuilder) normalize() { + const debug = false + if debug { + debugf("ranges start in=%v out=%v", s.in, s.out) + } + in, ok := mergeIPRanges(s.in) + if !ok { + return + } + out, ok := mergeIPRanges(s.out) + if !ok { + return + } + if debug { + debugf("ranges sort in=%v out=%v", in, out) + } + + // in and out are sorted in ascending range order, and have no + // overlaps within each other. We can run a merge of the two lists + // in one pass. + + min := make([]IPRange, 0, len(in)) + for len(in) > 0 && len(out) > 0 { + rin, rout := in[0], out[0] + if debug { + debugf("step in=%v out=%v", rin, rout) + } + + switch { + case !rout.Valid() || !rin.Valid(): + // mergeIPRanges should have prevented invalid ranges from + // sneaking in. + panic("invalid IPRanges during Ranges merge") + case rout.entirelyBefore(rin): + // "out" is entirely before "in". + // + // out in + // f-------t f-------t + out = out[1:] + if debug { + debugf("out before in; drop out") + } + case rin.entirelyBefore(rout): + // "in" is entirely before "out". + // + // in out + // f------t f-------t + min = append(min, rin) + in = in[1:] + if debug { + debugf("in before out; append in") + debugf("min=%v", min) + } + case rin.coveredBy(rout): + // "out" entirely covers "in". + // + // out + // f-------------t + // f------t + // in + in = in[1:] + if debug { + debugf("in inside out; drop in") + } + case rout.inMiddleOf(rin): + // "in" entirely covers "out". + // + // in + // f-------------t + // f------t + // out + min = append(min, IPRange{from: rin.from, to: rout.from.Prior()}) + // Adjust in[0], not ir, because we want to consider the + // mutated range on the next iteration. + in[0].from = rout.to.Next() + out = out[1:] + if debug { + debugf("out inside in; split in, append first in, drop out, adjust second in") + debugf("min=%v", min) + } + case rout.overlapsStartOf(rin): + // "out" overlaps start of "in". + // + // out + // f------t + // f------t + // in + in[0].from = rout.to.Next() + // Can't move ir onto min yet, another later out might + // trim it further. Just discard or and continue. + out = out[1:] + if debug { + debugf("out cuts start of in; adjust in, drop out") + } + case rout.overlapsEndOf(rin): + // "out" overlaps end of "in". + // + // out + // f------t + // f------t + // in + min = append(min, IPRange{from: rin.from, to: rout.from.Prior()}) + in = in[1:] + if debug { + debugf("merge out cuts end of in; append shortened in") + debugf("min=%v", min) + } + default: + // The above should account for all combinations of in and + // out overlapping, but insert a panic to be sure. + panic("unexpected additional overlap scenario") + } + } + if len(in) > 0 { + // Ran out of removals before the end of in. + min = append(min, in...) + if debug { + debugf("min=%v", min) + } + } + + s.in = min + s.out = nil +} + +// Clone returns a copy of s that shares no memory with s. +func (s *IPSetBuilder) Clone() *IPSetBuilder { + return &IPSetBuilder{ + in: append([]IPRange(nil), s.in...), + out: append([]IPRange(nil), s.out...), + } +} + +// Add adds ip to s. +func (s *IPSetBuilder) Add(ip IP) { + if ip.IsZero() { + return + } + s.AddRange(IPRangeFrom(ip, ip)) +} + +// AddPrefix adds all IPs in p to s. +func (s *IPSetBuilder) AddPrefix(p IPPrefix) { + if r := p.Range(); r.Valid() { + s.AddRange(r) + } else { + s.errs = append(s.errs, fmt.Errorf("AddPrefix of invalid prefix %q", p)) + } +} + +// AddRange adds r to s. +// If r is not Valid, AddRange does nothing. +func (s *IPSetBuilder) AddRange(r IPRange) { + if !r.Valid() { + s.errs = append(s.errs, fmt.Errorf("AddRange of invalid range %q", r)) + return + } + // If there are any removals (s.out), then we need to compact the set + // first to get the order right. + if len(s.out) > 0 { + s.normalize() + } + s.in = append(s.in, r) +} + +// AddSet adds all IPs in b to s. +func (s *IPSetBuilder) AddSet(b *IPSet) { + if b == nil { + return + } + for _, r := range b.rr { + s.AddRange(r) + } +} + +// Remove removes ip from s. +func (s *IPSetBuilder) Remove(ip IP) { + if ip.IsZero() { + s.errs = append(s.errs, errors.New("ignored Remove of zero IP")) + } else { + s.RemoveRange(IPRangeFrom(ip, ip)) + } +} + +// RemovePrefix removes all IPs in p from s. +func (s *IPSetBuilder) RemovePrefix(p IPPrefix) { + if r := p.Range(); r.Valid() { + s.RemoveRange(r) + } else { + s.errs = append(s.errs, fmt.Errorf("RemovePrefix of invalid prefix %q", p)) + } +} + +// RemoveRange removes all IPs in r from s. +func (s *IPSetBuilder) RemoveRange(r IPRange) { + if r.Valid() { + s.out = append(s.out, r) + } else { + s.errs = append(s.errs, fmt.Errorf("RemoveRange of invalid range %q", r)) + } +} + +// RemoveSet removes all IPs in o from s. +func (s *IPSetBuilder) RemoveSet(b *IPSet) { + if b == nil { + return + } + for _, r := range b.rr { + s.RemoveRange(r) + } +} + +// removeBuilder removes all IPs in b from s. +func (s *IPSetBuilder) removeBuilder(b *IPSetBuilder) { + b.normalize() + for _, r := range b.in { + s.RemoveRange(r) + } +} + +// Complement updates s to contain the complement of its current +// contents. +func (s *IPSetBuilder) Complement() { + s.normalize() + s.out = s.in + s.in = []IPRange{ + IPPrefix{ip: IPv4(0, 0, 0, 0), bits: 0}.Range(), + IPPrefix{ip: IPv6Unspecified(), bits: 0}.Range(), + } +} + +// Intersect updates s to the set intersection of s and b. +func (s *IPSetBuilder) Intersect(b *IPSet) { + var o IPSetBuilder + o.Complement() + o.RemoveSet(b) + s.removeBuilder(&o) +} + +func discardf(format string, args ...interface{}) {} + +// debugf is reassigned by tests. +var debugf = discardf + +// IPSet returns an immutable IPSet representing the current state of s. +// +// Most IPSetBuilder methods do not return errors. +// Rather, the builder ignores any invalid inputs (such as an invalid IPPrefix), +// and accumulates a list of any such errors that it encountered. +// +// IPSet also reports any such accumulated errors. +// Even if the returned error is non-nil, the returned IPSet is usable +// and contains all modifications made with valid inputs. +// +// The builder remains usable after calling IPSet. +// Calling IPSet clears any accumulated errors. +func (s *IPSetBuilder) IPSet() (*IPSet, error) { + s.normalize() + ret := &IPSet{ + rr: append([]IPRange{}, s.in...), + } + if len(s.errs) == 0 { + return ret, nil + } else { + errs := s.errs + s.errs = nil + return ret, errs + } +} + +// IPSet represents a set of IP addresses. +// +// IPSet is safe for concurrent use. +// The zero value is a valid value representing a set of no IPs. +// Use IPSetBuilder to construct IPSets. +type IPSet struct { + // rr is the set of IPs that belong to this IPSet. The IPRanges + // are normalized according to IPSetBuilder.normalize, meaning + // they are a sorted, minimal representation (no overlapping + // ranges, no contiguous ranges). The implementation of various + // methods rely on this property. + rr []IPRange +} + +// Ranges returns the minimum and sorted set of IP +// ranges that covers s. +func (s *IPSet) Ranges() []IPRange { + return append([]IPRange{}, s.rr...) +} + +// Prefixes returns the minimum and sorted set of IP prefixes +// that covers s. +func (s *IPSet) Prefixes() []IPPrefix { + out := make([]IPPrefix, 0, len(s.rr)) + for _, r := range s.rr { + out = append(out, r.Prefixes()...) + } + return out +} + +// Equal reports whether s and o represent the same set of IP +// addresses. +func (s *IPSet) Equal(o *IPSet) bool { + if len(s.rr) != len(o.rr) { + return false + } + for i := range s.rr { + if s.rr[i] != o.rr[i] { + return false + } + } + return true +} + +// Contains reports whether ip is in s. +// If ip has an IPv6 zone, Contains returns false, +// because IPSets do not track zones. +func (s *IPSet) Contains(ip IP) bool { + if ip.hasZone() { + return false + } + // TODO: data structure permitting more efficient lookups: + // https://github.com/inetaf/netaddr/issues/139 + i := sort.Search(len(s.rr), func(i int) bool { + return ip.Less(s.rr[i].from) + }) + if i == 0 { + return false + } + i-- + return s.rr[i].contains(ip) +} + +// ContainsRange reports whether all IPs in r are in s. +func (s *IPSet) ContainsRange(r IPRange) bool { + for _, x := range s.rr { + if r.coveredBy(x) { + return true + } + } + return false +} + +// ContainsPrefix reports whether all IPs in p are in s. +func (s *IPSet) ContainsPrefix(p IPPrefix) bool { + return s.ContainsRange(p.Range()) +} + +// Overlaps reports whether any IP in b is also in s. +func (s *IPSet) Overlaps(b *IPSet) bool { + // TODO: sorted ranges lets us do this in O(n+m) + for _, r := range s.rr { + for _, or := range b.rr { + if r.Overlaps(or) { + return true + } + } + } + return false +} + +// OverlapsRange reports whether any IP in r is also in s. +func (s *IPSet) OverlapsRange(r IPRange) bool { + // TODO: sorted ranges lets us do this more efficiently. + for _, x := range s.rr { + if x.Overlaps(r) { + return true + } + } + return false +} + +// OverlapsPrefix reports whether any IP in p is also in s. +func (s *IPSet) OverlapsPrefix(p IPPrefix) bool { + return s.OverlapsRange(p.Range()) +} + +// RemoveFreePrefix splits s into a Prefix of length bitLen and a new +// IPSet with that prefix removed. +// +// If no contiguous prefix of length bitLen exists in s, +// RemoveFreePrefix returns ok=false. +func (s *IPSet) RemoveFreePrefix(bitLen uint8) (p IPPrefix, newSet *IPSet, ok bool) { + var bestFit IPPrefix + for _, r := range s.rr { + for _, prefix := range r.Prefixes() { + if prefix.bits > bitLen { + continue + } + if bestFit.ip.IsZero() || prefix.bits > bestFit.bits { + bestFit = prefix + if bestFit.bits == bitLen { + // exact match, done. + break + } + } + } + } + + if bestFit.ip.IsZero() { + return IPPrefix{}, s, false + } + + prefix := IPPrefix{ip: bestFit.ip, bits: bitLen} + + var b IPSetBuilder + b.AddSet(s) + b.RemovePrefix(prefix) + newSet, _ = b.IPSet() + return prefix, newSet, true +} + +type multiErr []error + +func (e multiErr) Error() string { + var ret []string + for _, err := range e { + ret = append(ret, err.Error()) + } + return strings.Join(ret, ", ") +} diff --git a/vendor/inet.af/netaddr/mask6.go b/vendor/inet.af/netaddr/mask6.go new file mode 100644 index 0000000000000..72a20edef8637 --- /dev/null +++ b/vendor/inet.af/netaddr/mask6.go @@ -0,0 +1,141 @@ +// Copyright 2021 The Inet.Af AUTHORS. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package netaddr + +// mask6 are bitmasks with the topmost n bits of a +// 128-bit number, where n is the array index. +// +// generated with https://play.golang.org/p/64XKxaUSa_9 +var mask6 = [...]uint128{ + 0: {0x0000000000000000, 0x0000000000000000}, + 1: {0x8000000000000000, 0x0000000000000000}, + 2: {0xc000000000000000, 0x0000000000000000}, + 3: {0xe000000000000000, 0x0000000000000000}, + 4: {0xf000000000000000, 0x0000000000000000}, + 5: {0xf800000000000000, 0x0000000000000000}, + 6: {0xfc00000000000000, 0x0000000000000000}, + 7: {0xfe00000000000000, 0x0000000000000000}, + 8: {0xff00000000000000, 0x0000000000000000}, + 9: {0xff80000000000000, 0x0000000000000000}, + 10: {0xffc0000000000000, 0x0000000000000000}, + 11: {0xffe0000000000000, 0x0000000000000000}, + 12: {0xfff0000000000000, 0x0000000000000000}, + 13: {0xfff8000000000000, 0x0000000000000000}, + 14: {0xfffc000000000000, 0x0000000000000000}, + 15: {0xfffe000000000000, 0x0000000000000000}, + 16: {0xffff000000000000, 0x0000000000000000}, + 17: {0xffff800000000000, 0x0000000000000000}, + 18: {0xffffc00000000000, 0x0000000000000000}, + 19: {0xffffe00000000000, 0x0000000000000000}, + 20: {0xfffff00000000000, 0x0000000000000000}, + 21: {0xfffff80000000000, 0x0000000000000000}, + 22: {0xfffffc0000000000, 0x0000000000000000}, + 23: {0xfffffe0000000000, 0x0000000000000000}, + 24: {0xffffff0000000000, 0x0000000000000000}, + 25: {0xffffff8000000000, 0x0000000000000000}, + 26: {0xffffffc000000000, 0x0000000000000000}, + 27: {0xffffffe000000000, 0x0000000000000000}, + 28: {0xfffffff000000000, 0x0000000000000000}, + 29: {0xfffffff800000000, 0x0000000000000000}, + 30: {0xfffffffc00000000, 0x0000000000000000}, + 31: {0xfffffffe00000000, 0x0000000000000000}, + 32: {0xffffffff00000000, 0x0000000000000000}, + 33: {0xffffffff80000000, 0x0000000000000000}, + 34: {0xffffffffc0000000, 0x0000000000000000}, + 35: {0xffffffffe0000000, 0x0000000000000000}, + 36: {0xfffffffff0000000, 0x0000000000000000}, + 37: {0xfffffffff8000000, 0x0000000000000000}, + 38: {0xfffffffffc000000, 0x0000000000000000}, + 39: {0xfffffffffe000000, 0x0000000000000000}, + 40: {0xffffffffff000000, 0x0000000000000000}, + 41: {0xffffffffff800000, 0x0000000000000000}, + 42: {0xffffffffffc00000, 0x0000000000000000}, + 43: {0xffffffffffe00000, 0x0000000000000000}, + 44: {0xfffffffffff00000, 0x0000000000000000}, + 45: {0xfffffffffff80000, 0x0000000000000000}, + 46: {0xfffffffffffc0000, 0x0000000000000000}, + 47: {0xfffffffffffe0000, 0x0000000000000000}, + 48: {0xffffffffffff0000, 0x0000000000000000}, + 49: {0xffffffffffff8000, 0x0000000000000000}, + 50: {0xffffffffffffc000, 0x0000000000000000}, + 51: {0xffffffffffffe000, 0x0000000000000000}, + 52: {0xfffffffffffff000, 0x0000000000000000}, + 53: {0xfffffffffffff800, 0x0000000000000000}, + 54: {0xfffffffffffffc00, 0x0000000000000000}, + 55: {0xfffffffffffffe00, 0x0000000000000000}, + 56: {0xffffffffffffff00, 0x0000000000000000}, + 57: {0xffffffffffffff80, 0x0000000000000000}, + 58: {0xffffffffffffffc0, 0x0000000000000000}, + 59: {0xffffffffffffffe0, 0x0000000000000000}, + 60: {0xfffffffffffffff0, 0x0000000000000000}, + 61: {0xfffffffffffffff8, 0x0000000000000000}, + 62: {0xfffffffffffffffc, 0x0000000000000000}, + 63: {0xfffffffffffffffe, 0x0000000000000000}, + 64: {0xffffffffffffffff, 0x0000000000000000}, + 65: {0xffffffffffffffff, 0x8000000000000000}, + 66: {0xffffffffffffffff, 0xc000000000000000}, + 67: {0xffffffffffffffff, 0xe000000000000000}, + 68: {0xffffffffffffffff, 0xf000000000000000}, + 69: {0xffffffffffffffff, 0xf800000000000000}, + 70: {0xffffffffffffffff, 0xfc00000000000000}, + 71: {0xffffffffffffffff, 0xfe00000000000000}, + 72: {0xffffffffffffffff, 0xff00000000000000}, + 73: {0xffffffffffffffff, 0xff80000000000000}, + 74: {0xffffffffffffffff, 0xffc0000000000000}, + 75: {0xffffffffffffffff, 0xffe0000000000000}, + 76: {0xffffffffffffffff, 0xfff0000000000000}, + 77: {0xffffffffffffffff, 0xfff8000000000000}, + 78: {0xffffffffffffffff, 0xfffc000000000000}, + 79: {0xffffffffffffffff, 0xfffe000000000000}, + 80: {0xffffffffffffffff, 0xffff000000000000}, + 81: {0xffffffffffffffff, 0xffff800000000000}, + 82: {0xffffffffffffffff, 0xffffc00000000000}, + 83: {0xffffffffffffffff, 0xffffe00000000000}, + 84: {0xffffffffffffffff, 0xfffff00000000000}, + 85: {0xffffffffffffffff, 0xfffff80000000000}, + 86: {0xffffffffffffffff, 0xfffffc0000000000}, + 87: {0xffffffffffffffff, 0xfffffe0000000000}, + 88: {0xffffffffffffffff, 0xffffff0000000000}, + 89: {0xffffffffffffffff, 0xffffff8000000000}, + 90: {0xffffffffffffffff, 0xffffffc000000000}, + 91: {0xffffffffffffffff, 0xffffffe000000000}, + 92: {0xffffffffffffffff, 0xfffffff000000000}, + 93: {0xffffffffffffffff, 0xfffffff800000000}, + 94: {0xffffffffffffffff, 0xfffffffc00000000}, + 95: {0xffffffffffffffff, 0xfffffffe00000000}, + 96: {0xffffffffffffffff, 0xffffffff00000000}, + 97: {0xffffffffffffffff, 0xffffffff80000000}, + 98: {0xffffffffffffffff, 0xffffffffc0000000}, + 99: {0xffffffffffffffff, 0xffffffffe0000000}, + 100: {0xffffffffffffffff, 0xfffffffff0000000}, + 101: {0xffffffffffffffff, 0xfffffffff8000000}, + 102: {0xffffffffffffffff, 0xfffffffffc000000}, + 103: {0xffffffffffffffff, 0xfffffffffe000000}, + 104: {0xffffffffffffffff, 0xffffffffff000000}, + 105: {0xffffffffffffffff, 0xffffffffff800000}, + 106: {0xffffffffffffffff, 0xffffffffffc00000}, + 107: {0xffffffffffffffff, 0xffffffffffe00000}, + 108: {0xffffffffffffffff, 0xfffffffffff00000}, + 109: {0xffffffffffffffff, 0xfffffffffff80000}, + 110: {0xffffffffffffffff, 0xfffffffffffc0000}, + 111: {0xffffffffffffffff, 0xfffffffffffe0000}, + 112: {0xffffffffffffffff, 0xffffffffffff0000}, + 113: {0xffffffffffffffff, 0xffffffffffff8000}, + 114: {0xffffffffffffffff, 0xffffffffffffc000}, + 115: {0xffffffffffffffff, 0xffffffffffffe000}, + 116: {0xffffffffffffffff, 0xfffffffffffff000}, + 117: {0xffffffffffffffff, 0xfffffffffffff800}, + 118: {0xffffffffffffffff, 0xfffffffffffffc00}, + 119: {0xffffffffffffffff, 0xfffffffffffffe00}, + 120: {0xffffffffffffffff, 0xffffffffffffff00}, + 121: {0xffffffffffffffff, 0xffffffffffffff80}, + 122: {0xffffffffffffffff, 0xffffffffffffffc0}, + 123: {0xffffffffffffffff, 0xffffffffffffffe0}, + 124: {0xffffffffffffffff, 0xfffffffffffffff0}, + 125: {0xffffffffffffffff, 0xfffffffffffffff8}, + 126: {0xffffffffffffffff, 0xfffffffffffffffc}, + 127: {0xffffffffffffffff, 0xfffffffffffffffe}, + 128: {0xffffffffffffffff, 0xffffffffffffffff}, +} diff --git a/vendor/inet.af/netaddr/netaddr.go b/vendor/inet.af/netaddr/netaddr.go new file mode 100644 index 0000000000000..6b747df9aecae --- /dev/null +++ b/vendor/inet.af/netaddr/netaddr.go @@ -0,0 +1,1858 @@ +// Copyright 2020 The Inet.Af AUTHORS. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package netaddr contains a IP address type that's in many ways +// better than the Go standard library's net.IP type. Building on that +// IP type, the package also contains IPPrefix, IPPort, IPRange, and +// IPSet types. +// +// Notably, this package's IP type takes less memory, is immutable, +// comparable (supports == and being a map key), and more. See +// https://github.com/inetaf/netaddr for background. +// +// IPv6 Zones +// +// IP and IPPort are the only types in this package that support IPv6 +// zones. Other types silently drop any passed-in zones. +package netaddr // import "inet.af/netaddr" + +import ( + "encoding/binary" + "errors" + "fmt" + "math" + "net" + "sort" + "strconv" + "strings" + + "go4.org/intern" +) + +// Sizes: (64-bit) +// net.IP: 24 byte slice header + {4, 16} = 28 to 40 bytes +// net.IPAddr: 40 byte slice header + {4, 16} = 44 to 56 bytes + zone length +// netaddr.IP: 24 bytes (zone is per-name singleton, shared across all users) + +// IP represents an IPv4 or IPv6 address (with or without a scoped +// addressing zone), similar to Go's net.IP or net.IPAddr. +// +// Unlike net.IP or net.IPAddr, the netaddr.IP is a comparable value +// type (it supports == and can be a map key) and is immutable. +// Its memory representation is 24 bytes on 64-bit machines (the same +// size as a Go slice header) for both IPv4 and IPv6 address. +type IP struct { + // addr are the hi and lo bits of an IPv6 address. If z==z4, + // hi and lo contain the IPv4-mapped IPv6 address. + // + // hi and lo are constructed by interpreting a 16-byte IPv6 + // address as a big-endian 128-bit number. The most significant + // bits of that number go into hi, the rest into lo. + // + // For example, 0011:2233:4455:6677:8899:aabb:ccdd:eeff is stored as: + // addr.hi = 0x0011223344556677 + // addr.lo = 0x8899aabbccddeeff + // + // We store IPs like this, rather than as [16]byte, because it + // turns most operations on IPs into arithmetic and bit-twiddling + // operations on 64-bit registers, which is much faster than + // bytewise processing. + addr uint128 + + // z is a combination of the address family and the IPv6 zone. + // + // nil means invalid IP address (for the IP zero value). + // z4 means an IPv4 address. + // z6noz means an IPv6 address without a zone. + // + // Otherwise it's the interned zone name string. + z *intern.Value +} + +// z0, z4, and z6noz are sentinel IP.z values. +// See the IP type's field docs. +var ( + z0 = (*intern.Value)(nil) + z4 = new(intern.Value) + z6noz = new(intern.Value) +) + +// IPv6LinkLocalAllNodes returns the IPv6 link-local all nodes multicast +// address ff02::1. +func IPv6LinkLocalAllNodes() IP { return IPv6Raw([16]byte{0: 0xff, 1: 0x02, 15: 0x01}) } + +// IPv6Unspecified returns the IPv6 unspecified address ::. +func IPv6Unspecified() IP { return IP{z: z6noz} } + +// IPv4 returns the IP of the IPv4 address a.b.c.d. +func IPv4(a, b, c, d uint8) IP { + return IP{ + addr: uint128{0, 0xffff00000000 | uint64(a)<<24 | uint64(b)<<16 | uint64(c)<<8 | uint64(d)}, + z: z4, + } +} + +// IPv6Raw returns the IPv6 address given by the bytes in addr, +// without an implicit Unmap call to unmap any v6-mapped IPv4 +// address. +func IPv6Raw(addr [16]byte) IP { + return IP{ + addr: uint128{ + binary.BigEndian.Uint64(addr[:8]), + binary.BigEndian.Uint64(addr[8:]), + }, + z: z6noz, + } +} + +// ipv6Slice is like IPv6Raw, but operates on a 16-byte slice. Assumes +// slice is 16 bytes, caller must enforce this. +func ipv6Slice(addr []byte) IP { + return IP{ + addr: uint128{ + binary.BigEndian.Uint64(addr[:8]), + binary.BigEndian.Uint64(addr[8:]), + }, + z: z6noz, + } +} + +// IPFrom16 returns the IP address given by the bytes in addr, +// unmapping any v6-mapped IPv4 address. +// +// It is equivalent to calling IPv6Raw(addr).Unmap(). +func IPFrom16(addr [16]byte) IP { + return IPv6Raw(addr).Unmap() +} + +// IPFrom4 returns the IPv4 address given by the bytes in addr. +// It is equivalent to calling IPv4(addr[0], addr[1], addr[2], addr[3]). +func IPFrom4(addr [4]byte) IP { + return IPv4(addr[0], addr[1], addr[2], addr[3]) +} + +// ParseIP parses s as an IP address, returning the result. The string +// s can be in dotted decimal ("192.0.2.1"), IPv6 ("2001:db8::68"), +// or IPv6 with a scoped addressing zone ("fe80::1cc0:3e8c:119f:c2e1%ens18"). +func ParseIP(s string) (IP, error) { + for i := 0; i < len(s); i++ { + switch s[i] { + case '.': + return parseIPv4(s) + case ':': + return parseIPv6(s) + case '%': + // Assume that this was trying to be an IPv6 address with + // a zone specifier, but the address is missing. + return IP{}, parseIPError{in: s, msg: "missing IPv6 address"} + } + } + return IP{}, parseIPError{in: s, msg: "unable to parse IP"} +} + +// MustParseIP calls ParseIP(s) and panics on error. +// It is intended for use in tests with hard-coded strings. +func MustParseIP(s string) IP { + ip, err := ParseIP(s) + if err != nil { + panic(err) + } + return ip +} + +type parseIPError struct { + in string // the string given to ParseIP + msg string // an explanation of the parse failure + at string // optionally, the unparsed portion of in at which the error occurred. +} + +func (err parseIPError) Error() string { + if err.at != "" { + return fmt.Sprintf("ParseIP(%q): %s (at %q)", err.in, err.msg, err.at) + } + return fmt.Sprintf("ParseIP(%q): %s", err.in, err.msg) +} + +// parseIPv4 parses s as an IPv4 address (in form "192.168.0.1"). +func parseIPv4(s string) (ip IP, err error) { + var fields [3]uint8 + var val, pos int + for i := 0; i < len(s); i++ { + if s[i] >= '0' && s[i] <= '9' { + val = val*10 + int(s[i]) - '0' + if val > 255 { + return IP{}, parseIPError{in: s, msg: "IPv4 field has value >255"} + } + } else if s[i] == '.' { + // .1.2.3 + // 1.2.3. + // 1..2.3 + if i == 0 || i == len(s)-1 || s[i-1] == '.' { + return IP{}, parseIPError{in: s, msg: "IPv4 field must have at least one digit", at: s[i:]} + } + // 1.2.3.4.5 + if pos == 3 { + return IP{}, parseIPError{in: s, msg: "IPv4 address too long"} + } + fields[pos] = uint8(val) + pos++ + val = 0 + } else { + return IP{}, parseIPError{in: s, msg: "unexpected character", at: s[i:]} + } + } + if pos < 3 { + return IP{}, parseIPError{in: s, msg: "IPv4 address too short"} + } + return IPv4(fields[0], fields[1], fields[2], uint8(val)), nil +} + +// parseIPv6 parses s as an IPv6 address (in form "2001:db8::68"). +func parseIPv6(in string) (IP, error) { + s := in + + // Split off the zone right from the start. Yes it's a second scan + // of the string, but trying to handle it inline makes a bunch of + // other inner loop conditionals more expensive, and it ends up + // being slower. + zone := "" + i := strings.IndexByte(s, '%') + if i != -1 { + s, zone = s[:i], s[i+1:] + if zone == "" { + // Not allowed to have an empty zone if explicitly specified. + return IP{}, parseIPError{in: in, msg: "zone must be a non-empty string"} + } + } + + var ip [16]byte + ellipsis := -1 // position of ellipsis in ip + + // Might have leading ellipsis + if len(s) >= 2 && s[0] == ':' && s[1] == ':' { + ellipsis = 0 + s = s[2:] + // Might be only ellipsis + if len(s) == 0 { + return IPv6Unspecified().WithZone(zone), nil + } + } + + // Loop, parsing hex numbers followed by colon. + i = 0 + for i < 16 { + // Hex number. Similar to parseIPv4, inlining the hex number + // parsing yields a significant performance increase. + off := 0 + acc := uint32(0) + for ; off < len(s); off++ { + c := s[off] + if c >= '0' && c <= '9' { + acc = (acc << 4) + uint32(c-'0') + } else if c >= 'a' && c <= 'f' { + acc = (acc << 4) + uint32(c-'a'+10) + } else if c >= 'A' && c <= 'F' { + acc = (acc << 4) + uint32(c-'A'+10) + } else { + break + } + if acc > math.MaxUint16 { + // Overflow, fail. + return IP{}, parseIPError{in: in, msg: "IPv6 field has value >=2^16", at: s} + } + } + if off == 0 { + // No digits found, fail. + return IP{}, parseIPError{in: in, msg: "each colon-separated field must have at least one digit", at: s} + } + + // If followed by dot, might be in trailing IPv4. + if off < len(s) && s[off] == '.' { + if ellipsis < 0 && i != 12 { + // Not the right place. + return IP{}, parseIPError{in: in, msg: "embedded IPv4 address must replace the final 2 fields of the address", at: s} + } + if i+4 > 16 { + // Not enough room. + return IP{}, parseIPError{in: in, msg: "too many hex fields to fit an embedded IPv4 at the end of the address", at: s} + } + // TODO: could make this a bit faster by having a helper + // that parses to a [4]byte, and have both parseIPv4 and + // parseIPv6 use it. + ip4, err := parseIPv4(s) + if err != nil { + return IP{}, parseIPError{in: in, msg: err.Error(), at: s} + } + ip[i] = ip4.v4(0) + ip[i+1] = ip4.v4(1) + ip[i+2] = ip4.v4(2) + ip[i+3] = ip4.v4(3) + s = "" + i += 4 + break + } + + // Save this 16-bit chunk. + ip[i] = byte(acc >> 8) + ip[i+1] = byte(acc) + i += 2 + + // Stop at end of string. + s = s[off:] + if len(s) == 0 { + break + } + + // Otherwise must be followed by colon and more. + if s[0] != ':' { + return IP{}, parseIPError{in: in, msg: "unexpected character, want colon", at: s} + } else if len(s) == 1 { + return IP{}, parseIPError{in: in, msg: "colon must be followed by more characters", at: s} + } + s = s[1:] + + // Look for ellipsis. + if s[0] == ':' { + if ellipsis >= 0 { // already have one + return IP{}, parseIPError{in: in, msg: "multiple :: in address", at: s} + } + ellipsis = i + s = s[1:] + if len(s) == 0 { // can be at end + break + } + } + } + + // Must have used entire string. + if len(s) != 0 { + return IP{}, parseIPError{in: in, msg: "trailing garbage after address", at: s} + } + + // If didn't parse enough, expand ellipsis. + if i < 16 { + if ellipsis < 0 { + return IP{}, parseIPError{in: in, msg: "address string too short"} + } + n := 16 - i + for j := i - 1; j >= ellipsis; j-- { + ip[j+n] = ip[j] + } + for j := ellipsis + n - 1; j >= ellipsis; j-- { + ip[j] = 0 + } + } else if ellipsis >= 0 { + // Ellipsis must represent at least one 0 group. + return IP{}, parseIPError{in: in, msg: "the :: must expand to at least one field of zeros"} + } + return IPv6Raw(ip).WithZone(zone), nil +} + +// FromStdIP returns an IP from the standard library's IP type. +// +// If std is invalid, ok is false. +// +// FromStdIP implicitly unmaps IPv6-mapped IPv4 addresses. That is, if +// len(std) == 16 and contains an IPv4 address, only the IPv4 part is +// returned, without the IPv6 wrapper. This is the common form returned by +// the standard library's ParseIP: https://play.golang.org/p/qdjylUkKWxl. +// To convert a standard library IP without the implicit unmapping, use +// FromStdIPRaw. +func FromStdIP(std net.IP) (ip IP, ok bool) { + ret, ok := FromStdIPRaw(std) + if ret.Is4in6() { + ret.z = z4 + } + return ret, ok +} + +// FromStdIPRaw returns an IP from the standard library's IP type. +// If std is invalid, ok is false. +// Unlike FromStdIP, FromStdIPRaw does not do an implicit Unmap if +// len(std) == 16 and contains an IPv6-mapped IPv4 address. +func FromStdIPRaw(std net.IP) (ip IP, ok bool) { + switch len(std) { + case 4: + return IPv4(std[0], std[1], std[2], std[3]), true + case 16: + return ipv6Slice(std), true + } + return IP{}, false +} + +// v4 returns the i'th byte of ip. If ip is not an IPv4, v4 returns +// unspecified garbage. +func (ip IP) v4(i uint8) uint8 { + return uint8(ip.addr.lo >> ((3 - i) * 8)) +} + +// v6 returns the i'th byte of ip. If ip is an IPv4 address, this +// accesses the IPv4-mapped IPv6 address form of the IP. +func (ip IP) v6(i uint8) uint8 { + return uint8(*(ip.addr.halves()[(i/8)%2]) >> ((7 - i%8) * 8)) +} + +// v6u16 returns the i'th 16-bit word of ip. If ip is an IPv4 address, +// this accesses the IPv4-mapped IPv6 address form of the IP. +func (ip IP) v6u16(i uint8) uint16 { + return uint16(*(ip.addr.halves()[(i/4)%2]) >> ((3 - i%4) * 16)) +} + +// IsZero reports whether ip is the zero value of the IP type. +// The zero value is not a valid IP address of any type. +// +// Note that "0.0.0.0" and "::" are not the zero value. Use IsUnspecified to +// check for these values instead. +func (ip IP) IsZero() bool { + // Faster than comparing ip == IP{}, but effectively equivalent, + // as there's no way to make an IP with a nil z from this package. + return ip.z == z0 +} + +// IsValid whether the IP is an initialized value and not the IP +// type's zero value. +// +// Note that both "0.0.0.0" and "::" are valid, non-zero values. +func (ip IP) IsValid() bool { return ip.z != z0 } + +// BitLen returns the number of bits in the IP address: +// 32 for IPv4 or 128 for IPv6. +// For the zero value (see IP.IsZero), it returns 0. +// For IP4-mapped IPv6 addresses, it returns 128. +func (ip IP) BitLen() uint8 { + switch ip.z { + case z0: + return 0 + case z4: + return 32 + } + return 128 +} + +// Zone returns ip's IPv6 scoped addressing zone, if any. +func (ip IP) Zone() string { + if ip.z == nil { + return "" + } + zone, _ := ip.z.Get().(string) + return zone +} + +// Compare returns an integer comparing two IPs. +// The result will be 0 if ip==ip2, -1 if ip < ip2, and +1 if ip > ip2. +// The definition of "less than" is the same as the IP.Less method. +func (ip IP) Compare(ip2 IP) int { + f1, f2 := ip.BitLen(), ip2.BitLen() + if f1 < f2 { + return -1 + } + if f1 > f2 { + return 1 + } + if hi1, hi2 := ip.addr.hi, ip2.addr.hi; hi1 < hi2 { + return -1 + } else if hi1 > hi2 { + return 1 + } + if lo1, lo2 := ip.addr.lo, ip2.addr.lo; lo1 < lo2 { + return -1 + } else if lo1 > lo2 { + return 1 + } + if ip.Is6() { + za, zb := ip.Zone(), ip2.Zone() + if za < zb { + return -1 + } else if za > zb { + return 1 + } + } + return 0 +} + +// Less reports whether ip sorts before ip2. +// IP addresses sort first by length, then their address. +// IPv6 addresses with zones sort just after the same address without a zone. +func (ip IP) Less(ip2 IP) bool { return ip.Compare(ip2) == -1 } + +func (ip IP) lessOrEq(ip2 IP) bool { return ip.Compare(ip2) <= 0 } + +// ipZone returns the standard library net.IP from ip, as well +// as the zone. +// The optional reuse IP provides memory to reuse. +func (ip IP) ipZone(reuse net.IP) (stdIP net.IP, zone string) { + base := reuse[:0] + switch { + case ip.z == z0: + return nil, "" + case ip.Is4(): + a4 := ip.As4() + return append(base, a4[:]...), "" + default: + a16 := ip.As16() + return append(base, a16[:]...), ip.Zone() + } +} + +// IPAddr returns the net.IPAddr representation of an IP. The returned value is +// always non-nil, but the IPAddr.IP will be nil if ip is the zero value. +// If ip contains a zone identifier, IPAddr.Zone is populated. +func (ip IP) IPAddr() *net.IPAddr { + stdIP, zone := ip.ipZone(nil) + return &net.IPAddr{IP: stdIP, Zone: zone} +} + +// Is4 reports whether ip is an IPv4 address. +// +// It returns false for IP4-mapped IPv6 addresses. See IP.Unmap. +func (ip IP) Is4() bool { + return ip.z == z4 +} + +// Is4in6 reports whether ip is an IPv4-mapped IPv6 address. +func (ip IP) Is4in6() bool { + return ip.Is6() && ip.addr.hi == 0 && ip.addr.lo>>32 == 0xffff +} + +// Is6 reports whether ip is an IPv6 address, including IPv4-mapped +// IPv6 addresses. +func (ip IP) Is6() bool { + return ip.z != z0 && ip.z != z4 +} + +// Unmap returns ip with any IPv4-mapped IPv6 address prefix removed. +// +// That is, if ip is an IPv6 address wrapping an IPv4 adddress, it +// returns the wrapped IPv4 address. Otherwise it returns ip, regardless +// of its type. +func (ip IP) Unmap() IP { + if ip.Is4in6() { + ip.z = z4 + } + return ip +} + +// WithZone returns an IP that's the same as ip but with the provided +// zone. If zone is empty, the zone is removed. If ip is an IPv4 +// address it's returned unchanged. +func (ip IP) WithZone(zone string) IP { + if !ip.Is6() { + return ip + } + if zone == "" { + ip.z = z6noz + return ip + } + ip.z = intern.GetByString(zone) + return ip +} + +// noZone unconditionally strips the zone from IP. +// It's similar to WithZone, but small enough to be inlinable. +func (ip IP) withoutZone() IP { + if !ip.Is6() { + return ip + } + ip.z = z6noz + return ip +} + +// hasZone reports whether IP has an IPv6 zone. +func (ip IP) hasZone() bool { + return ip.z != z0 && ip.z != z4 && ip.z != z6noz +} + +// IsLinkLocalUnicast reports whether ip is a link-local unicast address. +// If ip is the zero value, it will return false. +func (ip IP) IsLinkLocalUnicast() bool { + // Dynamic Configuration of IPv4 Link-Local Addresses + // https://datatracker.ietf.org/doc/html/rfc3927#section-2.1 + if ip.Is4() { + return ip.v4(0) == 169 && ip.v4(1) == 254 + } + // IP Version 6 Addressing Architecture (2.4 Address Type Identification) + // https://datatracker.ietf.org/doc/html/rfc4291#section-2.4 + if ip.Is6() { + return ip.v6u16(0)&0xffc0 == 0xfe80 + } + return false // zero value +} + +// IsLoopback reports whether ip is a loopback address. If ip is the zero value, +// it will return false. +func (ip IP) IsLoopback() bool { + // Requirements for Internet Hosts -- Communication Layers (3.2.1.3 Addressing) + // https://datatracker.ietf.org/doc/html/rfc1122#section-3.2.1.3 + if ip.Is4() { + return ip.v4(0) == 127 + } + // IP Version 6 Addressing Architecture (2.4 Address Type Identification) + // https://datatracker.ietf.org/doc/html/rfc4291#section-2.4 + if ip.Is6() { + return ip.addr.hi == 0 && ip.addr.lo == 1 + } + return false // zero value +} + +// IsMulticast reports whether ip is a multicast address. If ip is the zero +// value, it will return false. +func (ip IP) IsMulticast() bool { + // Host Extensions for IP Multicasting (4. HOST GROUP ADDRESSES) + // https://datatracker.ietf.org/doc/html/rfc1112#section-4 + if ip.Is4() { + return ip.v4(0)&0xf0 == 0xe0 + } + // IP Version 6 Addressing Architecture (2.4 Address Type Identification) + // https://datatracker.ietf.org/doc/html/rfc4291#section-2.4 + if ip.Is6() { + return ip.addr.hi>>(64-8) == 0xff // ip.v6(0) == 0xff + } + return false // zero value +} + +// IsInterfaceLocalMulticast reports whether ip is an IPv6 interface-local +// multicast address. If ip is the zero value or an IPv4 address, it will return +// false. +func (ip IP) IsInterfaceLocalMulticast() bool { + // IPv6 Addressing Architecture (2.7.1. Pre-Defined Multicast Addresses) + // https://datatracker.ietf.org/doc/html/rfc4291#section-2.7.1 + if ip.Is6() { + return ip.v6u16(0)&0xff0f == 0xff01 + } + return false // zero value +} + +// IsLinkLocalMulticast reports whether ip is a link-local multicast address. +// If ip is the zero value, it will return false. +func (ip IP) IsLinkLocalMulticast() bool { + // IPv4 Multicast Guidelines (4. Local Network Control Block (224.0.0/24)) + // https://datatracker.ietf.org/doc/html/rfc5771#section-4 + if ip.Is4() { + return ip.v4(0) == 224 && ip.v4(1) == 0 && ip.v4(2) == 0 + } + // IPv6 Addressing Architecture (2.7.1. Pre-Defined Multicast Addresses) + // https://datatracker.ietf.org/doc/html/rfc4291#section-2.7.1 + if ip.Is6() { + return ip.v6u16(0)&0xff0f == 0xff02 + } + return false // zero value +} + +// IsGlobalUnicast reports whether ip is a global unicast address. +// +// It returns true for IPv6 addresses which fall outside of the current +// IANA-allocated 2000::/3 global unicast space, with the exception of the +// link-local address space. It also returns true even if ip is in the IPv4 +// private address space or IPv6 unique local address space. If ip is the zero +// value, it will return false. +// +// For reference, see RFC 1122, RFC 4291, and RFC 4632. +func (ip IP) IsGlobalUnicast() bool { + if ip.z == z0 { + // Invalid or zero-value. + return false + } + + // Match the stdlib's IsGlobalUnicast logic. Notably private IPv4 addresses + // and ULA IPv6 addresses are still considered "global unicast". + if ip.Is4() && (ip == IPv4(0, 0, 0, 0) || ip == IPv4(255, 255, 255, 255)) { + return false + } + + return ip != IPv6Unspecified() && + !ip.IsLoopback() && + !ip.IsMulticast() && + !ip.IsLinkLocalUnicast() +} + +// IsPrivate reports whether ip is a private address, according to RFC 1918 +// (IPv4 addresses) and RFC 4193 (IPv6 addresses). That is, it reports whether +// ip is in 10.0.0.0/8, 172.16.0.0/12, 192.168.0.0/16, or fc00::/7. This is the +// same as the standard library's net.IP.IsPrivate. +func (ip IP) IsPrivate() bool { + // Match the stdlib's IsPrivate logic. + if ip.Is4() { + // RFC 1918 allocates 10.0.0.0/8, 172.16.0.0/12, and 192.168.0.0/16 as + // private IPv4 address subnets. + return ip.v4(0) == 10 || + (ip.v4(0) == 172 && ip.v4(1)&0xf0 == 16) || + (ip.v4(0) == 192 && ip.v4(1) == 168) + } + + if ip.Is6() { + // RFC 4193 allocates fc00::/7 as the unique local unicast IPv6 address + // subnet. + return ip.v6(0)&0xfe == 0xfc + } + + return false // zero value +} + +// IsUnspecified reports whether ip is an unspecified address, either the IPv4 +// address "0.0.0.0" or the IPv6 address "::". +// +// Note that the IP zero value is not an unspecified address. Use IsZero to +// check for the zero value instead. +func (ip IP) IsUnspecified() bool { + return ip == IPv4(0, 0, 0, 0) || ip == IPv6Unspecified() +} + +// Prefix applies a CIDR mask of leading bits to IP, producing an IPPrefix +// of the specified length. If IP is the zero value, a zero-value IPPrefix and +// a nil error are returned. If bits is larger than 32 for an IPv4 address or +// 128 for an IPv6 address, an error is returned. +func (ip IP) Prefix(bits uint8) (IPPrefix, error) { + effectiveBits := bits + switch ip.z { + case z0: + return IPPrefix{}, nil + case z4: + if bits > 32 { + return IPPrefix{}, fmt.Errorf("prefix length %d too large for IPv4", bits) + } + effectiveBits += 96 + default: + if bits > 128 { + return IPPrefix{}, fmt.Errorf("prefix length %d too large for IPv6", bits) + } + } + ip.addr = ip.addr.and(mask6[effectiveBits]) + return IPPrefixFrom(ip, bits), nil +} + +// Netmask applies a bit mask to IP, producing an IPPrefix. If IP is the +// zero value, a zero-value IPPrefix and a nil error are returned. If the +// netmask length is not 4 for IPv4 or 16 for IPv6, an error is +// returned. If the netmask is non-contiguous, an error is returned. +func (ip IP) Netmask(mask []byte) (IPPrefix, error) { + l := len(mask) + + switch ip.z { + case z0: + return IPPrefix{}, nil + case z4: + if l != net.IPv4len { + return IPPrefix{}, fmt.Errorf("netmask length %d incorrect for IPv4", l) + } + default: + if l != net.IPv6len { + return IPPrefix{}, fmt.Errorf("netmask length %d incorrect for IPv6", l) + } + } + + ones, bits := net.IPMask(mask).Size() + if ones == 0 && bits == 0 { + return IPPrefix{}, errors.New("netmask is non-contiguous") + } + + return ip.Prefix(uint8(ones)) +} + +// As16 returns the IP address in its 16 byte representation. +// IPv4 addresses are returned in their v6-mapped form. +// IPv6 addresses with zones are returned without their zone (use the +// Zone method to get it). +// The ip zero value returns all zeroes. +func (ip IP) As16() [16]byte { + var ret [16]byte + binary.BigEndian.PutUint64(ret[:8], ip.addr.hi) + binary.BigEndian.PutUint64(ret[8:], ip.addr.lo) + return ret +} + +// As4 returns an IPv4 or IPv4-in-IPv6 address in its 4 byte representation. +// If ip is the IP zero value or an IPv6 address, As4 panics. +// Note that 0.0.0.0 is not the zero value. +func (ip IP) As4() [4]byte { + if ip.z == z4 || ip.Is4in6() { + var ret [4]byte + binary.BigEndian.PutUint32(ret[:], uint32(ip.addr.lo)) + return ret + } + if ip.z == z0 { + panic("As4 called on IP zero value") + } + panic("As4 called on IPv6 address") +} + +// Next returns the IP following ip. +// If there is none, it returns the IP zero value. +func (ip IP) Next() IP { + ip.addr = ip.addr.addOne() + if ip.Is4() { + if uint32(ip.addr.lo) == 0 { + // Overflowed. + return IP{} + } + } else { + if ip.addr.isZero() { + // Overflowed + return IP{} + } + } + return ip +} + +// Prior returns the IP before ip. +// If there is none, it returns the IP zero value. +func (ip IP) Prior() IP { + if ip.Is4() { + if uint32(ip.addr.lo) == 0 { + return IP{} + } + } else if ip.addr.isZero() { + return IP{} + } + ip.addr = ip.addr.subOne() + return ip +} + +// String returns the string form of the IP address ip. +// It returns one of 4 forms: +// +// - "invalid IP", if ip is the zero value +// - IPv4 dotted decimal ("192.0.2.1") +// - IPv6 ("2001:db8::1") +// - IPv6 with zone ("fe80:db8::1%eth0") +// +// Note that unlike the Go standard library's IP.String method, +// IP4-mapped IPv6 addresses do not format as dotted decimals. +func (ip IP) String() string { + switch ip.z { + case z0: + return "invalid IP" + case z4: + return ip.string4() + default: + return ip.string6() + } +} + +// AppendTo appends a text encoding of ip, +// as generated by MarshalText, +// to b and returns the extended buffer. +func (ip IP) AppendTo(b []byte) []byte { + switch ip.z { + case z0: + return b + case z4: + return ip.appendTo4(b) + default: + return ip.appendTo6(b) + } +} + +// digits is a string of the hex digits from 0 to f. It's used in +// appendDecimal and appendHex to format IP addresses. +const digits = "0123456789abcdef" + +// appendDecimal appends the decimal string representation of x to b. +func appendDecimal(b []byte, x uint8) []byte { + // Using this function rather than strconv.AppendUint makes IPv4 + // string building 2x faster. + + if x >= 100 { + b = append(b, digits[x/100]) + } + if x >= 10 { + b = append(b, digits[x/10%10]) + } + return append(b, digits[x%10]) +} + +// appendHex appends the hex string representation of x to b. +func appendHex(b []byte, x uint16) []byte { + // Using this function rather than strconv.AppendUint makes IPv6 + // string building 2x faster. + + if x >= 0x1000 { + b = append(b, digits[x>>12]) + } + if x >= 0x100 { + b = append(b, digits[x>>8&0xf]) + } + if x >= 0x10 { + b = append(b, digits[x>>4&0xf]) + } + return append(b, digits[x&0xf]) +} + +// appendHexPad appends the fully padded hex string representation of x to b. +func appendHexPad(b []byte, x uint16) []byte { + return append(b, digits[x>>12], digits[x>>8&0xf], digits[x>>4&0xf], digits[x&0xf]) +} + +func (ip IP) string4() string { + const max = len("255.255.255.255") + ret := make([]byte, 0, max) + ret = ip.appendTo4(ret) + return string(ret) +} + +func (ip IP) appendTo4(ret []byte) []byte { + ret = appendDecimal(ret, ip.v4(0)) + ret = append(ret, '.') + ret = appendDecimal(ret, ip.v4(1)) + ret = append(ret, '.') + ret = appendDecimal(ret, ip.v4(2)) + ret = append(ret, '.') + ret = appendDecimal(ret, ip.v4(3)) + return ret +} + +// string6 formats ip in IPv6 textual representation. It follows the +// guidelines in section 4 of RFC 5952 +// (https://tools.ietf.org/html/rfc5952#section-4): no unnecessary +// zeros, use :: to elide the longest run of zeros, and don't use :: +// to compact a single zero field. +func (ip IP) string6() string { + // Use a zone with a "plausibly long" name, so that most zone-ful + // IP addresses won't require additional allocation. + // + // The compiler does a cool optimization here, where ret ends up + // stack-allocated and so the only allocation this function does + // is to construct the returned string. As such, it's okay to be a + // bit greedy here, size-wise. + const max = len("ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff%enp5s0") + ret := make([]byte, 0, max) + ret = ip.appendTo6(ret) + return string(ret) +} + +func (ip IP) appendTo6(ret []byte) []byte { + zeroStart, zeroEnd := uint8(255), uint8(255) + for i := uint8(0); i < 8; i++ { + j := i + for j < 8 && ip.v6u16(j) == 0 { + j++ + } + if l := j - i; l >= 2 && l > zeroEnd-zeroStart { + zeroStart, zeroEnd = i, j + } + } + + for i := uint8(0); i < 8; i++ { + if i == zeroStart { + ret = append(ret, ':', ':') + i = zeroEnd + if i >= 8 { + break + } + } else if i > 0 { + ret = append(ret, ':') + } + + ret = appendHex(ret, ip.v6u16(i)) + } + + if ip.z != z6noz { + ret = append(ret, '%') + ret = append(ret, ip.Zone()...) + } + return ret +} + +// StringExpanded is like String but IPv6 addresses are expanded with leading +// zeroes and no "::" compression. For example, "2001:db8::1" becomes +// "2001:0db8:0000:0000:0000:0000:0000:0001". +func (ip IP) StringExpanded() string { + switch ip.z { + case z0, z4: + return ip.String() + } + + const size = len("ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff") + ret := make([]byte, 0, size) + for i := uint8(0); i < 8; i++ { + if i > 0 { + ret = append(ret, ':') + } + + ret = appendHexPad(ret, ip.v6u16(i)) + } + + if ip.z != z6noz { + // The addition of a zone will cause a second allocation, but when there + // is no zone the ret slice will be stack allocated. + ret = append(ret, '%') + ret = append(ret, ip.Zone()...) + } + return string(ret) +} + +// MarshalText implements the encoding.TextMarshaler interface, +// The encoding is the same as returned by String, with one exception: +// If ip is the zero value, the encoding is the empty string. +func (ip IP) MarshalText() ([]byte, error) { + switch ip.z { + case z0: + return []byte(""), nil + case z4: + max := len("255.255.255.255") + b := make([]byte, 0, max) + return ip.appendTo4(b), nil + default: + max := len("ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff%enp5s0") + b := make([]byte, 0, max) + return ip.appendTo6(b), nil + } +} + +// UnmarshalText implements the encoding.TextUnmarshaler interface. +// The IP address is expected in a form accepted by ParseIP. +// It returns an error if *ip is not the IP zero value. +func (ip *IP) UnmarshalText(text []byte) error { + if ip.z != z0 { + return errors.New("netaddr: refusing to Unmarshal into non-zero IP") + } + if len(text) == 0 { + return nil + } + var err error + *ip, err = ParseIP(string(text)) + return err +} + +// MarshalBinary implements the encoding.BinaryMarshaler interface. +func (ip IP) MarshalBinary() ([]byte, error) { + switch ip.z { + case z0: + return nil, nil + case z4: + b := ip.As4() + return b[:], nil + default: + b16 := ip.As16() + b := b16[:] + if z := ip.Zone(); z != "" { + b = append(b, []byte(z)...) + } + return b, nil + } +} + +// UnmarshalBinary implements the encoding.BinaryUnmarshaler interface. +func (ip *IP) UnmarshalBinary(b []byte) error { + if ip.z != z0 { + return errors.New("netaddr: refusing to Unmarshal into non-zero IP") + } + n := len(b) + switch { + case n == 0: + return nil + case n == 4: + *ip = IPv4(b[0], b[1], b[2], b[3]) + return nil + case n == 16: + *ip = ipv6Slice(b) + return nil + case n > 16: + *ip = ipv6Slice(b[:16]).WithZone(string(b[16:])) + return nil + } + return fmt.Errorf("netaddr: unexpected ip size: %v", len(b)) +} + +// IPPort is an IP and a port number. +type IPPort struct { + ip IP + port uint16 +} + +// IPPortFrom returns an IPPort with IP ip and port port. +// It does not allocate. +func IPPortFrom(ip IP, port uint16) IPPort { return IPPort{ip: ip, port: port} } + +// WithIP returns an IPPort with IP ip and port p.Port(). +func (p IPPort) WithIP(ip IP) IPPort { return IPPort{ip: ip, port: p.port} } + +// WithIP returns an IPPort with IP p.IP() and port port. +func (p IPPort) WithPort(port uint16) IPPort { return IPPort{ip: p.ip, port: port} } + +// IP returns p's IP. +func (p IPPort) IP() IP { return p.ip } + +// Port returns p's port. +func (p IPPort) Port() uint16 { return p.port } + +// splitIPPort splits s into an IP address string and a port +// string. It splits strings shaped like "foo:bar" or "[foo]:bar", +// without further validating the substrings. v6 indicates whether the +// ip string should parse as an IPv6 address or an IPv4 address, in +// order for s to be a valid ip:port string. +func splitIPPort(s string) (ip, port string, v6 bool, err error) { + i := strings.LastIndexByte(s, ':') + if i == -1 { + return "", "", false, errors.New("not an ip:port") + } + + ip, port = s[:i], s[i+1:] + if len(ip) == 0 { + return "", "", false, errors.New("no IP") + } + if len(port) == 0 { + return "", "", false, errors.New("no port") + } + if ip[0] == '[' { + if len(ip) < 2 || ip[len(ip)-1] != ']' { + return "", "", false, errors.New("missing ]") + } + ip = ip[1 : len(ip)-1] + v6 = true + } + + return ip, port, v6, nil +} + +// ParseIPPort parses s as an IPPort. +// +// It doesn't do any name resolution, and ports must be numeric. +func ParseIPPort(s string) (IPPort, error) { + var ipp IPPort + ip, port, v6, err := splitIPPort(s) + if err != nil { + return ipp, err + } + port16, err := strconv.ParseUint(port, 10, 16) + if err != nil { + return ipp, fmt.Errorf("invalid port %q parsing %q", port, s) + } + ipp.port = uint16(port16) + ipp.ip, err = ParseIP(ip) + if err != nil { + return IPPort{}, err + } + if v6 && ipp.ip.Is4() { + return IPPort{}, fmt.Errorf("invalid ip:port %q, square brackets can only be used with IPv6 addresses", s) + } else if !v6 && ipp.ip.Is6() { + return IPPort{}, fmt.Errorf("invalid ip:port %q, IPv6 addresses must be surrounded by square brackets", s) + } + return ipp, nil +} + +// MustParseIPPort calls ParseIPPort(s) and panics on error. +// It is intended for use in tests with hard-coded strings. +func MustParseIPPort(s string) IPPort { + ip, err := ParseIPPort(s) + if err != nil { + panic(err) + } + return ip +} + +// IsZero reports whether p is its zero value. +func (p IPPort) IsZero() bool { return p == IPPort{} } + +// IsValid reports whether p.IP() is valid. +// All ports are valid, including zero. +func (p IPPort) IsValid() bool { return p.ip.IsValid() } + +// Valid reports whether p.IP() is valid. +// All ports are valid, including zero. +// +// Deprecated: use the correctly named and identical IsValid method instead. +func (p IPPort) Valid() bool { return p.IsValid() } + +func (p IPPort) String() string { + switch p.ip.z { + case z0: + return "invalid IPPort" + case z4: + a := p.ip.As4() + return fmt.Sprintf("%d.%d.%d.%d:%d", a[0], a[1], a[2], a[3], p.port) + default: + // TODO: this could be more efficient allocation-wise: + return net.JoinHostPort(p.ip.String(), strconv.Itoa(int(p.port))) + } +} + +// AppendTo appends a text encoding of p, +// as generated by MarshalText, +// to b and returns the extended buffer. +func (p IPPort) AppendTo(b []byte) []byte { + switch p.ip.z { + case z0: + return b + case z4: + b = p.ip.appendTo4(b) + default: + b = append(b, '[') + b = p.ip.appendTo6(b) + b = append(b, ']') + } + b = append(b, ':') + b = strconv.AppendInt(b, int64(p.port), 10) + return b +} + +// MarshalText implements the encoding.TextMarshaler interface. The +// encoding is the same as returned by String, with one exception: if +// p.IP() is the zero value, the encoding is the empty string. +func (p IPPort) MarshalText() ([]byte, error) { + var max int + switch p.ip.z { + case z0: + case z4: + max = len("255.255.255.255:65535") + default: + max = len("[ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff%enp5s0]:65535") + } + b := make([]byte, 0, max) + b = p.AppendTo(b) + return b, nil +} + +// UnmarshalText implements the encoding.TextUnmarshaler +// interface. The IPPort is expected in a form accepted by +// ParseIPPort. It returns an error if *p is not the IPPort zero +// value. +func (p *IPPort) UnmarshalText(text []byte) error { + if p.ip.z != z0 || p.port != 0 { + return errors.New("netaddr: refusing to UnmarshalText into non-zero IP") + } + if len(text) == 0 { + return nil + } + var err error + *p, err = ParseIPPort(string(text)) + return err +} + +// FromStdAddr maps the components of a standard library TCPAddr or +// UDPAddr into an IPPort. +func FromStdAddr(stdIP net.IP, port int, zone string) (_ IPPort, ok bool) { + ip, ok := FromStdIP(stdIP) + if !ok || port < 0 || port > math.MaxUint16 { + return + } + ip = ip.Unmap() + if zone != "" { + if ip.Is4() { + ok = false + return + } + ip = ip.WithZone(zone) + } + return IPPort{ip: ip, port: uint16(port)}, true +} + +// UDPAddr returns a standard library net.UDPAddr from p. +// The returned value is always non-nil. If p.IP() is the zero +// value, then UDPAddr.IP is nil. +// +// UDPAddr necessarily does two allocations. If you have an existing +// UDPAddr already allocated, see UDPAddrAt. +func (p IPPort) UDPAddr() *net.UDPAddr { + ret := &net.UDPAddr{ + Port: int(p.port), + } + ret.IP, ret.Zone = p.ip.ipZone(nil) + return ret +} + +// UDPAddrAt is like UDPAddr, but reuses the provided UDPAddr, which +// must be non-nil. If at.IP has a capacity of 16, UDPAddrAt is +// allocation-free. It returns at to facilitate using this method as a +// wrapper. +func (p IPPort) UDPAddrAt(at *net.UDPAddr) *net.UDPAddr { + at.Port = int(p.port) + at.IP, at.Zone = p.ip.ipZone(at.IP) + return at +} + +// TCPAddr returns a standard library net.TCPAddr from p. +// The returned value is always non-nil. If p.IP() is the zero +// value, then TCPAddr.IP is nil. +func (p IPPort) TCPAddr() *net.TCPAddr { + ip, zone := p.ip.ipZone(nil) + return &net.TCPAddr{ + IP: ip, + Port: int(p.port), + Zone: zone, + } +} + +// IPPrefix is an IP address prefix (CIDR) representing an IP network. +// +// The first Bits() of IP() are specified. The remaining bits match any address. +// The range of Bits() is [0,32] for IPv4 or [0,128] for IPv6. +type IPPrefix struct { + ip IP + bits uint8 +} + +// IPPrefixFrom returns an IPPrefix with IP ip and port port. +// It does not allocate. +func IPPrefixFrom(ip IP, bits uint8) IPPrefix { + return IPPrefix{ + ip: ip.withoutZone(), + bits: bits, + } +} + +// IP returns p's IP. +func (p IPPrefix) IP() IP { return p.ip } + +// Bits returns p's prefix length. +func (p IPPrefix) Bits() uint8 { return p.bits } + +// IsValid reports whether whether p.Bits() has a valid range for p.IP(). +// If p.IP() is zero, Valid returns false. +func (p IPPrefix) IsValid() bool { return !p.ip.IsZero() && p.bits <= p.ip.BitLen() } + +// Valid reports whether whether p.Bits() has a valid range for p.IP(). +// If p.IP() is zero, Valid returns false. +// +// Deprecated: use the correctly named and identical IsValid method instead. +func (p IPPrefix) Valid() bool { return p.IsValid() } + +// IsZero reports whether p is its zero value. +func (p IPPrefix) IsZero() bool { return p == IPPrefix{} } + +// IsSingleIP reports whether p contains exactly one IP. +func (p IPPrefix) IsSingleIP() bool { return p.bits != 0 && p.bits == p.ip.BitLen() } + +// FromStdIPNet returns an IPPrefix from the standard library's IPNet type. +// If std is invalid, ok is false. +func FromStdIPNet(std *net.IPNet) (prefix IPPrefix, ok bool) { + ip, ok := FromStdIP(std.IP) + if !ok { + return IPPrefix{}, false + } + + if l := len(std.Mask); l != net.IPv4len && l != net.IPv6len { + // Invalid mask. + return IPPrefix{}, false + } + + ones, bits := std.Mask.Size() + if ones == 0 && bits == 0 { + // IPPrefix does not support non-contiguous masks. + return IPPrefix{}, false + } + + return IPPrefix{ + ip: ip, + bits: uint8(ones), + }, true +} + +// ParseIPPrefix parses s as an IP address prefix. +// The string can be in the form "192.168.1.0/24" or "2001::db8::/32", +// the CIDR notation defined in RFC 4632 and RFC 4291. +// +// Note that masked address bits are not zeroed. Use Masked for that. +func ParseIPPrefix(s string) (IPPrefix, error) { + i := strings.LastIndexByte(s, '/') + if i < 0 { + return IPPrefix{}, fmt.Errorf("netaddr.ParseIPPrefix(%q): no '/'", s) + } + ip, err := ParseIP(s[:i]) + if err != nil { + return IPPrefix{}, fmt.Errorf("netaddr.ParseIPPrefix(%q): %v", s, err) + } + s = s[i+1:] + bits, err := strconv.Atoi(s) + if err != nil { + return IPPrefix{}, fmt.Errorf("netaddr.ParseIPPrefix(%q): bad prefix: %v", s, err) + } + maxBits := 32 + if ip.Is6() { + maxBits = 128 + } + if bits < 0 || bits > maxBits { + return IPPrefix{}, fmt.Errorf("netaddr.ParseIPPrefix(%q): prefix length out of range", s) + } + return IPPrefixFrom(ip, uint8(bits)), nil +} + +// MustParseIPPrefix calls ParseIPPrefix(s) and panics on error. +// It is intended for use in tests with hard-coded strings. +func MustParseIPPrefix(s string) IPPrefix { + ip, err := ParseIPPrefix(s) + if err != nil { + panic(err) + } + return ip +} + +// Masked returns p in its canonical form, with bits of p.IP() not in p.Bits() masked off. +// If p is zero or otherwise invalid, Masked returns the zero value. +func (p IPPrefix) Masked() IPPrefix { + if m, err := p.ip.Prefix(p.bits); err == nil { + return m + } + return IPPrefix{} +} + +// Range returns the inclusive range of IPs that p covers. +// +// If p is zero or otherwise invalid, Range returns the zero value. +func (p IPPrefix) Range() IPRange { + p = p.Masked() + if p.IsZero() { + return IPRange{} + } + return IPRangeFrom(p.ip, p.lastIP()) +} + +// IPNet returns the net.IPNet representation of an IPPrefix. +// The returned value is always non-nil. +// Any zone identifier is dropped in the conversion. +func (p IPPrefix) IPNet() *net.IPNet { + if !p.Valid() { + return &net.IPNet{} + } + stdIP, _ := p.ip.ipZone(nil) + return &net.IPNet{ + IP: stdIP, + Mask: net.CIDRMask(int(p.bits), int(p.ip.BitLen())), + } +} + +// Contains reports whether the network p includes ip. +// +// An IPv4 address will not match an IPv6 prefix. +// A v6-mapped IPv6 address will not match an IPv4 prefix. +// A zero-value IP will not match any prefix. +// If ip has an IPv6 zone, Contains returns false, +// because IPPrefixes strip zones. +func (p IPPrefix) Contains(ip IP) bool { + if !p.Valid() || ip.hasZone() { + return false + } + if f1, f2 := p.ip.BitLen(), ip.BitLen(); f1 == 0 || f2 == 0 || f1 != f2 { + return false + } + if ip.Is4() { + // xor the IP addresses together; mismatched bits are now ones. + // Shift away the number of bits we don't care about. + // Shifts in Go are more efficient if the compiler can prove + // that the shift amount is smaller than the width of the shifted type (64 here). + // We know that p.bits is in the range 0..32 because p is Valid; + // the compiler doesn't know that, so mask with 63 to help it. + // Now truncate to 32 bits, because this is IPv4. + // If all the bits we care about are equal, the result will be zero. + return uint32((ip.addr.lo^p.ip.addr.lo)>>((32-p.bits)&63)) == 0 + } else { + // xor the IP addresses together. + // Mask away the bits we don't care about. + // If all the bits we care about are equal, the result will be zero. + return ip.addr.xor(p.ip.addr).and(mask6[p.bits]).isZero() + } +} + +// Overlaps reports whether p and o overlap at all. +// +// If p and o are of different address families or either have a zero +// IP, it reports false. Like the Contains method, a prefix with a +// v6-mapped IPv4 IP is still treated as an IPv6 mask. +// +// If either has a Bits of zero, it returns true. +func (p IPPrefix) Overlaps(o IPPrefix) bool { + if !p.Valid() || !o.Valid() { + return false + } + if p == o { + return true + } + if p.ip.Is4() != o.ip.Is4() { + return false + } + var minBits uint8 + if p.bits < o.bits { + minBits = p.bits + } else { + minBits = o.bits + } + if minBits == 0 { + return true + } + // One of these Prefix calls might look redundant, but we don't require + // that p and o values are normalized (via IPPrefix.Masked) first, + // so the Prefix call on the one that's already minBits serves to zero + // out any remaining bits in IP. + var err error + if p, err = p.ip.Prefix(minBits); err != nil { + return false + } + if o, err = o.ip.Prefix(minBits); err != nil { + return false + } + return p.ip == o.ip +} + +// AppendTo appends a text encoding of p, +// as generated by MarshalText, +// to b and returns the extended buffer. +func (p IPPrefix) AppendTo(b []byte) []byte { + if p.IsZero() { + return b + } + if !p.Valid() { + return append(b, "invalid IPPrefix"...) + } + + // p.IP is non-zero, because p is valid. + if p.ip.z == z4 { + b = p.ip.appendTo4(b) + } else { + b = p.ip.appendTo6(b) + } + + b = append(b, '/') + b = appendDecimal(b, p.bits) + return b +} + +// MarshalText implements the encoding.TextMarshaler interface, +// The encoding is the same as returned by String, with one exception: +// If p is the zero value, the encoding is the empty string. +func (p IPPrefix) MarshalText() ([]byte, error) { + var max int + switch p.ip.z { + case z0: + case z4: + max = len("255.255.255.255/32") + default: + max = len("ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff%enp5s0/128") + } + b := make([]byte, 0, max) + b = p.AppendTo(b) + return b, nil +} + +// UnmarshalText implements the encoding.TextUnmarshaler interface. +// The IP address is expected in a form accepted by ParseIPPrefix. +// It returns an error if *p is not the IPPrefix zero value. +func (p *IPPrefix) UnmarshalText(text []byte) error { + if *p != (IPPrefix{}) { + return errors.New("netaddr: refusing to Unmarshal into non-zero IPPrefix") + } + + if len(text) == 0 { + return nil + } + + var err error + *p, err = ParseIPPrefix(string(text)) + return err +} + +// String returns the CIDR notation of p: "/". +func (p IPPrefix) String() string { + if !p.Valid() { + return "invalid IPPrefix" + } + return fmt.Sprintf("%s/%d", p.ip, p.bits) +} + +// lastIP returns the last IP in the prefix. +func (p IPPrefix) lastIP() IP { + if !p.Valid() { + return IP{} + } + a16 := p.ip.As16() + var off uint8 + var bits uint8 = 128 + if p.ip.Is4() { + off = 12 + bits = 32 + } + for b := p.bits; b < bits; b++ { + byteNum, bitInByte := b/8, 7-(b%8) + a16[off+byteNum] |= 1 << uint(bitInByte) + } + if p.ip.Is4() { + return IPFrom16(a16) + } else { + return IPv6Raw(a16) // doesn't unmap + } +} + +// IPRange represents an inclusive range of IP addresses +// from the same address family. +// +// The From() and To() IPs are inclusive bounds, both included in the +// range. +// +// To be valid, the From() and To() values must be non-zero, have matching +// address families (IPv4 vs IPv6), and From() must be less than or equal to To(). +// IPv6 zones are stripped out and ignored. +// An invalid range may be ignored. +type IPRange struct { + // from is the initial IP address in the range. + from IP + + // to is the final IP address in the range. + to IP +} + +// IPRangeFrom returns an IPRange from from to to. +// It does not allocate. +func IPRangeFrom(from, to IP) IPRange { + return IPRange{ + from: from.withoutZone(), + to: to.withoutZone(), + } +} + +// From returns the lower bound of r. +func (r IPRange) From() IP { return r.from } + +// To returns the upper bound of r. +func (r IPRange) To() IP { return r.to } + +// ParseIPRange parses a range out of two IPs separated by a hyphen. +// +// It returns an error if the range is not valid. +func ParseIPRange(s string) (IPRange, error) { + var r IPRange + h := strings.IndexByte(s, '-') + if h == -1 { + return r, fmt.Errorf("no hyphen in range %q", s) + } + from, to := s[:h], s[h+1:] + var err error + r.from, err = ParseIP(from) + if err != nil { + return r, fmt.Errorf("invalid From IP %q in range %q", from, s) + } + r.from = r.from.withoutZone() + r.to, err = ParseIP(to) + if err != nil { + return r, fmt.Errorf("invalid To IP %q in range %q", to, s) + } + r.to = r.to.withoutZone() + if !r.Valid() { + return r, fmt.Errorf("range %v to %v not valid", r.from, r.to) + } + return r, nil +} + +// String returns a string representation of the range. +// +// For a valid range, the form is "From-To" with a single hyphen +// separating the IPs, the same format recognized by +// ParseIPRange. +func (r IPRange) String() string { + if r.Valid() { + return fmt.Sprintf("%s-%s", r.from, r.to) + } + if r.from.IsZero() || r.to.IsZero() { + return "zero IPRange" + } + return "invalid IPRange" +} + +// IsValid reports whether r.From() and r.To() are both non-zero and +// obey the documented requirements: address families match, and From +// is less than or equal to To. +func (r IPRange) IsValid() bool { + return !r.from.IsZero() && + r.from.z == r.to.z && + !r.to.Less(r.from) +} + +// Valid reports whether r.From() and r.To() are both non-zero and +// obey the documented requirements: address families match, and From +// is less than or equal to To. +// +// Deprecated: use the correctly named and identical IsValid method instead. +func (r IPRange) Valid() bool { return r.IsValid() } + +// Contains reports whether the range r includes addr. +// +// An invalid range always reports false. +// +// If ip has an IPv6 zone, Contains returns false, +// because IPPrefixes strip zones. +func (r IPRange) Contains(addr IP) bool { + return r.Valid() && !addr.hasZone() && r.contains(addr) +} + +// contains is like Contains, but without the validity check. +// addr must not have a zone. +func (r IPRange) contains(addr IP) bool { + return r.from.Compare(addr) <= 0 && r.to.Compare(addr) >= 0 +} + +// less reports whether r is "before" other. It is before if r.From() +// is before other.From(). If they're equal, then the larger range +// (higher To()) comes first. +func (r IPRange) less(other IPRange) bool { + if cmp := r.from.Compare(other.from); cmp != 0 { + return cmp < 0 + } + return other.to.Less(r.to) +} + +// entirelyBefore returns whether r lies entirely before other in IP +// space. +func (r IPRange) entirelyBefore(other IPRange) bool { + return r.to.Less(other.from) +} + +// entirelyWithin returns whether r is entirely contained within +// other. +func (r IPRange) coveredBy(other IPRange) bool { + return other.from.lessOrEq(r.from) && r.to.lessOrEq(other.to) +} + +// inMiddleOf returns whether r is inside other, but not touching the +// edges of other. +func (r IPRange) inMiddleOf(other IPRange) bool { + return other.from.Less(r.from) && r.to.Less(other.to) +} + +// overlapsStartOf returns whether r entirely overlaps the start of +// other, but not all of other. +func (r IPRange) overlapsStartOf(other IPRange) bool { + return r.from.lessOrEq(other.from) && r.to.Less(other.to) +} + +// overlapsEndOf returns whether r entirely overlaps the end of +// other, but not all of other. +func (r IPRange) overlapsEndOf(other IPRange) bool { + return other.from.Less(r.from) && other.to.lessOrEq(r.to) +} + +// mergeIPRanges returns the minimum and sorted set of IP ranges that +// cover r. +func mergeIPRanges(rr []IPRange) (out []IPRange, valid bool) { + // Always return a copy of r, to avoid aliasing slice memory in + // the caller. + switch len(rr) { + case 0: + return nil, true + case 1: + return []IPRange{rr[0]}, true + } + + sort.Slice(rr, func(i, j int) bool { return rr[i].less(rr[j]) }) + out = make([]IPRange, 1, len(rr)) + out[0] = rr[0] + for _, r := range rr[1:] { + prev := &out[len(out)-1] + switch { + case !r.Valid(): + // Invalid ranges make no sense to merge, refuse to + // perform. + return nil, false + case prev.to.Next() == r.from: + // prev and r touch, merge them. + // + // prev r + // f------tf-----t + prev.to = r.to + case prev.to.Less(r.from): + // No overlap and not adjacent (per previous case), no + // merging possible. + // + // prev r + // f------t f-----t + out = append(out, r) + case prev.to.Less(r.to): + // Partial overlap, update prev + // + // prev + // f------t + // f-----t + // r + prev.to = r.to + default: + // r entirely contained in prev, nothing to do. + // + // prev + // f--------t + // f-----t + // r + } + } + return out, true +} + +// Overlaps reports whether p and o overlap at all. +// +// If p and o are of different address families or either are invalid, +// it reports false. +func (r IPRange) Overlaps(o IPRange) bool { + return r.Valid() && + o.Valid() && + r.from.Compare(o.to) <= 0 && + o.from.Compare(r.to) <= 0 +} + +// prefixMaker returns a address-family-corrected IPPrefix from a and bits, +// where the input bits is always in the IPv6-mapped form for IPv4 addresses. +type prefixMaker func(a uint128, bits uint8) IPPrefix + +// Prefixes returns the set of IPPrefix entries that covers r. +// +// If either of r's bounds are invalid, in the wrong order, or if +// they're of different address families, then Prefixes returns nil. +// +// Prefixes necessarily allocates. See AppendPrefixes for a version that uses +// memory you provide. +func (r IPRange) Prefixes() []IPPrefix { + return r.AppendPrefixes(nil) +} + +// AppendPrefixes is an append version of IPRange.Prefixes. It appends +// the IPPrefix entries that cover r to dst. +func (r IPRange) AppendPrefixes(dst []IPPrefix) []IPPrefix { + if !r.Valid() { + return nil + } + return appendRangePrefixes(dst, r.prefixFrom128AndBits, r.from.addr, r.to.addr) +} + +func (r IPRange) prefixFrom128AndBits(a uint128, bits uint8) IPPrefix { + ip := IP{addr: a, z: r.from.z} + if r.from.Is4() { + bits -= 12 * 8 + } + return IPPrefix{ip, bits} +} + +// aZeroBSet is whether, after the common bits, a is all zero bits and +// b is all set (one) bits. +func comparePrefixes(a, b uint128) (common uint8, aZeroBSet bool) { + common = a.commonPrefixLen(b) + + // See whether a and b, after their common shared bits, end + // in all zero bits or all one bits, respectively. + if common == 128 { + return common, true + } + + m := mask6[common] + return common, (a.xor(a.and(m)).isZero() && + b.or(m) == uint128{^uint64(0), ^uint64(0)}) +} + +// Prefix returns r as an IPPrefix, if it can be presented exactly as such. +// If r is not valid or is not exactly equal to one prefix, ok is false. +func (r IPRange) Prefix() (p IPPrefix, ok bool) { + if !r.Valid() { + return + } + if common, ok := comparePrefixes(r.from.addr, r.to.addr); ok { + return r.prefixFrom128AndBits(r.from.addr, common), true + } + return +} + +func appendRangePrefixes(dst []IPPrefix, makePrefix prefixMaker, a, b uint128) []IPPrefix { + common, ok := comparePrefixes(a, b) + if ok { + // a to b represents a whole range, like 10.50.0.0/16. + // (a being 10.50.0.0 and b being 10.50.255.255) + return append(dst, makePrefix(a, common)) + } + // Otherwise recursively do both halves. + dst = appendRangePrefixes(dst, makePrefix, a, a.bitsSetFrom(common+1)) + dst = appendRangePrefixes(dst, makePrefix, b.bitsClearedFrom(common+1), b) + return dst +} diff --git a/vendor/inet.af/netaddr/uint128.go b/vendor/inet.af/netaddr/uint128.go new file mode 100644 index 0000000000000..2ba93f31beee0 --- /dev/null +++ b/vendor/inet.af/netaddr/uint128.go @@ -0,0 +1,82 @@ +// Copyright 2020 The Inet.Af AUTHORS. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package netaddr + +import "math/bits" + +// uint128 represents a uint128 using two uint64s. +// +// When the methods below mention a bit number, bit 0 is the most +// significant bit (in hi) and bit 127 is the lowest (lo&1). +type uint128 struct { + hi uint64 + lo uint64 +} + +// isZero reports whether u == 0. +// +// It's faster than u == (uint128{}) because the compiler (as of Go +// 1.15/1.16b1) doesn't do this trick and instead inserts a branch in +// its eq alg's generated code. +func (u uint128) isZero() bool { return u.hi|u.lo == 0 } + +// and returns the bitwise AND of u and m (u&m). +func (u uint128) and(m uint128) uint128 { + return uint128{u.hi & m.hi, u.lo & m.lo} +} + +// xor returns the bitwise XOR of u and m (u^m). +func (u uint128) xor(m uint128) uint128 { + return uint128{u.hi ^ m.hi, u.lo ^ m.lo} +} + +// or returns the bitwise OR of u and m (u|m). +func (u uint128) or(m uint128) uint128 { + return uint128{u.hi | m.hi, u.lo | m.lo} +} + +// not returns the bitwise NOT of u. +func (u uint128) not() uint128 { + return uint128{^u.hi, ^u.lo} +} + +// subOne returns u - 1. +func (u uint128) subOne() uint128 { + lo, borrow := bits.Sub64(u.lo, 1, 0) + return uint128{u.hi - borrow, lo} +} + +// addOne returns u + 1. +func (u uint128) addOne() uint128 { + lo, carry := bits.Add64(u.lo, 1, 0) + return uint128{u.hi + carry, lo} +} + +func u64CommonPrefixLen(a, b uint64) uint8 { + return uint8(bits.LeadingZeros64(a ^ b)) +} + +func (u uint128) commonPrefixLen(v uint128) (n uint8) { + if n = u64CommonPrefixLen(u.hi, v.hi); n == 64 { + n += u64CommonPrefixLen(u.lo, v.lo) + } + return +} + +func (u *uint128) halves() [2]*uint64 { + return [2]*uint64{&u.hi, &u.lo} +} + +// bitsSetFrom returns a copy of u with the given bit +// and all subsequent ones set. +func (u uint128) bitsSetFrom(bit uint8) uint128 { + return u.or(mask6[bit].not()) +} + +// bitsClearedFrom returns a copy of u with the given bit +// and all subsequent ones cleared. +func (u uint128) bitsClearedFrom(bit uint8) uint128 { + return u.and(mask6[bit]) +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 5dacb1f600be9..61e87b118a1a2 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1144,6 +1144,10 @@ go.uber.org/zap/internal/bufferpool go.uber.org/zap/internal/color go.uber.org/zap/internal/exit go.uber.org/zap/zapcore +# go4.org/intern v0.0.0-20210108033219-3eb7198706b2 +go4.org/intern +# go4.org/unsafe/assume-no-moving-gc v0.0.0-20201222180813-1025295fd063 +go4.org/unsafe/assume-no-moving-gc # golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83 ## explicit golang.org/x/crypto/argon2 @@ -1396,6 +1400,9 @@ gopkg.in/yaml.v2 # gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b ## explicit gopkg.in/yaml.v3 +# inet.af/netaddr v0.0.0-20210707202901-70468d781e6c +## explicit +inet.af/netaddr # k8s.io/api v0.21.0 => k8s.io/api v0.21.0 k8s.io/api/admissionregistration/v1 k8s.io/api/admissionregistration/v1beta1 From af78e908b610ef0bd74aa53cb920709b20f541f3 Mon Sep 17 00:00:00 2001 From: Kaviraj Date: Mon, 12 Jul 2021 21:04:43 +0200 Subject: [PATCH 02/24] Adding IP filter into label filter gramer 1. ast tests 2. parse tests Signed-off-by: Kaviraj --- pkg/logql/ast.go | 3 + pkg/logql/ast_test.go | 3 + pkg/logql/expr.y | 12 +- pkg/logql/expr.y.go | 515 ++++++++++++++++++++------------------- pkg/logql/lex.go | 3 + pkg/logql/log/ip.go | 67 ++++- pkg/logql/parser_test.go | 23 ++ 7 files changed, 366 insertions(+), 260 deletions(-) diff --git a/pkg/logql/ast.go b/pkg/logql/ast.go index f07faaa07797e..7a784d45fdddd 100644 --- a/pkg/logql/ast.go +++ b/pkg/logql/ast.go @@ -678,6 +678,9 @@ const ( OpConvDurationSeconds = "duration_seconds" OpLabelReplace = "label_replace" + + // function filters + OpFilterIP = "ip" ) func IsComparisonOperator(op string) bool { diff --git a/pkg/logql/ast_test.go b/pkg/logql/ast_test.go index 2f9fa8b62bf40..474dccb2bd009 100644 --- a/pkg/logql/ast_test.go +++ b/pkg/logql/ast_test.go @@ -30,6 +30,9 @@ func Test_logSelectorExpr_String(t *testing.T) { {`{foo="bar"} |= "baz" |~ "blip" != "flip" !~ "flap" | unpack | foo>5`, true}, {`{foo="bar"} |= "baz" |~ "blip" != "flip" !~ "flap" | pattern " bar " | foo>5`, true}, {`{foo="bar"} |= "baz" |~ "blip" != "flip" !~ "flap" | logfmt | b>=10GB`, true}, + {`{foo="bar"} |= "baz" |~ "blip" != "flip" !~ "flap" | logfmt | b=ip("127.0.0.1")`, true}, + {`{foo="bar"} |= "baz" |~ "blip" != "flip" !~ "flap" | logfmt | b=ip("127.0.0.1") | level="error"`, true}, + {`{foo="bar"} |= "baz" |~ "blip" != "flip" !~ "flap" | logfmt | b=ip("127.0.0.1") | level="error" | c=ip("1.2.3.4")`, true}, // chain inside label filters. {`{foo="bar"} |= "baz" |~ "blip" != "flip" !~ "flap" | regexp "(?Pfoo|bar)"`, true}, {`{foo="bar"} |= "baz" |~ "blip" != "flip" !~ "flap" | regexp "(?Pfoo|bar)" | ( ( foo<5.01 , bar>20ms ) or foo="bar" ) | line_format "blip{{.boop}}bap" | label_format foo=bar,bar="blip{{.blop}}"`, true}, } diff --git a/pkg/logql/expr.y b/pkg/logql/expr.y index b2d7e92b148b4..b58368c813cda 100644 --- a/pkg/logql/expr.y +++ b/pkg/logql/expr.y @@ -42,8 +42,9 @@ import ( DurationFilter log.LabelFilterer LabelFilter log.LabelFilterer UnitFilter log.LabelFilterer - LineFormatExpr *LineFmtExpr - LabelFormatExpr *LabelFmtExpr + IPFilter log.LabelFilterer + LineFormatExpr *lineFmtExpr + LabelFormatExpr *labelFmtExpr LabelFormat log.LabelFmt LabelsFormat []log.LabelFmt JSONExpressionParser *jsonExpressionParser @@ -91,6 +92,7 @@ import ( %type jsonExpressionList %type unwrapExpr %type unitFilter +%type ipFilter %type offsetExpr %token BYTES @@ -100,7 +102,7 @@ import ( OPEN_PARENTHESIS CLOSE_PARENTHESIS BY WITHOUT COUNT_OVER_TIME RATE SUM AVG MAX MIN COUNT STDDEV STDVAR BOTTOMK TOPK BYTES_OVER_TIME BYTES_RATE BOOL JSON REGEXP LOGFMT PIPE LINE_FMT LABEL_FMT UNWRAP AVG_OVER_TIME SUM_OVER_TIME MIN_OVER_TIME MAX_OVER_TIME STDVAR_OVER_TIME STDDEV_OVER_TIME QUANTILE_OVER_TIME BYTES_CONV DURATION_CONV DURATION_SECONDS_CONV - FIRST_OVER_TIME LAST_OVER_TIME ABSENT_OVER_TIME LABEL_REPLACE UNPACK OFFSET PATTERN + FIRST_OVER_TIME LAST_OVER_TIME ABSENT_OVER_TIME LABEL_REPLACE UNPACK OFFSET PATTERN IP // Operators are listed with increasing precedence. %left OR @@ -269,6 +271,7 @@ labelFormatExpr: LABEL_FMT labelsFormat { $$ = newLabelFmtExpr($2) }; labelFilter: matcher { $$ = log.NewStringLabelFilter($1) } + | ipFilter { $$ = $1 } | unitFilter { $$ = $1 } | numberFilter { $$ = $1 } | OPEN_PARENTHESIS labelFilter CLOSE_PARENTHESIS { $$ = $2 } @@ -286,6 +289,9 @@ jsonExpressionList: | jsonExpressionList COMMA jsonExpression { $$ = append($1, $3) } ; +ipFilter: +IDENTIFIER EQ IP OPEN_PARENTHESIS STRING CLOSE_PARENTHESIS { $$ = log.NewIPLabelFilter($5, $1) } + unitFilter: durationFilter { $$ = $1 } | bytesFilter { $$ = $1 } diff --git a/pkg/logql/expr.y.go b/pkg/logql/expr.y.go index f0f9e18811c05..ea56e5719935f 100644 --- a/pkg/logql/expr.y.go +++ b/pkg/logql/expr.y.go @@ -4,6 +4,7 @@ package logql import __yyfmt__ "fmt" + import ( "github.com/grafana/loki/pkg/logql/log" "github.com/prometheus/prometheus/pkg/labels" @@ -17,7 +18,7 @@ type exprSymType struct { Grouping *grouping Labels []string LogExpr LogSelectorExpr - LogRangeExpr *LogRange + LogRangeExpr *logRange Matcher *labels.Matcher Matchers []*labels.Matcher RangeAggregationExpr SampleExpr @@ -33,10 +34,10 @@ type exprSymType struct { bytes uint64 str string duration time.Duration - LiteralExpr *LiteralExpr + LiteralExpr *literalExpr BinOpModifier BinOpOptions - LabelParser *LabelParserExpr - LineFilters *LineFilterExpr + LabelParser *labelParserExpr + LineFilters *lineFilterExpr PipelineExpr MultiStageExpr PipelineStage StageExpr BytesFilter log.LabelFilterer @@ -44,15 +45,16 @@ type exprSymType struct { DurationFilter log.LabelFilterer LabelFilter log.LabelFilterer UnitFilter log.LabelFilterer - LineFormatExpr *LineFmtExpr - LabelFormatExpr *LabelFmtExpr + IPFilter log.LabelFilterer + LineFormatExpr *lineFmtExpr + LabelFormatExpr *labelFmtExpr LabelFormat log.LabelFmt LabelsFormat []log.LabelFmt - JSONExpressionParser *JSONExpressionParser + JSONExpressionParser *jsonExpressionParser JSONExpression log.JSONExpression JSONExpressionList []log.JSONExpression - UnwrapExpr *UnwrapExpr - OffsetExpr *OffsetExpr + UnwrapExpr *unwrapExpr + OffsetExpr *offsetExpr } const BYTES = 57346 @@ -116,21 +118,22 @@ const LABEL_REPLACE = 57403 const UNPACK = 57404 const OFFSET = 57405 const PATTERN = 57406 -const OR = 57407 -const AND = 57408 -const UNLESS = 57409 -const CMP_EQ = 57410 -const NEQ = 57411 -const LT = 57412 -const LTE = 57413 -const GT = 57414 -const GTE = 57415 -const ADD = 57416 -const SUB = 57417 -const MUL = 57418 -const DIV = 57419 -const MOD = 57420 -const POW = 57421 +const IP = 57407 +const OR = 57408 +const AND = 57409 +const UNLESS = 57410 +const CMP_EQ = 57411 +const NEQ = 57412 +const LT = 57413 +const LTE = 57414 +const GT = 57415 +const GTE = 57416 +const ADD = 57417 +const SUB = 57418 +const MUL = 57419 +const DIV = 57420 +const MOD = 57421 +const POW = 57422 var exprToknames = [...]string{ "$end", @@ -197,6 +200,7 @@ var exprToknames = [...]string{ "UNPACK", "OFFSET", "PATTERN", + "IP", "OR", "AND", "UNLESS", @@ -219,6 +223,7 @@ const exprEofCode = 1 const exprErrCode = 2 const exprInitialStackSize = 16 + var exprExca = [...]int{ -1, 1, 1, -1, @@ -227,100 +232,101 @@ var exprExca = [...]int{ const exprPrivate = 57344 -const exprLast = 488 +const exprLast = 493 var exprAct = [...]int{ - 231, 185, 75, 4, 167, 58, 155, 5, 160, 194, - 66, 109, 50, 57, 234, 129, 68, 2, 45, 46, + 233, 186, 75, 4, 168, 58, 156, 5, 161, 195, + 66, 109, 50, 57, 236, 130, 68, 2, 45, 46, 47, 48, 49, 50, 71, 42, 43, 44, 51, 52, 55, 56, 53, 54, 45, 46, 47, 48, 49, 50, 43, 44, 51, 52, 55, 56, 53, 54, 45, 46, - 47, 48, 49, 50, 47, 48, 49, 50, 125, 127, - 128, 64, 239, 97, 169, 127, 128, 101, 62, 63, - 61, 236, 286, 82, 76, 77, 267, 304, 133, 266, - 286, 131, 266, 299, 138, 51, 52, 55, 56, 53, - 54, 45, 46, 47, 48, 49, 50, 289, 139, 234, - 140, 141, 142, 143, 144, 145, 146, 147, 148, 149, - 150, 151, 152, 153, 236, 126, 65, 236, 115, 164, - 175, 170, 173, 174, 171, 172, 269, 270, 271, 248, - 98, 291, 157, 235, 250, 176, 112, 237, 181, 192, - 188, 184, 64, 276, 246, 186, 64, 197, 189, 62, - 63, 64, 274, 62, 63, 181, 240, 248, 62, 63, - 258, 237, 249, 200, 201, 202, 64, 190, 236, 235, - 184, 120, 187, 62, 63, 64, 187, 243, 158, 156, - 119, 187, 62, 63, 273, 229, 232, 181, 238, 115, - 241, 131, 97, 244, 101, 245, 187, 65, 233, 230, - 234, 65, 242, 157, 236, 187, 65, 112, 205, 182, - 64, 254, 252, 115, 115, 196, 64, 62, 63, 115, - 283, 65, 196, 62, 63, 302, 137, 157, 157, 115, - 65, 112, 112, 259, 198, 261, 263, 112, 265, 97, - 187, 195, 136, 264, 275, 260, 60, 112, 97, 158, - 156, 277, 135, 80, 73, 104, 106, 105, 298, 113, - 114, 280, 281, 279, 130, 65, 97, 282, 247, 12, - 206, 65, 12, 284, 285, 156, 107, 132, 108, 290, - 132, 203, 74, 15, 76, 77, 294, 199, 295, 296, - 191, 12, 183, 124, 207, 204, 297, 227, 300, 6, - 228, 226, 122, 19, 20, 33, 34, 36, 37, 35, - 38, 39, 40, 41, 21, 22, 121, 288, 212, 123, - 178, 213, 211, 262, 23, 24, 25, 26, 27, 28, - 29, 287, 193, 272, 30, 31, 32, 18, 79, 209, - 12, 177, 210, 208, 256, 257, 293, 78, 6, 303, - 16, 17, 19, 20, 33, 34, 36, 37, 35, 38, - 39, 40, 41, 21, 22, 224, 301, 221, 225, 223, - 222, 220, 292, 23, 24, 25, 26, 27, 28, 29, - 278, 134, 253, 30, 31, 32, 18, 218, 251, 12, - 219, 217, 215, 180, 161, 216, 214, 6, 179, 16, - 17, 19, 20, 33, 34, 36, 37, 35, 38, 39, - 40, 41, 21, 22, 3, 178, 255, 81, 115, 168, - 72, 67, 23, 24, 25, 26, 27, 28, 29, 177, - 165, 163, 30, 31, 32, 18, 112, 162, 154, 118, - 70, 168, 110, 72, 159, 100, 166, 103, 16, 17, - 102, 59, 116, 111, 104, 106, 105, 117, 113, 114, - 239, 83, 84, 85, 86, 87, 88, 89, 90, 91, - 92, 93, 94, 95, 96, 107, 99, 108, 11, 10, - 9, 14, 8, 268, 13, 7, 69, 1, + 47, 48, 49, 50, 47, 48, 49, 50, 126, 128, + 129, 64, 241, 97, 170, 128, 129, 101, 62, 63, + 211, 238, 178, 212, 210, 290, 287, 82, 134, 309, + 290, 132, 61, 304, 139, 51, 52, 55, 56, 53, + 54, 45, 46, 47, 48, 49, 50, 116, 140, 236, + 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, + 151, 152, 153, 154, 116, 113, 127, 65, 270, 297, + 165, 176, 171, 174, 175, 172, 173, 74, 158, 76, + 77, 209, 113, 104, 106, 105, 177, 114, 115, 241, + 193, 189, 98, 295, 269, 239, 187, 269, 198, 190, + 64, 76, 77, 237, 107, 279, 108, 62, 63, 293, + 277, 182, 185, 237, 201, 202, 203, 64, 272, 273, + 274, 248, 191, 12, 62, 63, 157, 242, 276, 238, + 188, 133, 238, 261, 121, 182, 231, 234, 238, 240, + 116, 243, 132, 97, 246, 101, 247, 188, 238, 235, + 232, 120, 64, 244, 158, 197, 65, 245, 113, 62, + 63, 250, 256, 254, 239, 250, 252, 185, 116, 64, + 251, 260, 64, 65, 199, 197, 62, 63, 307, 62, + 63, 303, 188, 182, 138, 262, 113, 264, 266, 137, + 268, 97, 136, 116, 196, 267, 278, 263, 80, 188, + 97, 236, 188, 280, 282, 183, 73, 158, 65, 249, + 207, 113, 206, 131, 284, 285, 64, 123, 204, 97, + 286, 12, 200, 62, 63, 65, 288, 289, 65, 133, + 192, 122, 294, 184, 124, 125, 15, 208, 205, 302, + 299, 64, 300, 301, 12, 292, 188, 229, 62, 63, + 230, 228, 6, 305, 159, 157, 19, 20, 33, 34, + 36, 37, 35, 38, 39, 40, 41, 21, 22, 291, + 226, 60, 65, 227, 225, 79, 275, 23, 24, 25, + 26, 27, 28, 29, 265, 116, 194, 30, 31, 32, + 18, 78, 223, 308, 12, 224, 222, 65, 306, 158, + 258, 259, 6, 113, 16, 17, 19, 20, 33, 34, + 36, 37, 35, 38, 39, 40, 41, 21, 22, 214, + 296, 179, 215, 213, 283, 255, 253, 23, 24, 25, + 26, 27, 28, 29, 3, 116, 135, 30, 31, 32, + 18, 67, 181, 180, 12, 220, 159, 157, 221, 219, + 179, 178, 6, 113, 16, 17, 19, 20, 33, 34, + 36, 37, 35, 38, 39, 40, 41, 21, 22, 81, + 166, 104, 106, 105, 164, 114, 115, 23, 24, 25, + 26, 27, 28, 29, 163, 155, 119, 30, 31, 32, + 18, 298, 107, 217, 108, 257, 218, 216, 169, 70, + 281, 162, 72, 72, 16, 17, 169, 110, 111, 160, + 100, 167, 103, 83, 84, 85, 86, 87, 88, 89, + 90, 91, 92, 93, 94, 95, 96, 102, 59, 117, + 112, 118, 99, 11, 10, 9, 14, 8, 271, 13, + 7, 69, 1, } var exprPact = [...]int{ - 276, -1000, -40, -1000, -1000, 202, 276, -1000, -1000, -1000, - -1000, -1000, 438, 231, 259, -1000, 340, 331, 230, -1000, + 279, -1000, -41, -1000, -1000, 277, 279, -1000, -1000, -1000, + -1000, -1000, 447, 233, 104, -1000, 334, 318, 225, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, - -1000, -1000, 33, 33, 33, 33, 33, 33, 33, 33, - 33, 33, 33, 33, 33, 33, 33, 202, -1000, 47, - 214, 433, -1000, -1000, -1000, -1000, 156, 147, -40, 300, - 277, -1000, 46, 257, 374, 229, 219, 203, -1000, -1000, - 276, 276, -1000, 276, 276, 276, 276, 276, 276, 276, - 276, 276, 276, 276, 276, 276, 276, -1000, 432, -1000, - -1000, 113, -1000, -1000, 389, -1000, 431, -1000, 425, -1000, - -1000, -1000, 224, 424, 436, 52, -1000, -1000, -1000, -1000, - -1000, -1000, -1000, 415, -1000, 423, 409, 392, 387, 185, - 273, 161, 254, 143, 271, 325, 217, 210, 268, -26, - 17, 17, -22, -22, -67, -67, -67, -67, -56, -56, - -56, -56, -56, -56, -1000, 113, 224, 224, 224, 262, - -1000, 283, -1000, -1000, 184, -1000, 251, -1000, 282, 335, - 314, 388, 383, 363, 361, 293, -1000, -1000, -1000, -1000, - -1000, -1000, 49, 254, 137, 124, 152, 413, 132, 153, - 49, 276, 120, 249, 138, -1000, -1000, 110, -1000, 382, - 208, 113, 209, 389, 376, -1000, 414, 339, -1000, -1000, + -1000, -1000, 37, 37, 37, 37, 37, 37, 37, 37, + 37, 37, 37, 37, 37, 37, 37, 277, -1000, 47, + 380, 430, -1000, -1000, -1000, -1000, 177, 160, -41, 265, + 269, -1000, 46, 256, 379, 219, 216, 211, -1000, -1000, + 279, 279, -1000, 279, 279, 279, 279, 279, 279, 279, + 279, 279, 279, 279, 279, 279, 279, -1000, 429, -1000, + -1000, 330, -1000, -1000, 446, -1000, 428, -1000, 418, -1000, + -1000, -1000, -1000, 213, 414, 451, 52, -1000, -1000, -1000, + -1000, -1000, -1000, -1000, 448, -1000, 395, 394, 387, 386, + 231, 264, 208, 158, 148, 261, 329, 220, 200, 253, + -27, 16, 16, -23, -23, -68, -68, -68, -68, -57, + -57, -57, -57, -57, -57, -1000, 330, 213, 213, 213, + 249, -1000, 276, -1000, -1000, 238, -1000, 241, -1000, 275, + 66, 365, 439, 391, 338, 316, 293, -1000, -1000, -1000, + -1000, -1000, -1000, 126, 158, 188, 144, 205, 92, 153, + 183, 126, 279, 147, 240, 196, -1000, -1000, 192, -1000, + 370, 185, 330, 109, 446, 369, -1000, 443, 345, 198, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, - 136, 196, 27, 196, 315, -49, 224, -49, 70, 71, - 324, 160, 128, -1000, -1000, 119, -1000, 276, 375, -1000, - -1000, 244, -1000, -1000, -1000, -1000, -1000, -1000, 49, 27, - 196, 27, -1000, -1000, 113, -1000, -49, -1000, 197, -1000, - -1000, -1000, 36, 322, 308, 73, 49, 107, -1000, 366, - -1000, 27, -1000, 341, 28, 27, 15, -49, -49, 287, - -1000, -1000, 239, 59, 27, -1000, -1000, -49, 360, -1000, - -1000, 206, 343, 53, -1000, + -1000, -1000, 159, 252, 27, 252, 326, -49, 213, -49, + 138, 113, 317, 154, 136, -1000, -1000, 131, -1000, 279, + 445, -1000, -1000, 235, -1000, -1000, -1000, -1000, -1000, -1000, + 368, 126, 27, 252, 27, -1000, -1000, 330, -1000, -49, + -1000, 53, -1000, -1000, -1000, 36, 310, 286, 135, 126, + 119, -1000, 364, 95, -1000, 27, -1000, 436, 31, 27, + 15, -49, -49, 280, -1000, -1000, 212, -1000, 59, 27, + -1000, -1000, -49, 342, -1000, -1000, 209, 337, 55, -1000, } var exprPgo = [...]int{ - 0, 487, 16, 70, 2, 9, 414, 3, 15, 11, - 486, 485, 484, 483, 7, 482, 481, 480, 479, 478, - 417, 476, 13, 5, 457, 453, 452, 6, 451, 450, - 447, 4, 446, 445, 8, 444, 1, 442, 0, + 0, 492, 16, 82, 2, 9, 384, 3, 15, 11, + 491, 490, 489, 488, 7, 487, 486, 485, 484, 483, + 419, 482, 13, 5, 481, 480, 479, 6, 478, 477, + 462, 4, 461, 460, 8, 459, 1, 458, 457, 0, } var exprR1 = [...]int{ @@ -334,14 +340,14 @@ var exprR1 = [...]int{ 22, 23, 23, 23, 23, 23, 23, 28, 28, 21, 21, 21, 21, 21, 33, 29, 31, 31, 32, 32, 32, 30, 27, 27, 27, 27, 27, 27, 27, 27, - 34, 35, 35, 37, 37, 26, 26, 26, 26, 26, - 26, 26, 24, 24, 24, 24, 24, 24, 24, 25, - 25, 25, 25, 25, 25, 25, 17, 17, 17, 17, + 27, 34, 35, 35, 38, 37, 37, 26, 26, 26, + 26, 26, 26, 26, 24, 24, 24, 24, 24, 24, + 24, 25, 25, 25, 25, 25, 25, 25, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 20, 20, 18, 18, 18, 16, 16, 16, 16, - 16, 16, 16, 16, 16, 12, 12, 12, 12, 12, - 12, 12, 12, 12, 12, 12, 12, 12, 12, 38, - 5, 5, 4, 4, 4, 4, + 17, 17, 17, 20, 20, 18, 18, 18, 16, 16, + 16, 16, 16, 16, 16, 16, 16, 12, 12, 12, + 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, + 12, 39, 5, 5, 4, 4, 4, 4, } var exprR2 = [...]int{ @@ -354,83 +360,83 @@ var exprR2 = [...]int{ 3, 3, 3, 1, 3, 3, 3, 3, 3, 1, 2, 1, 2, 2, 2, 2, 2, 2, 3, 1, 1, 2, 1, 2, 2, 2, 3, 3, 1, 3, - 3, 2, 1, 1, 1, 3, 2, 3, 3, 3, - 3, 1, 3, 1, 1, 3, 3, 3, 3, 3, + 3, 2, 1, 1, 1, 1, 3, 2, 3, 3, + 3, 3, 1, 3, 6, 1, 1, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, - 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, + 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 0, 1, 1, 2, 2, 1, 1, 1, 1, + 4, 4, 4, 0, 1, 1, 2, 2, 1, 1, + 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, - 1, 3, 4, 4, 3, 3, + 1, 2, 1, 3, 4, 4, 3, 3, } var exprChk = [...]int{ -1000, -1, -2, -6, -7, -14, 23, -11, -15, -17, - -18, -19, 15, -12, -16, 7, 74, 75, 61, 27, + -18, -19, 15, -12, -16, 7, 75, 76, 61, 27, 28, 38, 39, 48, 49, 50, 51, 52, 53, 54, 58, 59, 60, 29, 30, 33, 31, 32, 34, 35, - 36, 37, 65, 66, 67, 74, 75, 76, 77, 78, - 79, 68, 69, 72, 73, 70, 71, -22, -23, -28, - 44, -3, 21, 22, 14, 69, -7, -6, -2, -10, + 36, 37, 66, 67, 68, 75, 76, 77, 78, 79, + 80, 69, 70, 73, 74, 71, 72, -22, -23, -28, + 44, -3, 21, 22, 14, 70, -7, -6, -2, -10, 2, -9, 5, 23, 23, -4, 25, 26, 7, 7, 23, -20, 40, -20, -20, -20, -20, -20, -20, -20, -20, -20, -20, -20, -20, -20, -20, -23, -3, -21, -33, -27, -29, -30, 41, 43, 42, 62, 64, -9, - -37, -25, 23, 45, 46, 5, -26, -24, 6, 24, - 24, 16, 2, 19, 16, 12, 69, 13, 14, -8, - 7, -14, 23, -7, 7, 23, 23, 23, -7, -2, + -38, -37, -25, 23, 45, 46, 5, -26, -24, 6, + 24, 24, 16, 2, 19, 16, 12, 70, 13, 14, + -8, 7, -14, 23, -7, 7, 23, 23, 23, -7, -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, - -2, -2, -2, -2, 6, -27, 66, 19, 65, -35, - -34, 5, 6, 6, -27, 6, -32, -31, 5, 12, - 69, 72, 73, 70, 71, 68, -9, 6, 6, 6, - 6, 2, 24, 19, 9, -36, -22, 44, -14, -8, - 24, 19, -7, 7, -5, 24, 5, -5, 24, 19, - -27, -27, -27, 19, 12, 24, 19, 12, 8, 4, - 7, 8, 4, 7, 8, 4, 7, 8, 4, 7, - 8, 4, 7, 8, 4, 7, 8, 4, 7, -4, - -8, -38, -36, -22, 63, 9, 44, 9, -36, 47, - 24, -36, -22, 24, -4, -7, 24, 19, 19, 24, - 24, 6, -34, 6, -31, 2, 5, 6, 24, -36, - -22, -36, 8, -38, -27, -38, 9, 5, -13, 55, - 56, 57, 9, 24, 24, -36, 24, -7, 5, 19, - -4, -36, -38, 23, -38, -36, 44, 9, 9, 24, - -4, 24, 6, 5, -36, -38, -38, 9, 19, 24, - -38, 6, 19, 6, 24, + -2, -2, -2, -2, -2, 6, -27, 67, 19, 66, + -35, -34, 5, 6, 6, -27, 6, -32, -31, 5, + 12, 70, 73, 74, 71, 72, 69, -9, 6, 6, + 6, 6, 2, 24, 19, 9, -36, -22, 44, -14, + -8, 24, 19, -7, 7, -5, 24, 5, -5, 24, + 19, -27, -27, -27, 19, 12, 24, 19, 12, 65, + 8, 4, 7, 8, 4, 7, 8, 4, 7, 8, + 4, 7, 8, 4, 7, 8, 4, 7, 8, 4, + 7, -4, -8, -39, -36, -22, 63, 9, 44, 9, + -36, 47, 24, -36, -22, 24, -4, -7, 24, 19, + 19, 24, 24, 6, -34, 6, -31, 2, 5, 6, + 23, 24, -36, -22, -36, 8, -39, -27, -39, 9, + 5, -13, 55, 56, 57, 9, 24, 24, -36, 24, + -7, 5, 19, 6, -4, -36, -39, 23, -39, -36, + 44, 9, 9, 24, -4, 24, 6, 24, 5, -36, + -39, -39, 9, 19, 24, -39, 6, 19, 6, 24, } var exprDef = [...]int{ 0, -2, 1, 2, 3, 10, 0, 4, 5, 6, - 7, 8, 0, 0, 0, 143, 0, 0, 0, 155, - 156, 157, 158, 159, 160, 161, 162, 163, 164, 165, - 166, 167, 168, 146, 147, 148, 149, 150, 151, 152, - 153, 154, 141, 141, 141, 141, 141, 141, 141, 141, - 141, 141, 141, 141, 141, 141, 141, 11, 69, 71, + 7, 8, 0, 0, 0, 145, 0, 0, 0, 157, + 158, 159, 160, 161, 162, 163, 164, 165, 166, 167, + 168, 169, 170, 148, 149, 150, 151, 152, 153, 154, + 155, 156, 143, 143, 143, 143, 143, 143, 143, 143, + 143, 143, 143, 143, 143, 143, 143, 11, 69, 71, 0, 0, 56, 57, 58, 59, 3, 2, 0, 0, - 0, 63, 0, 0, 0, 0, 0, 0, 144, 145, - 0, 0, 142, 0, 0, 0, 0, 0, 0, 0, + 0, 63, 0, 0, 0, 0, 0, 0, 146, 147, + 0, 0, 144, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 70, 0, 72, 73, 74, 75, 76, 79, 80, 0, 82, 0, 92, - 93, 94, 0, 0, 0, 0, 103, 104, 77, 9, - 12, 60, 61, 0, 62, 0, 0, 0, 0, 0, - 0, 0, 0, 3, 143, 0, 0, 0, 3, 126, - 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, - 137, 138, 139, 140, 78, 96, 0, 0, 0, 84, - 101, 0, 81, 83, 0, 85, 91, 88, 0, 0, - 0, 0, 0, 0, 0, 0, 64, 65, 66, 67, - 68, 38, 45, 0, 13, 0, 0, 0, 0, 0, - 49, 0, 3, 143, 0, 174, 170, 0, 175, 0, - 97, 98, 99, 0, 0, 95, 0, 0, 110, 117, - 124, 109, 116, 123, 105, 112, 119, 106, 113, 120, - 107, 114, 121, 108, 115, 122, 111, 118, 125, 47, - 0, 14, 17, 33, 0, 21, 0, 25, 0, 0, - 0, 0, 0, 37, 51, 3, 50, 0, 0, 172, - 173, 0, 102, 100, 89, 90, 86, 87, 46, 18, - 34, 35, 169, 22, 41, 26, 29, 39, 0, 42, - 43, 44, 15, 0, 0, 0, 52, 3, 171, 0, - 48, 36, 30, 0, 16, 19, 0, 23, 27, 0, - 53, 54, 0, 0, 20, 24, 28, 31, 0, 40, - 32, 0, 0, 0, 55, + 93, 94, 95, 0, 0, 0, 0, 105, 106, 77, + 9, 12, 60, 61, 0, 62, 0, 0, 0, 0, + 0, 0, 0, 0, 3, 145, 0, 0, 0, 3, + 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, + 138, 139, 140, 141, 142, 78, 97, 0, 0, 0, + 84, 102, 0, 81, 83, 0, 85, 91, 88, 0, + 0, 0, 0, 0, 0, 0, 0, 64, 65, 66, + 67, 68, 38, 45, 0, 13, 0, 0, 0, 0, + 0, 49, 0, 3, 145, 0, 176, 172, 0, 177, + 0, 98, 99, 100, 0, 0, 96, 0, 0, 0, + 112, 119, 126, 111, 118, 125, 107, 114, 121, 108, + 115, 122, 109, 116, 123, 110, 117, 124, 113, 120, + 127, 47, 0, 14, 17, 33, 0, 21, 0, 25, + 0, 0, 0, 0, 0, 37, 51, 3, 50, 0, + 0, 174, 175, 0, 103, 101, 89, 90, 86, 87, + 0, 46, 18, 34, 35, 171, 22, 41, 26, 29, + 39, 0, 42, 43, 44, 15, 0, 0, 0, 52, + 3, 173, 0, 0, 48, 36, 30, 0, 16, 19, + 0, 23, 27, 0, 53, 54, 0, 104, 0, 20, + 24, 28, 31, 0, 40, 32, 0, 0, 0, 55, } var exprTok1 = [...]int{ @@ -445,7 +451,7 @@ var exprTok2 = [...]int{ 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, - 72, 73, 74, 75, 76, 77, 78, 79, + 72, 73, 74, 75, 76, 77, 78, 79, 80, } var exprTok3 = [...]int{ 0, @@ -457,6 +463,7 @@ var exprErrorMessages = [...]struct { msg string }{} + /* parser for yacc output */ var ( @@ -1148,7 +1155,7 @@ exprdefault: case 74: exprDollar = exprS[exprpt-2 : exprpt+1] { - exprVAL.PipelineStage = &LabelFilterExpr{LabelFilterer: exprDollar[2].LabelFilter} + exprVAL.PipelineStage = &labelFilterExpr{LabelFilterer: exprDollar[2].LabelFilter} } case 75: exprDollar = exprS[exprpt-2 : exprpt+1] @@ -1238,414 +1245,424 @@ exprdefault: case 93: exprDollar = exprS[exprpt-1 : exprpt+1] { - exprVAL.LabelFilter = exprDollar[1].UnitFilter + exprVAL.LabelFilter = exprDollar[1].IPFilter } case 94: exprDollar = exprS[exprpt-1 : exprpt+1] { - exprVAL.LabelFilter = exprDollar[1].NumberFilter + exprVAL.LabelFilter = exprDollar[1].UnitFilter } case 95: + exprDollar = exprS[exprpt-1 : exprpt+1] + { + exprVAL.LabelFilter = exprDollar[1].NumberFilter + } + case 96: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.LabelFilter = exprDollar[2].LabelFilter } - case 96: + case 97: exprDollar = exprS[exprpt-2 : exprpt+1] { exprVAL.LabelFilter = log.NewAndLabelFilter(exprDollar[1].LabelFilter, exprDollar[2].LabelFilter) } - case 97: + case 98: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.LabelFilter = log.NewAndLabelFilter(exprDollar[1].LabelFilter, exprDollar[3].LabelFilter) } - case 98: + case 99: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.LabelFilter = log.NewAndLabelFilter(exprDollar[1].LabelFilter, exprDollar[3].LabelFilter) } - case 99: + case 100: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.LabelFilter = log.NewOrLabelFilter(exprDollar[1].LabelFilter, exprDollar[3].LabelFilter) } - case 100: + case 101: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.JSONExpression = log.NewJSONExpr(exprDollar[1].str, exprDollar[3].str) } - case 101: + case 102: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.JSONExpressionList = []log.JSONExpression{exprDollar[1].JSONExpression} } - case 102: + case 103: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.JSONExpressionList = append(exprDollar[1].JSONExpressionList, exprDollar[3].JSONExpression) } - case 103: + case 104: + exprDollar = exprS[exprpt-6 : exprpt+1] + { + exprVAL.IPFilter = log.NewIPLabelFilter(exprDollar[5].str, exprDollar[1].str) + } + case 105: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.UnitFilter = exprDollar[1].DurationFilter } - case 104: + case 106: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.UnitFilter = exprDollar[1].BytesFilter } - case 105: + case 107: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.DurationFilter = log.NewDurationLabelFilter(log.LabelFilterGreaterThan, exprDollar[1].str, exprDollar[3].duration) } - case 106: + case 108: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.DurationFilter = log.NewDurationLabelFilter(log.LabelFilterGreaterThanOrEqual, exprDollar[1].str, exprDollar[3].duration) } - case 107: + case 109: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.DurationFilter = log.NewDurationLabelFilter(log.LabelFilterLesserThan, exprDollar[1].str, exprDollar[3].duration) } - case 108: + case 110: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.DurationFilter = log.NewDurationLabelFilter(log.LabelFilterLesserThanOrEqual, exprDollar[1].str, exprDollar[3].duration) } - case 109: + case 111: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.DurationFilter = log.NewDurationLabelFilter(log.LabelFilterNotEqual, exprDollar[1].str, exprDollar[3].duration) } - case 110: + case 112: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.DurationFilter = log.NewDurationLabelFilter(log.LabelFilterEqual, exprDollar[1].str, exprDollar[3].duration) } - case 111: + case 113: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.DurationFilter = log.NewDurationLabelFilter(log.LabelFilterEqual, exprDollar[1].str, exprDollar[3].duration) } - case 112: + case 114: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.BytesFilter = log.NewBytesLabelFilter(log.LabelFilterGreaterThan, exprDollar[1].str, exprDollar[3].bytes) } - case 113: + case 115: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.BytesFilter = log.NewBytesLabelFilter(log.LabelFilterGreaterThanOrEqual, exprDollar[1].str, exprDollar[3].bytes) } - case 114: + case 116: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.BytesFilter = log.NewBytesLabelFilter(log.LabelFilterLesserThan, exprDollar[1].str, exprDollar[3].bytes) } - case 115: + case 117: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.BytesFilter = log.NewBytesLabelFilter(log.LabelFilterLesserThanOrEqual, exprDollar[1].str, exprDollar[3].bytes) } - case 116: + case 118: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.BytesFilter = log.NewBytesLabelFilter(log.LabelFilterNotEqual, exprDollar[1].str, exprDollar[3].bytes) } - case 117: + case 119: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.BytesFilter = log.NewBytesLabelFilter(log.LabelFilterEqual, exprDollar[1].str, exprDollar[3].bytes) } - case 118: + case 120: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.BytesFilter = log.NewBytesLabelFilter(log.LabelFilterEqual, exprDollar[1].str, exprDollar[3].bytes) } - case 119: + case 121: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.NumberFilter = log.NewNumericLabelFilter(log.LabelFilterGreaterThan, exprDollar[1].str, mustNewFloat(exprDollar[3].str)) } - case 120: + case 122: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.NumberFilter = log.NewNumericLabelFilter(log.LabelFilterGreaterThanOrEqual, exprDollar[1].str, mustNewFloat(exprDollar[3].str)) } - case 121: + case 123: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.NumberFilter = log.NewNumericLabelFilter(log.LabelFilterLesserThan, exprDollar[1].str, mustNewFloat(exprDollar[3].str)) } - case 122: + case 124: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.NumberFilter = log.NewNumericLabelFilter(log.LabelFilterLesserThanOrEqual, exprDollar[1].str, mustNewFloat(exprDollar[3].str)) } - case 123: + case 125: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.NumberFilter = log.NewNumericLabelFilter(log.LabelFilterNotEqual, exprDollar[1].str, mustNewFloat(exprDollar[3].str)) } - case 124: + case 126: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.NumberFilter = log.NewNumericLabelFilter(log.LabelFilterEqual, exprDollar[1].str, mustNewFloat(exprDollar[3].str)) } - case 125: + case 127: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.NumberFilter = log.NewNumericLabelFilter(log.LabelFilterEqual, exprDollar[1].str, mustNewFloat(exprDollar[3].str)) } - case 126: + case 128: exprDollar = exprS[exprpt-4 : exprpt+1] { exprVAL.BinOpExpr = mustNewBinOpExpr("or", exprDollar[3].BinOpModifier, exprDollar[1].Expr, exprDollar[4].Expr) } - case 127: + case 129: exprDollar = exprS[exprpt-4 : exprpt+1] { exprVAL.BinOpExpr = mustNewBinOpExpr("and", exprDollar[3].BinOpModifier, exprDollar[1].Expr, exprDollar[4].Expr) } - case 128: + case 130: exprDollar = exprS[exprpt-4 : exprpt+1] { exprVAL.BinOpExpr = mustNewBinOpExpr("unless", exprDollar[3].BinOpModifier, exprDollar[1].Expr, exprDollar[4].Expr) } - case 129: + case 131: exprDollar = exprS[exprpt-4 : exprpt+1] { exprVAL.BinOpExpr = mustNewBinOpExpr("+", exprDollar[3].BinOpModifier, exprDollar[1].Expr, exprDollar[4].Expr) } - case 130: + case 132: exprDollar = exprS[exprpt-4 : exprpt+1] { exprVAL.BinOpExpr = mustNewBinOpExpr("-", exprDollar[3].BinOpModifier, exprDollar[1].Expr, exprDollar[4].Expr) } - case 131: + case 133: exprDollar = exprS[exprpt-4 : exprpt+1] { exprVAL.BinOpExpr = mustNewBinOpExpr("*", exprDollar[3].BinOpModifier, exprDollar[1].Expr, exprDollar[4].Expr) } - case 132: + case 134: exprDollar = exprS[exprpt-4 : exprpt+1] { exprVAL.BinOpExpr = mustNewBinOpExpr("/", exprDollar[3].BinOpModifier, exprDollar[1].Expr, exprDollar[4].Expr) } - case 133: + case 135: exprDollar = exprS[exprpt-4 : exprpt+1] { exprVAL.BinOpExpr = mustNewBinOpExpr("%", exprDollar[3].BinOpModifier, exprDollar[1].Expr, exprDollar[4].Expr) } - case 134: + case 136: exprDollar = exprS[exprpt-4 : exprpt+1] { exprVAL.BinOpExpr = mustNewBinOpExpr("^", exprDollar[3].BinOpModifier, exprDollar[1].Expr, exprDollar[4].Expr) } - case 135: + case 137: exprDollar = exprS[exprpt-4 : exprpt+1] { exprVAL.BinOpExpr = mustNewBinOpExpr("==", exprDollar[3].BinOpModifier, exprDollar[1].Expr, exprDollar[4].Expr) } - case 136: + case 138: exprDollar = exprS[exprpt-4 : exprpt+1] { exprVAL.BinOpExpr = mustNewBinOpExpr("!=", exprDollar[3].BinOpModifier, exprDollar[1].Expr, exprDollar[4].Expr) } - case 137: + case 139: exprDollar = exprS[exprpt-4 : exprpt+1] { exprVAL.BinOpExpr = mustNewBinOpExpr(">", exprDollar[3].BinOpModifier, exprDollar[1].Expr, exprDollar[4].Expr) } - case 138: + case 140: exprDollar = exprS[exprpt-4 : exprpt+1] { exprVAL.BinOpExpr = mustNewBinOpExpr(">=", exprDollar[3].BinOpModifier, exprDollar[1].Expr, exprDollar[4].Expr) } - case 139: + case 141: exprDollar = exprS[exprpt-4 : exprpt+1] { exprVAL.BinOpExpr = mustNewBinOpExpr("<", exprDollar[3].BinOpModifier, exprDollar[1].Expr, exprDollar[4].Expr) } - case 140: + case 142: exprDollar = exprS[exprpt-4 : exprpt+1] { exprVAL.BinOpExpr = mustNewBinOpExpr("<=", exprDollar[3].BinOpModifier, exprDollar[1].Expr, exprDollar[4].Expr) } - case 141: + case 143: exprDollar = exprS[exprpt-0 : exprpt+1] { exprVAL.BinOpModifier = BinOpOptions{} } - case 142: + case 144: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.BinOpModifier = BinOpOptions{ReturnBool: true} } - case 143: + case 145: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.LiteralExpr = mustNewLiteralExpr(exprDollar[1].str, false) } - case 144: + case 146: exprDollar = exprS[exprpt-2 : exprpt+1] { exprVAL.LiteralExpr = mustNewLiteralExpr(exprDollar[2].str, false) } - case 145: + case 147: exprDollar = exprS[exprpt-2 : exprpt+1] { exprVAL.LiteralExpr = mustNewLiteralExpr(exprDollar[2].str, true) } - case 146: + case 148: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.VectorOp = OpTypeSum } - case 147: + case 149: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.VectorOp = OpTypeAvg } - case 148: + case 150: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.VectorOp = OpTypeCount } - case 149: + case 151: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.VectorOp = OpTypeMax } - case 150: + case 152: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.VectorOp = OpTypeMin } - case 151: + case 153: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.VectorOp = OpTypeStddev } - case 152: + case 154: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.VectorOp = OpTypeStdvar } - case 153: + case 155: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.VectorOp = OpTypeBottomK } - case 154: + case 156: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.VectorOp = OpTypeTopK } - case 155: + case 157: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.RangeOp = OpRangeTypeCount } - case 156: + case 158: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.RangeOp = OpRangeTypeRate } - case 157: + case 159: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.RangeOp = OpRangeTypeBytes } - case 158: + case 160: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.RangeOp = OpRangeTypeBytesRate } - case 159: + case 161: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.RangeOp = OpRangeTypeAvg } - case 160: + case 162: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.RangeOp = OpRangeTypeSum } - case 161: + case 163: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.RangeOp = OpRangeTypeMin } - case 162: + case 164: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.RangeOp = OpRangeTypeMax } - case 163: + case 165: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.RangeOp = OpRangeTypeStdvar } - case 164: + case 166: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.RangeOp = OpRangeTypeStddev } - case 165: + case 167: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.RangeOp = OpRangeTypeQuantile } - case 166: + case 168: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.RangeOp = OpRangeTypeFirst } - case 167: + case 169: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.RangeOp = OpRangeTypeLast } - case 168: + case 170: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.RangeOp = OpRangeTypeAbsent } - case 169: + case 171: exprDollar = exprS[exprpt-2 : exprpt+1] { exprVAL.OffsetExpr = newOffsetExpr(exprDollar[2].duration) } - case 170: + case 172: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.Labels = []string{exprDollar[1].str} } - case 171: + case 173: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.Labels = append(exprDollar[1].Labels, exprDollar[3].str) } - case 172: + case 174: exprDollar = exprS[exprpt-4 : exprpt+1] { exprVAL.Grouping = &grouping{without: false, groups: exprDollar[3].Labels} } - case 173: + case 175: exprDollar = exprS[exprpt-4 : exprpt+1] { exprVAL.Grouping = &grouping{without: true, groups: exprDollar[3].Labels} } - case 174: + case 176: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.Grouping = &grouping{without: false, groups: nil} } - case 175: + case 177: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.Grouping = &grouping{without: true, groups: nil} diff --git a/pkg/logql/lex.go b/pkg/logql/lex.go index 85ba038034538..aa4f5a0d1eea2 100644 --- a/pkg/logql/lex.go +++ b/pkg/logql/lex.go @@ -64,6 +64,9 @@ var tokens = map[string]int{ // fmt OpFmtLabel: LABEL_FMT, OpFmtLine: LINE_FMT, + + // filter functions + OpFilterIP: IP, } // functionTokens are tokens that needs to be suffixes with parenthesis diff --git a/pkg/logql/log/ip.go b/pkg/logql/log/ip.go index a7015e1bc880a..2d8c251c28609 100644 --- a/pkg/logql/log/ip.go +++ b/pkg/logql/log/ip.go @@ -13,16 +13,30 @@ var ( errInvalidPattern = errors.New("invalid pattern") ) +type IPMatchType int + const ( IPV4_CHARSET = "0123456789." IPV6_CHARSET = "0123456789abcdefABCDEF.:" + + IPMatchLine IPMatchType = iota + IPMatchLabel ) // Should be one of the netaddr.IP, netaddr.IPRange, netadd.IPPrefix. type IPMatcher interface{} type IPFilter struct { - matcher IPMatcher + pattern string + matcher IPMatcher + matchType IPMatchType + + // if used as label matcher, this holds the identifier label name. + // e.g: (|remote_addr = ip("xxx")). Here labelName is `remote_addr` + labelName string + + // patternError represents any invalid pattern provided to match the ip. + patternError error } // IPFilter search for IP addresses of given `pattern` in the given `line`. @@ -32,12 +46,21 @@ type IPFilter struct { // 1. SINGLE-IP - "192.168.0.1" // 2. IP RANGE - "192.168.0.1-192.168.0.23" // 3. CIDR - "192.168.0.0/16" -func NewIPFilter(pattern string) (*IPFilter, error) { +func NewIPFilter(pattern string) *IPFilter { + filter := &IPFilter{pattern: pattern} + matcher, err := getMatcher(pattern) - if err != nil { - return nil, err - } - return &IPFilter{matcher: matcher}, nil + filter.matcher = matcher + filter.patternError = err + + return filter +} + +func NewIPLabelFilter(pattern string, label string) *IPFilter { + filter := NewIPFilter(pattern) + filter.labelName = label + filter.matchType = IPMatchLabel + return filter } func (ipf *IPFilter) Filter(line string) bool { @@ -77,14 +100,42 @@ func (ipf *IPFilter) Filter(line string) bool { return false } -func (ipf *IPFilter) Process(line []byte, _ *LabelsBuilder) ([]byte, bool) { - return line, ipf.Filter(string(line)) +// `Process` implements `Stage` interface +func (ipf *IPFilter) Process(line []byte, lbs *LabelsBuilder) ([]byte, bool) { + + // make sure the pattern provided was valid, even before trying to match. + if ipf.patternError != nil { + lbs.SetErr(fmt.Errorf("%s: %s", errLabelFilter, ipf.patternError.Error()).Error()) + return line, false + } + + input := string(line) + + if ipf.matchType == IPMatchLabel { + v, ok := lbs.Get(ipf.labelName) + if !ok { + // we have not found the corresponding label. + return line, false + } + input = v + } + + return line, ipf.Filter(input) } +// `RequiredLabelNames` implements `Stage` interface func (ipf *IPFilter) RequiredLabelNames() []string { return []string{} } +// `String` implements fmt.Stringer inteface, by which also implements `LabelFilterer` inteface. +func (ipf *IPFilter) String() string { + if ipf.matchType == IPMatchLabel { + return fmt.Sprintf("%s=ip(%q)", ipf.labelName, ipf.pattern) + } + return fmt.Sprintf("ip(%q)", ipf.pattern) +} + func contains(matcher IPMatcher, ip netaddr.IP) bool { switch m := matcher.(type) { case netaddr.IP: diff --git a/pkg/logql/parser_test.go b/pkg/logql/parser_test.go index e6a2c677ea55d..919ec5cb4a5f8 100644 --- a/pkg/logql/parser_test.go +++ b/pkg/logql/parser_test.go @@ -338,6 +338,29 @@ func TestParse(t *testing.T) { in: `min({ foo = "bar" }[5m])`, err: logqlmodel.NewParseError("syntax error: unexpected RANGE", 0, 20), }, + { + in: `{ foo = "bar" }|logfmt|addr!=ip("1.2.3.4")`, + err: logqlmodel.NewParseError("syntax error: unexpected ip, expecting BYTES or STRING or NUMBER or DURATION", 1, 30), + }, + { + in: `{ foo = "bar" }|logfmt|addr>=ip("1.2.3.4")`, + err: logqlmodel.NewParseError("syntax error: unexpected ip, expecting BYTES or NUMBER or DURATION", 1, 30), + }, + { + in: `{ foo = "bar" }|logfmt|addr>ip("1.2.3.4")`, + err: logqlmodel.NewParseError("syntax error: unexpected ip, expecting BYTES or NUMBER or DURATION", 1, 29), + }, + { + in: `{ foo = "bar" }|logfmt|addr<=ip("1.2.3.4")`, + err: logqlmodel.NewParseError("syntax error: unexpected ip, expecting BYTES or NUMBER or DURATION", 1, 30), + }, + { + in: `{ foo = "bar" }|logfmt|addr Date: Mon, 12 Jul 2021 22:30:22 +0200 Subject: [PATCH 03/24] Label filters all test cases passes Signed-off-by: Kaviraj --- pkg/logql/ast.go | 8 +++++++- pkg/logql/parser_test.go | 28 ++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/pkg/logql/ast.go b/pkg/logql/ast.go index 7a784d45fdddd..5818146039f42 100644 --- a/pkg/logql/ast.go +++ b/pkg/logql/ast.go @@ -394,7 +394,13 @@ type LabelFilterExpr struct { implicit } -func (e *LabelFilterExpr) Shardable() bool { return true } +func newLabelFilterExpr(filterer log.LabelFilterer) *labelFilterExpr { + return &labelFilterExpr{ + LabelFilterer: filterer, + } +} + +func (e *labelFilterExpr) Shardable() bool { return true } func (e *LabelFilterExpr) Walk(f WalkFn) { f(e) } diff --git a/pkg/logql/parser_test.go b/pkg/logql/parser_test.go index 919ec5cb4a5f8..3ad48ef993bb8 100644 --- a/pkg/logql/parser_test.go +++ b/pkg/logql/parser_test.go @@ -360,7 +360,35 @@ func TestParse(t *testing.T) { }, { in: `{ foo = "bar" }|logfmt|addr=ip("1.2.3.4")`, + exp: newPipelineExpr( + newMatcherExpr([]*labels.Matcher{mustNewMatcher(labels.MatchEqual, "foo", "bar")}), + MultiStageExpr{newLabelParserExpr(OpParserTypeLogfmt, ""), newLabelFilterExpr(log.NewIPLabelFilter("1.2.3.4", "addr"))}, + ), + }, + { + in: `{ foo = "bar" }|logfmt|level="error"| addr=ip("1.2.3.4")`, + exp: newPipelineExpr( + newMatcherExpr([]*labels.Matcher{mustNewMatcher(labels.MatchEqual, "foo", "bar")}), + MultiStageExpr{ + newLabelParserExpr(OpParserTypeLogfmt, ""), + newLabelFilterExpr(log.NewStringLabelFilter(mustNewMatcher(labels.MatchEqual, "level", "error"))), + newLabelFilterExpr(log.NewIPLabelFilter("1.2.3.4", "addr")), + }, + ), }, + { + in: `{ foo = "bar" }|logfmt|remote_addr=ip("2.3.4.5")|level="error"| addr=ip("1.2.3.4")`, // chain label filters with ip matcher + exp: newPipelineExpr( + newMatcherExpr([]*labels.Matcher{mustNewMatcher(labels.MatchEqual, "foo", "bar")}), + MultiStageExpr{ + newLabelParserExpr(OpParserTypeLogfmt, ""), + newLabelFilterExpr(log.NewIPLabelFilter("2.3.4.5", "remote_addr")), + newLabelFilterExpr(log.NewStringLabelFilter(mustNewMatcher(labels.MatchEqual, "level", "error"))), + newLabelFilterExpr(log.NewIPLabelFilter("1.2.3.4", "addr")), + }, + ), + }, + { in: `sum(3 ,count_over_time({ foo = "bar" }[5h]))`, err: logqlmodel.NewParseError("unsupported parameter for operation sum(3,", 0, 0), From 6810cfe8565b6f4060bf0a75454aa83f0ddf2c96 Mon Sep 17 00:00:00 2001 From: Kaviraj Date: Mon, 12 Jul 2021 22:34:49 +0200 Subject: [PATCH 04/24] Rename ipFilter -> ipLabelFilter Signed-off-by: Kaviraj --- pkg/logql/expr.y | 10 +++++----- pkg/logql/expr.y.go | 6 +++--- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/pkg/logql/expr.y b/pkg/logql/expr.y index b58368c813cda..8a49017bb6a59 100644 --- a/pkg/logql/expr.y +++ b/pkg/logql/expr.y @@ -42,7 +42,7 @@ import ( DurationFilter log.LabelFilterer LabelFilter log.LabelFilterer UnitFilter log.LabelFilterer - IPFilter log.LabelFilterer + IPLabelFilter log.LabelFilterer LineFormatExpr *lineFmtExpr LabelFormatExpr *labelFmtExpr LabelFormat log.LabelFmt @@ -92,7 +92,7 @@ import ( %type jsonExpressionList %type unwrapExpr %type unitFilter -%type ipFilter +%type ipLabelFilter %type offsetExpr %token BYTES @@ -271,7 +271,7 @@ labelFormatExpr: LABEL_FMT labelsFormat { $$ = newLabelFmtExpr($2) }; labelFilter: matcher { $$ = log.NewStringLabelFilter($1) } - | ipFilter { $$ = $1 } + | ipLabelFilter { $$ = $1 } | unitFilter { $$ = $1 } | numberFilter { $$ = $1 } | OPEN_PARENTHESIS labelFilter CLOSE_PARENTHESIS { $$ = $2 } @@ -289,8 +289,8 @@ jsonExpressionList: | jsonExpressionList COMMA jsonExpression { $$ = append($1, $3) } ; -ipFilter: -IDENTIFIER EQ IP OPEN_PARENTHESIS STRING CLOSE_PARENTHESIS { $$ = log.NewIPLabelFilter($5, $1) } +ipLabelFilter: + IDENTIFIER EQ IP OPEN_PARENTHESIS STRING CLOSE_PARENTHESIS { $$ = log.NewIPLabelFilter($5, $1) } unitFilter: durationFilter { $$ = $1 } diff --git a/pkg/logql/expr.y.go b/pkg/logql/expr.y.go index ea56e5719935f..cf0c5e69c91bc 100644 --- a/pkg/logql/expr.y.go +++ b/pkg/logql/expr.y.go @@ -45,7 +45,7 @@ type exprSymType struct { DurationFilter log.LabelFilterer LabelFilter log.LabelFilterer UnitFilter log.LabelFilterer - IPFilter log.LabelFilterer + IPLabelFilter log.LabelFilterer LineFormatExpr *lineFmtExpr LabelFormatExpr *labelFmtExpr LabelFormat log.LabelFmt @@ -1245,7 +1245,7 @@ exprdefault: case 93: exprDollar = exprS[exprpt-1 : exprpt+1] { - exprVAL.LabelFilter = exprDollar[1].IPFilter + exprVAL.LabelFilter = exprDollar[1].IPLabelFilter } case 94: exprDollar = exprS[exprpt-1 : exprpt+1] @@ -1300,7 +1300,7 @@ exprdefault: case 104: exprDollar = exprS[exprpt-6 : exprpt+1] { - exprVAL.IPFilter = log.NewIPLabelFilter(exprDollar[5].str, exprDollar[1].str) + exprVAL.IPLabelFilter = log.NewIPLabelFilter(exprDollar[5].str, exprDollar[1].str) } case 105: exprDollar = exprS[exprpt-1 : exprpt+1] From ae7b03b08655ede99be2a757cb7d38710ce4f2f7 Mon Sep 17 00:00:00 2001 From: Kaviraj Date: Tue, 13 Jul 2021 00:31:15 +0200 Subject: [PATCH 05/24] More tests Signed-off-by: Kaviraj --- pkg/logql/ast_test.go | 2 +- pkg/logql/log/ip.go | 58 +++++++++++++++++++++++++++----------- pkg/logql/log/ip_test.go | 60 ++++++++++++++++++++++++++++++++++------ 3 files changed, 94 insertions(+), 26 deletions(-) diff --git a/pkg/logql/ast_test.go b/pkg/logql/ast_test.go index 474dccb2bd009..750546d6391ed 100644 --- a/pkg/logql/ast_test.go +++ b/pkg/logql/ast_test.go @@ -32,7 +32,7 @@ func Test_logSelectorExpr_String(t *testing.T) { {`{foo="bar"} |= "baz" |~ "blip" != "flip" !~ "flap" | logfmt | b>=10GB`, true}, {`{foo="bar"} |= "baz" |~ "blip" != "flip" !~ "flap" | logfmt | b=ip("127.0.0.1")`, true}, {`{foo="bar"} |= "baz" |~ "blip" != "flip" !~ "flap" | logfmt | b=ip("127.0.0.1") | level="error"`, true}, - {`{foo="bar"} |= "baz" |~ "blip" != "flip" !~ "flap" | logfmt | b=ip("127.0.0.1") | level="error" | c=ip("1.2.3.4")`, true}, // chain inside label filters. + {`{foo="bar"} |= "baz" |~ "blip" != "flip" !~ "flap" | logfmt | b=ip("127.0.0.1") | level="error" | c=ip("::1")`, true}, // chain inside label filters. {`{foo="bar"} |= "baz" |~ "blip" != "flip" !~ "flap" | regexp "(?Pfoo|bar)"`, true}, {`{foo="bar"} |= "baz" |~ "blip" != "flip" !~ "flap" | regexp "(?Pfoo|bar)" | ( ( foo<5.01 , bar>20ms ) or foo="bar" ) | line_format "blip{{.boop}}bap" | label_format foo=bar,bar="blip{{.blop}}"`, true}, } diff --git a/pkg/logql/log/ip.go b/pkg/logql/log/ip.go index 2d8c251c28609..b5874c7b11278 100644 --- a/pkg/logql/log/ip.go +++ b/pkg/logql/log/ip.go @@ -3,7 +3,6 @@ package log import ( "errors" "fmt" - "strings" "unicode" "inet.af/netaddr" @@ -17,7 +16,7 @@ type IPMatchType int const ( IPV4_CHARSET = "0123456789." - IPV6_CHARSET = "0123456789abcdefABCDEF.:" + IPV6_CHARSET = "0123456789abcdefABCDEF:." IPMatchLine IPMatchType = iota IPMatchLabel @@ -70,28 +69,36 @@ func (ipf *IPFilter) Filter(line string) bool { n := len(line) + filterFn := func(line string, start int, charset string) (bool, int) { + iplen := stringSpan(line[start:], charset) + if iplen < 0 { + return false, 0 + } + ip, err := netaddr.ParseIP(line[start : start+iplen]) + if err == nil { + if contains(ipf.matcher, ip) { + return true, 0 + } + } + return false, iplen + } + + // This loop try to extract IPv4 or IPv6 address from the arbitrary string. + // It uses IPv4 and IPv6 prefix hints to find the IP addresses faster without using regexp. for i := 0; i < n; i++ { if i+3 < n && ipv4Hint([4]byte{line[i], line[i+1], line[i+2], line[i+3]}) { - start := i - iplen := strings.LastIndexAny(line[start:], IPV4_CHARSET) - ip, err := netaddr.ParseIP(line[start : start+iplen+1]) - if err == nil { - if contains(ipf.matcher, ip) { - return true - } + ok, iplen := filterFn(line, i, IPV4_CHARSET) + if ok { + return true } i += iplen continue } if i+4 < n && ipv6Hint([5]byte{line[i], line[i+1], line[i+2], line[i+3], line[i+4]}) { - start := i - iplen := strings.LastIndex(line[start:], IPV6_CHARSET) - ip, err := netaddr.ParseIP(line[start : start+iplen+1]) - if err == nil { - if contains(ipf.matcher, ip) { - return true - } + ok, iplen := filterFn(line, i, IPV6_CHARSET) + if ok { + return true } i += iplen continue @@ -188,3 +195,22 @@ func ipv6Hint(prefix [5]byte) bool { func isHexDigit(r byte) bool { return unicode.IsDigit(rune(r)) || (r >= 'a' && r <= 'f') || (r >= 'A' && r <= 'F') } + +// stringSpan is same as C's `strcspan()` function. +// It returns the number of chars in the initial segment of `s` +// which consist only of chars from `accept`. +func stringSpan(s, accept string) int { + m := make(map[rune]bool) + + for _, r := range accept { + m[r] = true + } + + for i, r := range s { + if !m[r] { + return i + } + } + + return len(s) +} diff --git a/pkg/logql/log/ip_test.go b/pkg/logql/log/ip_test.go index 626b1df58c995..c15ba13b2b029 100644 --- a/pkg/logql/log/ip_test.go +++ b/pkg/logql/log/ip_test.go @@ -20,7 +20,7 @@ func Test_IPFilter(t *testing.T) { fail bool }{ { - name: "single IP", + name: "single IPv4", pat: "192.168.0.1", input: []string{ "vpn 192.168.0.5 connected to vm", @@ -30,14 +30,28 @@ func Test_IPFilter(t *testing.T) { "", }, expected: []int{1}, // should match with only line at index `1` from the input + }, { - name: "IP range", + name: "single IPv6", + pat: "::1", // localhost address + input: []string{ + "vpn ::1 connected to vm", // still a match + "vpn 192.168.0.1 connected to vm", + "x", + "hello world!", + "", + }, + expected: []int{0}, // should match with only line at index `0` from the input + + }, + { + name: "IPv4 range", pat: "192.168.0.1-192.189.10.12", input: []string{ "vpn 192.168.0.0 connected to vm", - "vpn 192.168.0.1 connected to vm", - "vpn 192.172.6.1 connected to vm", + "vpn 192.168.0.0 192.168.0.1 connected to vm", // match + "vpn 192.172.6.1 connected to vm", // match "vpn 192.255.255.255 connected to vm", "x", "hello world!", @@ -45,6 +59,20 @@ func Test_IPFilter(t *testing.T) { }, expected: []int{1, 2}, }, + { + name: "IPv6 range", + pat: "2001:db8::1-2001:db8::8", + input: []string{ + "vpn 192.168.0.0 connected to vm", // not match + "vpn 2001:db8::2 connected to vm", // match + "vpn 2001:db8::9 connected to vm", // not match + "vpn 2001:db8::5 connected to vm", // match + "x", + "hello world!", + "", + }, + expected: []int{1, 3}, + }, { name: "wrong IP range syntax extra space", // NOTE(kavi): Should we handle this as normal valid range pattern? pat: "192.168.0.1 - 192.189.10.12", @@ -65,16 +93,30 @@ func Test_IPFilter(t *testing.T) { }, expected: []int{0, 1, 3}, }, + { + name: "CIDR IPv6", + pat: "2001:db8::/32", + input: []string{ + "vpn 2001:db8::1 connected to vm", // match + "vpn 2001:db9::3 connected to vm", // not a match + "vpn 2001:dc8::1 and 2001:db8::2 connected to vm", // firt not match, but second did match. So overall its a match + "vpn 192.168.255.255 connected to vm", // not match + "x", + "hello world!", + "", + }, + expected: []int{0, 2}, + }, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { - ip, err := NewIPFilter(c.pat) + ip := NewIPFilter(c.pat) if c.fail { - assert.Error(t, c.err, err) + assert.Error(t, c.err, ip.patternError) return } - require.NoError(t, err) + assert.NoError(t, ip.patternError) got := make([]int, 0) for i, in := range c.input { @@ -103,8 +145,8 @@ func Benchmark_IPFilter(b *testing.B) { "192.168.4.5/16", } { b.Run(pattern, func(b *testing.B) { - stage, err := NewIPFilter(pattern) - require.Nil(b, err) + stage := NewIPFilter(pattern) + require.Nil(b, stage.patternError) b.ResetTimer() for n := 0; n < b.N; n++ { From 7ae87757bd472c57e0dfb115848a909bb1ed1ffb Mon Sep 17 00:00:00 2001 From: Kaviraj Date: Tue, 13 Jul 2021 00:43:11 +0200 Subject: [PATCH 06/24] Remove testlogs.txt Signed-off-by: Kaviraj --- testlogs.txt | 8 -------- 1 file changed, 8 deletions(-) delete mode 100644 testlogs.txt diff --git a/testlogs.txt b/testlogs.txt deleted file mode 100644 index 2846e8460e69b..0000000000000 --- a/testlogs.txt +++ /dev/null @@ -1,8 +0,0 @@ -level=info ts=2021-07-10T20:29:12.464740436Z caller=main.go:129 msg="Starting Loki" version="(version=, branch=, revision=)" -level=info ts=2021-07-10T20:29:12.477325579Z caller=server.go:239 http=[::]:3100 grpc=[::]:9096 msg="server listening on addresses" -level=info ts=2021-07-10T20:29:12.478423515Z caller=shipper_index_client.go:108 msg="starting boltdb shipper in 0 mode" -level=info ts=2021-07-10T20:29:12.478421065Z caller=table_manager.go:171 msg="uploading tables" -level=info ts=2021-07-10T20:29:12.481869432Z caller=mapper.go:46 msg="cleaning up mapped rules directory" path=/tmp/loki/rules-temp -level=error ts=2021-07-10T20:29:12.481942251Z caller=mapper.go:50 msg="unable to read rules directory" path=/tmp/loki/rules-temp err="open /tmp/loki/rules-temp: no such file or directory" -level=info ts=2021-07-10T20:29:12.485403993Z caller=module_service.go:59 msg=initialising module=memberlist-kv -level=info ts=2021-07-10T20:29:12.485512367Z caller=module_service.go:59 msg=initialising module=server From d4a6939531c9f134d9bc72b0a3630ade993a0211 Mon Sep 17 00:00:00 2001 From: Kaviraj Date: Tue, 13 Jul 2021 00:52:26 +0200 Subject: [PATCH 07/24] Make linter happy Signed-off-by: Kaviraj --- pkg/logql/log/ip.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/logql/log/ip.go b/pkg/logql/log/ip.go index b5874c7b11278..d90b01a55da2c 100644 --- a/pkg/logql/log/ip.go +++ b/pkg/logql/log/ip.go @@ -15,8 +15,8 @@ var ( type IPMatchType int const ( - IPV4_CHARSET = "0123456789." - IPV6_CHARSET = "0123456789abcdefABCDEF:." + IPv4Charset = "0123456789." + IPv6Charset = "0123456789abcdefABCDEF:." IPMatchLine IPMatchType = iota IPMatchLabel @@ -87,7 +87,7 @@ func (ipf *IPFilter) Filter(line string) bool { // It uses IPv4 and IPv6 prefix hints to find the IP addresses faster without using regexp. for i := 0; i < n; i++ { if i+3 < n && ipv4Hint([4]byte{line[i], line[i+1], line[i+2], line[i+3]}) { - ok, iplen := filterFn(line, i, IPV4_CHARSET) + ok, iplen := filterFn(line, i, IPv4Charset) if ok { return true } @@ -96,7 +96,7 @@ func (ipf *IPFilter) Filter(line string) bool { } if i+4 < n && ipv6Hint([5]byte{line[i], line[i+1], line[i+2], line[i+3], line[i+4]}) { - ok, iplen := filterFn(line, i, IPV6_CHARSET) + ok, iplen := filterFn(line, i, IPv6Charset) if ok { return true } From e30771455bce91a27b3eb3d77c0cb823f73cb9dc Mon Sep 17 00:00:00 2001 From: Kaviraj Date: Tue, 13 Jul 2021 11:52:56 +0200 Subject: [PATCH 08/24] Add more cases to the benchmark Signed-off-by: Kaviraj --- pkg/logql/log/ip_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pkg/logql/log/ip_test.go b/pkg/logql/log/ip_test.go index c15ba13b2b029..db88ebd8a8fb8 100644 --- a/pkg/logql/log/ip_test.go +++ b/pkg/logql/log/ip_test.go @@ -134,7 +134,8 @@ func Benchmark_IPFilter(b *testing.B) { line := [][]byte{ []byte(`vpn 192.168.0.0 connected to vm`), - /// todo add more cases,long line without match, with match etc.... + []byte(`vpn connected to vm just wanted to make some long line without match`), + []byte(`vpn ::1 connected to vm just wanted to make some long line with match at the end 127.0.0.1`), } lbbb := NewBaseLabelsBuilder() lbb := lbbb.ForLabels(labels.Labels{labels.Label{Name: "foo", Value: "bar"}}, 0) From e5e46d0146bd1805c3153917c00c0271932c4128 Mon Sep 17 00:00:00 2001 From: Kaviraj Date: Tue, 13 Jul 2021 17:16:17 +0200 Subject: [PATCH 09/24] PR remarks 1. support NEQ 2. reduce allocation. 3. Minor tweaks Signed-off-by: Kaviraj --- pkg/logql/expr.y | 4 +- pkg/logql/expr.y.go | 384 ++++++++++++++++++++------------------- pkg/logql/log/ip.go | 47 +++-- pkg/logql/log/ip_test.go | 93 +++++++++- pkg/logql/parser_test.go | 59 ++++-- 5 files changed, 366 insertions(+), 221 deletions(-) diff --git a/pkg/logql/expr.y b/pkg/logql/expr.y index 8a49017bb6a59..e48232b982597 100644 --- a/pkg/logql/expr.y +++ b/pkg/logql/expr.y @@ -290,7 +290,9 @@ jsonExpressionList: ; ipLabelFilter: - IDENTIFIER EQ IP OPEN_PARENTHESIS STRING CLOSE_PARENTHESIS { $$ = log.NewIPLabelFilter($5, $1) } + IDENTIFIER EQ IP OPEN_PARENTHESIS STRING CLOSE_PARENTHESIS { $$ = log.NewIPLabelFilter($1, log.LabelFilterEqual, $5) } + | IDENTIFIER NEQ IP OPEN_PARENTHESIS STRING CLOSE_PARENTHESIS { $$ = log.NewIPLabelFilter($1, log.LabelFilterNotEqual, $5) } + ; unitFilter: durationFilter { $$ = $1 } diff --git a/pkg/logql/expr.y.go b/pkg/logql/expr.y.go index cf0c5e69c91bc..7ece14922eef7 100644 --- a/pkg/logql/expr.y.go +++ b/pkg/logql/expr.y.go @@ -232,101 +232,102 @@ var exprExca = [...]int{ const exprPrivate = 57344 -const exprLast = 493 +const exprLast = 500 var exprAct = [...]int{ - 233, 186, 75, 4, 168, 58, 156, 5, 161, 195, - 66, 109, 50, 57, 236, 130, 68, 2, 45, 46, + 234, 186, 75, 4, 168, 58, 156, 5, 161, 195, + 66, 109, 50, 57, 237, 130, 68, 2, 45, 46, 47, 48, 49, 50, 71, 42, 43, 44, 51, 52, 55, 56, 53, 54, 45, 46, 47, 48, 49, 50, 43, 44, 51, 52, 55, 56, 53, 54, 45, 46, 47, 48, 49, 50, 47, 48, 49, 50, 126, 128, - 129, 64, 241, 97, 170, 128, 129, 101, 62, 63, - 211, 238, 178, 212, 210, 290, 287, 82, 134, 309, - 290, 132, 61, 304, 139, 51, 52, 55, 56, 53, - 54, 45, 46, 47, 48, 49, 50, 116, 140, 236, + 129, 64, 242, 97, 170, 128, 129, 101, 62, 63, + 215, 239, 179, 216, 214, 293, 290, 82, 134, 313, + 293, 132, 61, 271, 139, 51, 52, 55, 56, 53, + 54, 45, 46, 47, 48, 49, 50, 116, 140, 237, 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, - 151, 152, 153, 154, 116, 113, 127, 65, 270, 297, + 151, 152, 153, 154, 116, 113, 127, 65, 239, 308, 165, 176, 171, 174, 175, 172, 173, 74, 158, 76, - 77, 209, 113, 104, 106, 105, 177, 114, 115, 241, - 193, 189, 98, 295, 269, 239, 187, 269, 198, 190, - 64, 76, 77, 237, 107, 279, 108, 62, 63, 293, - 277, 182, 185, 237, 201, 202, 203, 64, 272, 273, - 274, 248, 191, 12, 62, 63, 157, 242, 276, 238, - 188, 133, 238, 261, 121, 182, 231, 234, 238, 240, - 116, 243, 132, 97, 246, 101, 247, 188, 238, 235, - 232, 120, 64, 244, 158, 197, 65, 245, 113, 62, - 63, 250, 256, 254, 239, 250, 252, 185, 116, 64, - 251, 260, 64, 65, 199, 197, 62, 63, 307, 62, - 63, 303, 188, 182, 138, 262, 113, 264, 266, 137, - 268, 97, 136, 116, 196, 267, 278, 263, 80, 188, - 97, 236, 188, 280, 282, 183, 73, 158, 65, 249, - 207, 113, 206, 131, 284, 285, 64, 123, 204, 97, - 286, 12, 200, 62, 63, 65, 288, 289, 65, 133, - 192, 122, 294, 184, 124, 125, 15, 208, 205, 302, - 299, 64, 300, 301, 12, 292, 188, 229, 62, 63, - 230, 228, 6, 305, 159, 157, 19, 20, 33, 34, - 36, 37, 35, 38, 39, 40, 41, 21, 22, 291, - 226, 60, 65, 227, 225, 79, 275, 23, 24, 25, - 26, 27, 28, 29, 265, 116, 194, 30, 31, 32, - 18, 78, 223, 308, 12, 224, 222, 65, 306, 158, - 258, 259, 6, 113, 16, 17, 19, 20, 33, 34, - 36, 37, 35, 38, 39, 40, 41, 21, 22, 214, - 296, 179, 215, 213, 283, 255, 253, 23, 24, 25, - 26, 27, 28, 29, 3, 116, 135, 30, 31, 32, - 18, 67, 181, 180, 12, 220, 159, 157, 221, 219, - 179, 178, 6, 113, 16, 17, 19, 20, 33, 34, - 36, 37, 35, 38, 39, 40, 41, 21, 22, 81, - 166, 104, 106, 105, 164, 114, 115, 23, 24, 25, - 26, 27, 28, 29, 163, 155, 119, 30, 31, 32, - 18, 298, 107, 217, 108, 257, 218, 216, 169, 70, - 281, 162, 72, 72, 16, 17, 169, 110, 111, 160, - 100, 167, 103, 83, 84, 85, 86, 87, 88, 89, - 90, 91, 92, 93, 94, 95, 96, 102, 59, 117, - 112, 118, 99, 11, 10, 9, 14, 8, 271, 13, - 7, 69, 1, + 77, 213, 113, 104, 106, 105, 177, 114, 115, 242, + 193, 189, 98, 76, 77, 240, 187, 238, 198, 190, + 64, 301, 300, 271, 107, 116, 108, 62, 63, 272, + 279, 182, 185, 298, 201, 202, 203, 64, 296, 158, + 238, 64, 281, 113, 62, 63, 157, 243, 62, 63, + 188, 262, 239, 263, 249, 278, 232, 235, 239, 241, + 116, 244, 132, 97, 247, 101, 248, 188, 191, 236, + 233, 188, 240, 245, 158, 239, 65, 64, 113, 274, + 275, 276, 257, 255, 62, 63, 159, 157, 182, 251, + 237, 185, 182, 65, 253, 251, 64, 65, 121, 120, + 252, 261, 138, 62, 63, 125, 264, 188, 266, 268, + 246, 270, 97, 197, 183, 64, 269, 280, 265, 197, + 64, 97, 62, 63, 282, 116, 188, 62, 63, 311, + 137, 131, 199, 65, 136, 80, 287, 288, 196, 12, + 73, 97, 289, 113, 123, 188, 307, 133, 291, 292, + 60, 284, 65, 211, 297, 178, 212, 210, 122, 15, + 12, 124, 250, 303, 267, 304, 305, 12, 133, 207, + 204, 65, 200, 192, 184, 6, 65, 309, 208, 19, + 20, 33, 34, 36, 37, 35, 38, 39, 40, 41, + 21, 22, 205, 230, 306, 227, 231, 229, 228, 226, + 23, 24, 25, 26, 27, 28, 29, 295, 294, 194, + 30, 31, 32, 18, 209, 224, 277, 12, 225, 223, + 221, 79, 78, 222, 220, 6, 312, 16, 17, 19, + 20, 33, 34, 36, 37, 35, 38, 39, 40, 41, + 21, 22, 218, 310, 302, 219, 217, 259, 260, 299, + 23, 24, 25, 26, 27, 28, 29, 3, 116, 135, + 30, 31, 32, 18, 67, 286, 285, 12, 258, 256, + 254, 169, 110, 181, 180, 6, 113, 16, 17, 19, + 20, 33, 34, 36, 37, 35, 38, 39, 40, 41, + 21, 22, 81, 179, 104, 106, 105, 178, 114, 115, + 23, 24, 25, 26, 27, 28, 29, 116, 111, 166, + 30, 31, 32, 18, 164, 107, 163, 108, 155, 119, + 70, 158, 283, 72, 162, 113, 206, 16, 17, 72, + 169, 160, 100, 167, 103, 102, 83, 84, 85, 86, + 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, + 59, 117, 112, 118, 99, 11, 10, 9, 14, 8, + 273, 13, 7, 69, 1, 0, 0, 0, 159, 157, } var exprPact = [...]int{ - 279, -1000, -41, -1000, -1000, 277, 279, -1000, -1000, -1000, - -1000, -1000, 447, 233, 104, -1000, 334, 318, 225, -1000, + 282, -1000, -41, -1000, -1000, 236, 282, -1000, -1000, -1000, + -1000, -1000, 448, 247, 104, -1000, 345, 344, 242, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, 37, 37, 37, 37, 37, 37, 37, 37, - 37, 37, 37, 37, 37, 37, 37, 277, -1000, 47, - 380, 430, -1000, -1000, -1000, -1000, 177, 160, -41, 265, - 269, -1000, 46, 256, 379, 219, 216, 211, -1000, -1000, - 279, 279, -1000, 279, 279, 279, 279, 279, 279, 279, - 279, 279, 279, 279, 279, 279, 279, -1000, 429, -1000, - -1000, 330, -1000, -1000, 446, -1000, 428, -1000, 418, -1000, - -1000, -1000, -1000, 213, 414, 451, 52, -1000, -1000, -1000, - -1000, -1000, -1000, -1000, 448, -1000, 395, 394, 387, 386, - 231, 264, 208, 158, 148, 261, 329, 220, 200, 253, + 37, 37, 37, 37, 37, 37, 37, 236, -1000, 47, + 383, 443, -1000, -1000, -1000, -1000, 205, 204, -41, 272, + 219, -1000, 46, 254, 382, 241, 237, 209, -1000, -1000, + 282, 282, -1000, 282, 282, 282, 282, 282, 282, 282, + 282, 282, 282, 282, 282, 282, 282, -1000, 442, -1000, + -1000, 150, -1000, -1000, 449, -1000, 440, -1000, 438, -1000, + -1000, -1000, -1000, 250, 433, 455, 52, -1000, -1000, -1000, + -1000, -1000, -1000, -1000, 454, -1000, 421, 417, 398, 397, + 220, 285, 212, 275, 174, 284, 332, 244, 238, 283, -27, 16, 16, -23, -23, -68, -68, -68, -68, -57, - -57, -57, -57, -57, -57, -1000, 330, 213, 213, 213, - 249, -1000, 276, -1000, -1000, 238, -1000, 241, -1000, 275, - 66, 365, 439, 391, 338, 316, 293, -1000, -1000, -1000, - -1000, -1000, -1000, 126, 158, 188, 144, 205, 92, 153, - 183, 126, 279, 147, 240, 196, -1000, -1000, 192, -1000, - 370, 185, 330, 109, 446, 369, -1000, 443, 345, 198, + -57, -57, -57, -57, -57, -1000, 150, 250, 250, 250, + 281, -1000, 310, -1000, -1000, 432, -1000, 280, -1000, 296, + 279, 66, 368, 346, 341, 321, 319, -1000, -1000, -1000, + -1000, -1000, -1000, 118, 275, 157, 138, 193, 92, 153, + 216, 118, 282, 160, 273, 206, -1000, -1000, 200, -1000, + 394, 185, 150, 109, 449, 393, -1000, 396, 372, 208, + -1000, -1000, -1000, 158, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, - -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, - -1000, -1000, 159, 252, 27, 252, 326, -49, 213, -49, - 138, 113, 317, 154, 136, -1000, -1000, 131, -1000, 279, - 445, -1000, -1000, 235, -1000, -1000, -1000, -1000, -1000, -1000, - 368, 126, 27, 252, 27, -1000, -1000, 330, -1000, -49, - -1000, 53, -1000, -1000, -1000, 36, 310, 286, 135, 126, - 119, -1000, 364, 95, -1000, 27, -1000, 436, 31, 27, - 15, -49, -49, 280, -1000, -1000, 212, -1000, 59, 27, - -1000, -1000, -49, 342, -1000, -1000, 209, 337, 55, -1000, + -1000, -1000, -1000, 159, 231, 27, 231, 286, -49, 250, + -49, 74, 154, 337, 161, 136, -1000, -1000, 148, -1000, + 282, 447, -1000, -1000, 262, -1000, -1000, -1000, -1000, -1000, + -1000, 390, 389, 118, 27, 231, 27, -1000, -1000, 150, + -1000, -49, -1000, 53, -1000, -1000, -1000, 36, 329, 328, + 144, 118, 139, -1000, 373, 128, 127, -1000, 27, -1000, + 369, 31, 27, 15, -49, -49, 315, -1000, -1000, 257, + -1000, -1000, 95, 27, -1000, -1000, -49, 367, -1000, -1000, + 240, 350, 55, -1000, } var exprPgo = [...]int{ - 0, 492, 16, 82, 2, 9, 384, 3, 15, 11, - 491, 490, 489, 488, 7, 487, 486, 485, 484, 483, - 419, 482, 13, 5, 481, 480, 479, 6, 478, 477, - 462, 4, 461, 460, 8, 459, 1, 458, 457, 0, + 0, 494, 16, 82, 2, 9, 387, 3, 15, 11, + 493, 492, 491, 490, 7, 489, 488, 487, 486, 485, + 422, 484, 13, 5, 483, 482, 481, 6, 480, 465, + 464, 4, 463, 462, 8, 461, 1, 438, 402, 0, } var exprR1 = [...]int{ @@ -340,14 +341,14 @@ var exprR1 = [...]int{ 22, 23, 23, 23, 23, 23, 23, 28, 28, 21, 21, 21, 21, 21, 33, 29, 31, 31, 32, 32, 32, 30, 27, 27, 27, 27, 27, 27, 27, 27, - 27, 34, 35, 35, 38, 37, 37, 26, 26, 26, - 26, 26, 26, 26, 24, 24, 24, 24, 24, 24, - 24, 25, 25, 25, 25, 25, 25, 25, 17, 17, + 27, 34, 35, 35, 38, 38, 37, 37, 26, 26, + 26, 26, 26, 26, 26, 24, 24, 24, 24, 24, + 24, 24, 25, 25, 25, 25, 25, 25, 25, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 20, 20, 18, 18, 18, 16, 16, - 16, 16, 16, 16, 16, 16, 16, 12, 12, 12, + 17, 17, 17, 17, 20, 20, 18, 18, 18, 16, + 16, 16, 16, 16, 16, 16, 16, 16, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, - 12, 39, 5, 5, 4, 4, 4, 4, + 12, 12, 39, 5, 5, 4, 4, 4, 4, } var exprR2 = [...]int{ @@ -361,14 +362,14 @@ var exprR2 = [...]int{ 2, 1, 2, 2, 2, 2, 2, 2, 3, 1, 1, 2, 1, 2, 2, 2, 3, 3, 1, 3, 3, 2, 1, 1, 1, 1, 3, 2, 3, 3, - 3, 3, 1, 3, 6, 1, 1, 3, 3, 3, + 3, 3, 1, 3, 6, 6, 1, 1, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, - 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, + 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 0, 1, 1, 2, 2, 1, 1, + 4, 4, 4, 4, 0, 1, 1, 2, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 2, 1, 3, 4, 4, 3, 3, + 1, 1, 2, 1, 3, 4, 4, 3, 3, } var exprChk = [...]int{ @@ -393,50 +394,52 @@ var exprChk = [...]int{ 6, 6, 2, 24, 19, 9, -36, -22, 44, -14, -8, 24, 19, -7, 7, -5, 24, 5, -5, 24, 19, -27, -27, -27, 19, 12, 24, 19, 12, 65, + 8, 4, 7, 65, 8, 4, 7, 8, 4, 7, 8, 4, 7, 8, 4, 7, 8, 4, 7, 8, - 4, 7, 8, 4, 7, 8, 4, 7, 8, 4, - 7, -4, -8, -39, -36, -22, 63, 9, 44, 9, - -36, 47, 24, -36, -22, 24, -4, -7, 24, 19, - 19, 24, 24, 6, -34, 6, -31, 2, 5, 6, - 23, 24, -36, -22, -36, 8, -39, -27, -39, 9, - 5, -13, 55, 56, 57, 9, 24, 24, -36, 24, - -7, 5, 19, 6, -4, -36, -39, 23, -39, -36, - 44, 9, 9, 24, -4, 24, 6, 24, 5, -36, - -39, -39, 9, 19, 24, -39, 6, 19, 6, 24, + 4, 7, -4, -8, -39, -36, -22, 63, 9, 44, + 9, -36, 47, 24, -36, -22, 24, -4, -7, 24, + 19, 19, 24, 24, 6, -34, 6, -31, 2, 5, + 6, 23, 23, 24, -36, -22, -36, 8, -39, -27, + -39, 9, 5, -13, 55, 56, 57, 9, 24, 24, + -36, 24, -7, 5, 19, 6, 6, -4, -36, -39, + 23, -39, -36, 44, 9, 9, 24, -4, 24, 6, + 24, 24, 5, -36, -39, -39, 9, 19, 24, -39, + 6, 19, 6, 24, } var exprDef = [...]int{ 0, -2, 1, 2, 3, 10, 0, 4, 5, 6, - 7, 8, 0, 0, 0, 145, 0, 0, 0, 157, - 158, 159, 160, 161, 162, 163, 164, 165, 166, 167, - 168, 169, 170, 148, 149, 150, 151, 152, 153, 154, - 155, 156, 143, 143, 143, 143, 143, 143, 143, 143, - 143, 143, 143, 143, 143, 143, 143, 11, 69, 71, + 7, 8, 0, 0, 0, 146, 0, 0, 0, 158, + 159, 160, 161, 162, 163, 164, 165, 166, 167, 168, + 169, 170, 171, 149, 150, 151, 152, 153, 154, 155, + 156, 157, 144, 144, 144, 144, 144, 144, 144, 144, + 144, 144, 144, 144, 144, 144, 144, 11, 69, 71, 0, 0, 56, 57, 58, 59, 3, 2, 0, 0, - 0, 63, 0, 0, 0, 0, 0, 0, 146, 147, - 0, 0, 144, 0, 0, 0, 0, 0, 0, 0, + 0, 63, 0, 0, 0, 0, 0, 0, 147, 148, + 0, 0, 145, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 70, 0, 72, 73, 74, 75, 76, 79, 80, 0, 82, 0, 92, - 93, 94, 95, 0, 0, 0, 0, 105, 106, 77, + 93, 94, 95, 0, 0, 0, 0, 106, 107, 77, 9, 12, 60, 61, 0, 62, 0, 0, 0, 0, - 0, 0, 0, 0, 3, 145, 0, 0, 0, 3, - 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, - 138, 139, 140, 141, 142, 78, 97, 0, 0, 0, + 0, 0, 0, 0, 3, 146, 0, 0, 0, 3, + 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, + 139, 140, 141, 142, 143, 78, 97, 0, 0, 0, 84, 102, 0, 81, 83, 0, 85, 91, 88, 0, 0, 0, 0, 0, 0, 0, 0, 64, 65, 66, 67, 68, 38, 45, 0, 13, 0, 0, 0, 0, - 0, 49, 0, 3, 145, 0, 176, 172, 0, 177, + 0, 49, 0, 3, 146, 0, 177, 173, 0, 178, 0, 98, 99, 100, 0, 0, 96, 0, 0, 0, - 112, 119, 126, 111, 118, 125, 107, 114, 121, 108, - 115, 122, 109, 116, 123, 110, 117, 124, 113, 120, - 127, 47, 0, 14, 17, 33, 0, 21, 0, 25, - 0, 0, 0, 0, 0, 37, 51, 3, 50, 0, - 0, 174, 175, 0, 103, 101, 89, 90, 86, 87, - 0, 46, 18, 34, 35, 171, 22, 41, 26, 29, - 39, 0, 42, 43, 44, 15, 0, 0, 0, 52, - 3, 173, 0, 0, 48, 36, 30, 0, 16, 19, - 0, 23, 27, 0, 53, 54, 0, 104, 0, 20, - 24, 28, 31, 0, 40, 32, 0, 0, 0, 55, + 113, 120, 127, 0, 112, 119, 126, 108, 115, 122, + 109, 116, 123, 110, 117, 124, 111, 118, 125, 114, + 121, 128, 47, 0, 14, 17, 33, 0, 21, 0, + 25, 0, 0, 0, 0, 0, 37, 51, 3, 50, + 0, 0, 175, 176, 0, 103, 101, 89, 90, 86, + 87, 0, 0, 46, 18, 34, 35, 172, 22, 41, + 26, 29, 39, 0, 42, 43, 44, 15, 0, 0, + 0, 52, 3, 174, 0, 0, 0, 48, 36, 30, + 0, 16, 19, 0, 23, 27, 0, 53, 54, 0, + 104, 105, 0, 20, 24, 28, 31, 0, 40, 32, + 0, 0, 0, 55, } var exprTok1 = [...]int{ @@ -1300,47 +1303,47 @@ exprdefault: case 104: exprDollar = exprS[exprpt-6 : exprpt+1] { - exprVAL.IPLabelFilter = log.NewIPLabelFilter(exprDollar[5].str, exprDollar[1].str) + exprVAL.IPLabelFilter = log.NewIPLabelFilter(exprDollar[1].str, log.LabelFilterEqual, exprDollar[5].str) } case 105: - exprDollar = exprS[exprpt-1 : exprpt+1] + exprDollar = exprS[exprpt-6 : exprpt+1] { - exprVAL.UnitFilter = exprDollar[1].DurationFilter + exprVAL.IPLabelFilter = log.NewIPLabelFilter(exprDollar[1].str, log.LabelFilterNotEqual, exprDollar[5].str) } case 106: exprDollar = exprS[exprpt-1 : exprpt+1] { - exprVAL.UnitFilter = exprDollar[1].BytesFilter + exprVAL.UnitFilter = exprDollar[1].DurationFilter } case 107: - exprDollar = exprS[exprpt-3 : exprpt+1] + exprDollar = exprS[exprpt-1 : exprpt+1] { - exprVAL.DurationFilter = log.NewDurationLabelFilter(log.LabelFilterGreaterThan, exprDollar[1].str, exprDollar[3].duration) + exprVAL.UnitFilter = exprDollar[1].BytesFilter } case 108: exprDollar = exprS[exprpt-3 : exprpt+1] { - exprVAL.DurationFilter = log.NewDurationLabelFilter(log.LabelFilterGreaterThanOrEqual, exprDollar[1].str, exprDollar[3].duration) + exprVAL.DurationFilter = log.NewDurationLabelFilter(log.LabelFilterGreaterThan, exprDollar[1].str, exprDollar[3].duration) } case 109: exprDollar = exprS[exprpt-3 : exprpt+1] { - exprVAL.DurationFilter = log.NewDurationLabelFilter(log.LabelFilterLesserThan, exprDollar[1].str, exprDollar[3].duration) + exprVAL.DurationFilter = log.NewDurationLabelFilter(log.LabelFilterGreaterThanOrEqual, exprDollar[1].str, exprDollar[3].duration) } case 110: exprDollar = exprS[exprpt-3 : exprpt+1] { - exprVAL.DurationFilter = log.NewDurationLabelFilter(log.LabelFilterLesserThanOrEqual, exprDollar[1].str, exprDollar[3].duration) + exprVAL.DurationFilter = log.NewDurationLabelFilter(log.LabelFilterLesserThan, exprDollar[1].str, exprDollar[3].duration) } case 111: exprDollar = exprS[exprpt-3 : exprpt+1] { - exprVAL.DurationFilter = log.NewDurationLabelFilter(log.LabelFilterNotEqual, exprDollar[1].str, exprDollar[3].duration) + exprVAL.DurationFilter = log.NewDurationLabelFilter(log.LabelFilterLesserThanOrEqual, exprDollar[1].str, exprDollar[3].duration) } case 112: exprDollar = exprS[exprpt-3 : exprpt+1] { - exprVAL.DurationFilter = log.NewDurationLabelFilter(log.LabelFilterEqual, exprDollar[1].str, exprDollar[3].duration) + exprVAL.DurationFilter = log.NewDurationLabelFilter(log.LabelFilterNotEqual, exprDollar[1].str, exprDollar[3].duration) } case 113: exprDollar = exprS[exprpt-3 : exprpt+1] @@ -1350,32 +1353,32 @@ exprdefault: case 114: exprDollar = exprS[exprpt-3 : exprpt+1] { - exprVAL.BytesFilter = log.NewBytesLabelFilter(log.LabelFilterGreaterThan, exprDollar[1].str, exprDollar[3].bytes) + exprVAL.DurationFilter = log.NewDurationLabelFilter(log.LabelFilterEqual, exprDollar[1].str, exprDollar[3].duration) } case 115: exprDollar = exprS[exprpt-3 : exprpt+1] { - exprVAL.BytesFilter = log.NewBytesLabelFilter(log.LabelFilterGreaterThanOrEqual, exprDollar[1].str, exprDollar[3].bytes) + exprVAL.BytesFilter = log.NewBytesLabelFilter(log.LabelFilterGreaterThan, exprDollar[1].str, exprDollar[3].bytes) } case 116: exprDollar = exprS[exprpt-3 : exprpt+1] { - exprVAL.BytesFilter = log.NewBytesLabelFilter(log.LabelFilterLesserThan, exprDollar[1].str, exprDollar[3].bytes) + exprVAL.BytesFilter = log.NewBytesLabelFilter(log.LabelFilterGreaterThanOrEqual, exprDollar[1].str, exprDollar[3].bytes) } case 117: exprDollar = exprS[exprpt-3 : exprpt+1] { - exprVAL.BytesFilter = log.NewBytesLabelFilter(log.LabelFilterLesserThanOrEqual, exprDollar[1].str, exprDollar[3].bytes) + exprVAL.BytesFilter = log.NewBytesLabelFilter(log.LabelFilterLesserThan, exprDollar[1].str, exprDollar[3].bytes) } case 118: exprDollar = exprS[exprpt-3 : exprpt+1] { - exprVAL.BytesFilter = log.NewBytesLabelFilter(log.LabelFilterNotEqual, exprDollar[1].str, exprDollar[3].bytes) + exprVAL.BytesFilter = log.NewBytesLabelFilter(log.LabelFilterLesserThanOrEqual, exprDollar[1].str, exprDollar[3].bytes) } case 119: exprDollar = exprS[exprpt-3 : exprpt+1] { - exprVAL.BytesFilter = log.NewBytesLabelFilter(log.LabelFilterEqual, exprDollar[1].str, exprDollar[3].bytes) + exprVAL.BytesFilter = log.NewBytesLabelFilter(log.LabelFilterNotEqual, exprDollar[1].str, exprDollar[3].bytes) } case 120: exprDollar = exprS[exprpt-3 : exprpt+1] @@ -1385,32 +1388,32 @@ exprdefault: case 121: exprDollar = exprS[exprpt-3 : exprpt+1] { - exprVAL.NumberFilter = log.NewNumericLabelFilter(log.LabelFilterGreaterThan, exprDollar[1].str, mustNewFloat(exprDollar[3].str)) + exprVAL.BytesFilter = log.NewBytesLabelFilter(log.LabelFilterEqual, exprDollar[1].str, exprDollar[3].bytes) } case 122: exprDollar = exprS[exprpt-3 : exprpt+1] { - exprVAL.NumberFilter = log.NewNumericLabelFilter(log.LabelFilterGreaterThanOrEqual, exprDollar[1].str, mustNewFloat(exprDollar[3].str)) + exprVAL.NumberFilter = log.NewNumericLabelFilter(log.LabelFilterGreaterThan, exprDollar[1].str, mustNewFloat(exprDollar[3].str)) } case 123: exprDollar = exprS[exprpt-3 : exprpt+1] { - exprVAL.NumberFilter = log.NewNumericLabelFilter(log.LabelFilterLesserThan, exprDollar[1].str, mustNewFloat(exprDollar[3].str)) + exprVAL.NumberFilter = log.NewNumericLabelFilter(log.LabelFilterGreaterThanOrEqual, exprDollar[1].str, mustNewFloat(exprDollar[3].str)) } case 124: exprDollar = exprS[exprpt-3 : exprpt+1] { - exprVAL.NumberFilter = log.NewNumericLabelFilter(log.LabelFilterLesserThanOrEqual, exprDollar[1].str, mustNewFloat(exprDollar[3].str)) + exprVAL.NumberFilter = log.NewNumericLabelFilter(log.LabelFilterLesserThan, exprDollar[1].str, mustNewFloat(exprDollar[3].str)) } case 125: exprDollar = exprS[exprpt-3 : exprpt+1] { - exprVAL.NumberFilter = log.NewNumericLabelFilter(log.LabelFilterNotEqual, exprDollar[1].str, mustNewFloat(exprDollar[3].str)) + exprVAL.NumberFilter = log.NewNumericLabelFilter(log.LabelFilterLesserThanOrEqual, exprDollar[1].str, mustNewFloat(exprDollar[3].str)) } case 126: exprDollar = exprS[exprpt-3 : exprpt+1] { - exprVAL.NumberFilter = log.NewNumericLabelFilter(log.LabelFilterEqual, exprDollar[1].str, mustNewFloat(exprDollar[3].str)) + exprVAL.NumberFilter = log.NewNumericLabelFilter(log.LabelFilterNotEqual, exprDollar[1].str, mustNewFloat(exprDollar[3].str)) } case 127: exprDollar = exprS[exprpt-3 : exprpt+1] @@ -1418,251 +1421,256 @@ exprdefault: exprVAL.NumberFilter = log.NewNumericLabelFilter(log.LabelFilterEqual, exprDollar[1].str, mustNewFloat(exprDollar[3].str)) } case 128: + exprDollar = exprS[exprpt-3 : exprpt+1] + { + exprVAL.NumberFilter = log.NewNumericLabelFilter(log.LabelFilterEqual, exprDollar[1].str, mustNewFloat(exprDollar[3].str)) + } + case 129: exprDollar = exprS[exprpt-4 : exprpt+1] { exprVAL.BinOpExpr = mustNewBinOpExpr("or", exprDollar[3].BinOpModifier, exprDollar[1].Expr, exprDollar[4].Expr) } - case 129: + case 130: exprDollar = exprS[exprpt-4 : exprpt+1] { exprVAL.BinOpExpr = mustNewBinOpExpr("and", exprDollar[3].BinOpModifier, exprDollar[1].Expr, exprDollar[4].Expr) } - case 130: + case 131: exprDollar = exprS[exprpt-4 : exprpt+1] { exprVAL.BinOpExpr = mustNewBinOpExpr("unless", exprDollar[3].BinOpModifier, exprDollar[1].Expr, exprDollar[4].Expr) } - case 131: + case 132: exprDollar = exprS[exprpt-4 : exprpt+1] { exprVAL.BinOpExpr = mustNewBinOpExpr("+", exprDollar[3].BinOpModifier, exprDollar[1].Expr, exprDollar[4].Expr) } - case 132: + case 133: exprDollar = exprS[exprpt-4 : exprpt+1] { exprVAL.BinOpExpr = mustNewBinOpExpr("-", exprDollar[3].BinOpModifier, exprDollar[1].Expr, exprDollar[4].Expr) } - case 133: + case 134: exprDollar = exprS[exprpt-4 : exprpt+1] { exprVAL.BinOpExpr = mustNewBinOpExpr("*", exprDollar[3].BinOpModifier, exprDollar[1].Expr, exprDollar[4].Expr) } - case 134: + case 135: exprDollar = exprS[exprpt-4 : exprpt+1] { exprVAL.BinOpExpr = mustNewBinOpExpr("/", exprDollar[3].BinOpModifier, exprDollar[1].Expr, exprDollar[4].Expr) } - case 135: + case 136: exprDollar = exprS[exprpt-4 : exprpt+1] { exprVAL.BinOpExpr = mustNewBinOpExpr("%", exprDollar[3].BinOpModifier, exprDollar[1].Expr, exprDollar[4].Expr) } - case 136: + case 137: exprDollar = exprS[exprpt-4 : exprpt+1] { exprVAL.BinOpExpr = mustNewBinOpExpr("^", exprDollar[3].BinOpModifier, exprDollar[1].Expr, exprDollar[4].Expr) } - case 137: + case 138: exprDollar = exprS[exprpt-4 : exprpt+1] { exprVAL.BinOpExpr = mustNewBinOpExpr("==", exprDollar[3].BinOpModifier, exprDollar[1].Expr, exprDollar[4].Expr) } - case 138: + case 139: exprDollar = exprS[exprpt-4 : exprpt+1] { exprVAL.BinOpExpr = mustNewBinOpExpr("!=", exprDollar[3].BinOpModifier, exprDollar[1].Expr, exprDollar[4].Expr) } - case 139: + case 140: exprDollar = exprS[exprpt-4 : exprpt+1] { exprVAL.BinOpExpr = mustNewBinOpExpr(">", exprDollar[3].BinOpModifier, exprDollar[1].Expr, exprDollar[4].Expr) } - case 140: + case 141: exprDollar = exprS[exprpt-4 : exprpt+1] { exprVAL.BinOpExpr = mustNewBinOpExpr(">=", exprDollar[3].BinOpModifier, exprDollar[1].Expr, exprDollar[4].Expr) } - case 141: + case 142: exprDollar = exprS[exprpt-4 : exprpt+1] { exprVAL.BinOpExpr = mustNewBinOpExpr("<", exprDollar[3].BinOpModifier, exprDollar[1].Expr, exprDollar[4].Expr) } - case 142: + case 143: exprDollar = exprS[exprpt-4 : exprpt+1] { exprVAL.BinOpExpr = mustNewBinOpExpr("<=", exprDollar[3].BinOpModifier, exprDollar[1].Expr, exprDollar[4].Expr) } - case 143: + case 144: exprDollar = exprS[exprpt-0 : exprpt+1] { exprVAL.BinOpModifier = BinOpOptions{} } - case 144: + case 145: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.BinOpModifier = BinOpOptions{ReturnBool: true} } - case 145: + case 146: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.LiteralExpr = mustNewLiteralExpr(exprDollar[1].str, false) } - case 146: + case 147: exprDollar = exprS[exprpt-2 : exprpt+1] { exprVAL.LiteralExpr = mustNewLiteralExpr(exprDollar[2].str, false) } - case 147: + case 148: exprDollar = exprS[exprpt-2 : exprpt+1] { exprVAL.LiteralExpr = mustNewLiteralExpr(exprDollar[2].str, true) } - case 148: + case 149: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.VectorOp = OpTypeSum } - case 149: + case 150: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.VectorOp = OpTypeAvg } - case 150: + case 151: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.VectorOp = OpTypeCount } - case 151: + case 152: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.VectorOp = OpTypeMax } - case 152: + case 153: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.VectorOp = OpTypeMin } - case 153: + case 154: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.VectorOp = OpTypeStddev } - case 154: + case 155: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.VectorOp = OpTypeStdvar } - case 155: + case 156: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.VectorOp = OpTypeBottomK } - case 156: + case 157: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.VectorOp = OpTypeTopK } - case 157: + case 158: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.RangeOp = OpRangeTypeCount } - case 158: + case 159: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.RangeOp = OpRangeTypeRate } - case 159: + case 160: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.RangeOp = OpRangeTypeBytes } - case 160: + case 161: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.RangeOp = OpRangeTypeBytesRate } - case 161: + case 162: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.RangeOp = OpRangeTypeAvg } - case 162: + case 163: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.RangeOp = OpRangeTypeSum } - case 163: + case 164: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.RangeOp = OpRangeTypeMin } - case 164: + case 165: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.RangeOp = OpRangeTypeMax } - case 165: + case 166: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.RangeOp = OpRangeTypeStdvar } - case 166: + case 167: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.RangeOp = OpRangeTypeStddev } - case 167: + case 168: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.RangeOp = OpRangeTypeQuantile } - case 168: + case 169: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.RangeOp = OpRangeTypeFirst } - case 169: + case 170: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.RangeOp = OpRangeTypeLast } - case 170: + case 171: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.RangeOp = OpRangeTypeAbsent } - case 171: + case 172: exprDollar = exprS[exprpt-2 : exprpt+1] { exprVAL.OffsetExpr = newOffsetExpr(exprDollar[2].duration) } - case 172: + case 173: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.Labels = []string{exprDollar[1].str} } - case 173: + case 174: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.Labels = append(exprDollar[1].Labels, exprDollar[3].str) } - case 174: + case 175: exprDollar = exprS[exprpt-4 : exprpt+1] { exprVAL.Grouping = &grouping{without: false, groups: exprDollar[3].Labels} } - case 175: + case 176: exprDollar = exprS[exprpt-4 : exprpt+1] { exprVAL.Grouping = &grouping{without: true, groups: exprDollar[3].Labels} } - case 176: + case 177: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.Grouping = &grouping{without: false, groups: nil} } - case 177: + case 178: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.Grouping = &grouping{without: true, groups: nil} diff --git a/pkg/logql/log/ip.go b/pkg/logql/log/ip.go index d90b01a55da2c..c0036b250961d 100644 --- a/pkg/logql/log/ip.go +++ b/pkg/logql/log/ip.go @@ -9,7 +9,8 @@ import ( ) var ( - errInvalidPattern = errors.New("invalid pattern") + errIPFilterInvalidPattern = errors.New("ip: invalid pattern") + errIPFilterInvalidOperation = errors.New("ip: invalid operation") ) type IPMatchType int @@ -36,6 +37,9 @@ type IPFilter struct { // patternError represents any invalid pattern provided to match the ip. patternError error + + // filter `operation` used during label filter. + labelOp LabelFilterType } // IPFilter search for IP addresses of given `pattern` in the given `line`. @@ -45,7 +49,7 @@ type IPFilter struct { // 1. SINGLE-IP - "192.168.0.1" // 2. IP RANGE - "192.168.0.1-192.168.0.23" // 3. CIDR - "192.168.0.0/16" -func NewIPFilter(pattern string) *IPFilter { +func newIPFilter(pattern string) *IPFilter { filter := &IPFilter{pattern: pattern} matcher, err := getMatcher(pattern) @@ -55,26 +59,28 @@ func NewIPFilter(pattern string) *IPFilter { return filter } -func NewIPLabelFilter(pattern string, label string) *IPFilter { - filter := NewIPFilter(pattern) +func NewIPLabelFilter(label string, op LabelFilterType, pattern string) *IPFilter { + filter := newIPFilter(pattern) filter.labelName = label filter.matchType = IPMatchLabel + filter.labelOp = op return filter } -func (ipf *IPFilter) Filter(line string) bool { +// filter does the heavy lifting finding ip `pattern` in the givin `line`. +func (ipf *IPFilter) filter(line []byte) bool { if len(line) == 0 { return false } n := len(line) - filterFn := func(line string, start int, charset string) (bool, int) { - iplen := stringSpan(line[start:], charset) + filterFn := func(line []byte, start int, charset string) (bool, int) { + iplen := bytesSpan(line[start:], []byte(charset)) if iplen < 0 { return false, 0 } - ip, err := netaddr.ParseIP(line[start : start+iplen]) + ip, err := netaddr.ParseIP(string(line[start : start+iplen])) if err == nil { if contains(ipf.matcher, ip) { return true, 0 @@ -116,23 +122,30 @@ func (ipf *IPFilter) Process(line []byte, lbs *LabelsBuilder) ([]byte, bool) { return line, false } - input := string(line) - if ipf.matchType == IPMatchLabel { v, ok := lbs.Get(ipf.labelName) if !ok { // we have not found the corresponding label. return line, false } - input = v + + input := []byte(v) + switch ipf.labelOp { + case LabelFilterEqual: + return line, ipf.filter(input) + case LabelFilterNotEqual: + return line, !ipf.filter(input) + default: + lbs.SetErr(errIPFilterInvalidOperation.Error()) + } } - return line, ipf.Filter(input) + return line, ipf.filter(line) } // `RequiredLabelNames` implements `Stage` interface func (ipf *IPFilter) RequiredLabelNames() []string { - return []string{} + return []string{ipf.labelName} } // `String` implements fmt.Stringer inteface, by which also implements `LabelFilterer` inteface. @@ -175,7 +188,7 @@ func getMatcher(pattern string) (IPMatcher, error) { return matcher, nil } - return nil, fmt.Errorf("%w: %q", errInvalidPattern, pattern) + return nil, fmt.Errorf("%w: %q", errIPFilterInvalidPattern, pattern) } func ipv4Hint(prefix [4]byte) bool { @@ -196,11 +209,11 @@ func isHexDigit(r byte) bool { return unicode.IsDigit(rune(r)) || (r >= 'a' && r <= 'f') || (r >= 'A' && r <= 'F') } -// stringSpan is same as C's `strcspan()` function. +// bytesSpan is same as C's `strcspan()` function. // It returns the number of chars in the initial segment of `s` // which consist only of chars from `accept`. -func stringSpan(s, accept string) int { - m := make(map[rune]bool) +func bytesSpan(s, accept []byte) int { + m := make(map[byte]bool) for _, r := range accept { m[r] = true diff --git a/pkg/logql/log/ip_test.go b/pkg/logql/log/ip_test.go index db88ebd8a8fb8..dd5e49146c27a 100644 --- a/pkg/logql/log/ip_test.go +++ b/pkg/logql/log/ip_test.go @@ -77,7 +77,7 @@ func Test_IPFilter(t *testing.T) { name: "wrong IP range syntax extra space", // NOTE(kavi): Should we handle this as normal valid range pattern? pat: "192.168.0.1 - 192.189.10.12", fail: true, - err: errInvalidPattern, + err: errIPFilterInvalidPattern, }, { name: "CIDR", @@ -111,7 +111,7 @@ func Test_IPFilter(t *testing.T) { for _, c := range cases { t.Run(c.name, func(t *testing.T) { - ip := NewIPFilter(c.pat) + ip := newIPFilter(c.pat) if c.fail { assert.Error(t, c.err, ip.patternError) return @@ -120,7 +120,7 @@ func Test_IPFilter(t *testing.T) { got := make([]int, 0) for i, in := range c.input { - if ip.Filter(in) { + if ip.filter([]byte(in)) { got = append(got, i) } } @@ -129,6 +129,91 @@ func Test_IPFilter(t *testing.T) { } } +func Test_IPLabelFilterOp(t *testing.T) { + cases := []struct { + name string + pat string + op LabelFilterType + label string + val []byte + expectedMatch bool + + fail bool + err string // label filter errors are just treated as strings :( + }{ + { + name: "equal operator", + pat: "192.168.0.1", + op: LabelFilterEqual, + label: "addr", + val: []byte("192.168.0.1"), + expectedMatch: true, + }, + { + name: "not equal operator", + pat: "192.168.0.2", + op: LabelFilterEqual, + label: "addr", + val: []byte("192.168.0.1"), // no match + expectedMatch: false, + }, + { + name: "great than operator-not-supported", + pat: "192.168.0.2", + op: LabelFilterGreaterThan, // not supported + label: "addr", + val: []byte("192.168.0.1"), + fail: true, + err: errIPFilterInvalidOperation.Error(), + }, + { + name: "great than or equal to operator-not-supported", + pat: "192.168.0.2", + op: LabelFilterGreaterThanOrEqual, // not supported + label: "addr", + val: []byte("192.168.0.1"), + fail: true, + err: errIPFilterInvalidOperation.Error(), + }, + { + name: "less than operator-not-supported", + pat: "192.168.0.2", + op: LabelFilterLesserThan, // not supported + label: "addr", + val: []byte("192.168.0.1"), + fail: true, + err: errIPFilterInvalidOperation.Error(), + }, + { + name: "less than or equal to operator-not-supported", + pat: "192.168.0.2", + op: LabelFilterLesserThanOrEqual, // not supported + label: "addr", + val: []byte("192.168.0.1"), + fail: true, + err: errIPFilterInvalidOperation.Error(), + }, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + lf := NewIPLabelFilter(c.label, c.op, c.pat) + require.NoError(t, lf.patternError) + + lbs := labels.Labels{labels.Label{Name: c.label, Value: string(c.val)}} + lbb := NewBaseLabelsBuilder().ForLabels(lbs, lbs.Hash()) + _, ok := lf.Process([]byte("x"), lbb) + if c.fail { + assert.NotEmpty(t, lbb.GetErr()) + assert.False(t, ok) + return + } + require.Empty(t, lbb.GetErr()) + assert.Equal(t, c.expectedMatch, ok) + }) + } +} + func Benchmark_IPFilter(b *testing.B) { b.ReportAllocs() @@ -146,7 +231,7 @@ func Benchmark_IPFilter(b *testing.B) { "192.168.4.5/16", } { b.Run(pattern, func(b *testing.B) { - stage := NewIPFilter(pattern) + stage := newIPFilter(pattern) require.Nil(b, stage.patternError) b.ResetTimer() diff --git a/pkg/logql/parser_test.go b/pkg/logql/parser_test.go index 3ad48ef993bb8..7283c445b6b9c 100644 --- a/pkg/logql/parser_test.go +++ b/pkg/logql/parser_test.go @@ -338,10 +338,6 @@ func TestParse(t *testing.T) { in: `min({ foo = "bar" }[5m])`, err: logqlmodel.NewParseError("syntax error: unexpected RANGE", 0, 20), }, - { - in: `{ foo = "bar" }|logfmt|addr!=ip("1.2.3.4")`, - err: logqlmodel.NewParseError("syntax error: unexpected ip, expecting BYTES or STRING or NUMBER or DURATION", 1, 30), - }, { in: `{ foo = "bar" }|logfmt|addr>=ip("1.2.3.4")`, err: logqlmodel.NewParseError("syntax error: unexpected ip, expecting BYTES or NUMBER or DURATION", 1, 30), @@ -362,33 +358,74 @@ func TestParse(t *testing.T) { in: `{ foo = "bar" }|logfmt|addr=ip("1.2.3.4")`, exp: newPipelineExpr( newMatcherExpr([]*labels.Matcher{mustNewMatcher(labels.MatchEqual, "foo", "bar")}), - MultiStageExpr{newLabelParserExpr(OpParserTypeLogfmt, ""), newLabelFilterExpr(log.NewIPLabelFilter("1.2.3.4", "addr"))}, + MultiStageExpr{newLabelParserExpr(OpParserTypeLogfmt, ""), newLabelFilterExpr(log.NewIPLabelFilter("addr", log.LabelFilterEqual, "1.2.3.4"))}, ), }, { - in: `{ foo = "bar" }|logfmt|level="error"| addr=ip("1.2.3.4")`, + in: `{ foo = "bar" }|logfmt|addr!=ip("1.2.3.4")`, + exp: newPipelineExpr( + newMatcherExpr([]*labels.Matcher{mustNewMatcher(labels.MatchEqual, "foo", "bar")}), + MultiStageExpr{newLabelParserExpr(OpParserTypeLogfmt, ""), newLabelFilterExpr(log.NewIPLabelFilter("addr", log.LabelFilterNotEqual, "1.2.3.4"))}, + ), + }, + { + in: `{ foo = "bar" }|logfmt|level="error"|addr=ip("1.2.3.4")`, exp: newPipelineExpr( newMatcherExpr([]*labels.Matcher{mustNewMatcher(labels.MatchEqual, "foo", "bar")}), MultiStageExpr{ newLabelParserExpr(OpParserTypeLogfmt, ""), newLabelFilterExpr(log.NewStringLabelFilter(mustNewMatcher(labels.MatchEqual, "level", "error"))), - newLabelFilterExpr(log.NewIPLabelFilter("1.2.3.4", "addr")), + newLabelFilterExpr(log.NewIPLabelFilter("addr", log.LabelFilterEqual, "1.2.3.4")), }, ), }, { - in: `{ foo = "bar" }|logfmt|remote_addr=ip("2.3.4.5")|level="error"| addr=ip("1.2.3.4")`, // chain label filters with ip matcher + in: `{ foo = "bar" }|logfmt|level="error"|addr!=ip("1.2.3.4")`, exp: newPipelineExpr( newMatcherExpr([]*labels.Matcher{mustNewMatcher(labels.MatchEqual, "foo", "bar")}), MultiStageExpr{ newLabelParserExpr(OpParserTypeLogfmt, ""), - newLabelFilterExpr(log.NewIPLabelFilter("2.3.4.5", "remote_addr")), newLabelFilterExpr(log.NewStringLabelFilter(mustNewMatcher(labels.MatchEqual, "level", "error"))), - newLabelFilterExpr(log.NewIPLabelFilter("1.2.3.4", "addr")), + newLabelFilterExpr(log.NewIPLabelFilter("addr", log.LabelFilterNotEqual, "1.2.3.4")), + }, + ), + }, + { + in: `{ foo = "bar" }|logfmt|remote_addr=ip("2.3.4.5")|level="error"|addr=ip("1.2.3.4")`, // chain label filters with ip matcher + exp: newPipelineExpr( + newMatcherExpr([]*labels.Matcher{mustNewMatcher(labels.MatchEqual, "foo", "bar")}), + MultiStageExpr{ + newLabelParserExpr(OpParserTypeLogfmt, ""), + newLabelFilterExpr(log.NewIPLabelFilter("remote_addr", log.LabelFilterEqual, "2.3.4.5")), + newLabelFilterExpr(log.NewStringLabelFilter(mustNewMatcher(labels.MatchEqual, "level", "error"))), + newLabelFilterExpr(log.NewIPLabelFilter("addr", log.LabelFilterEqual, "1.2.3.4")), + }, + ), + }, + { + in: `{ foo = "bar" }|logfmt|remote_addr!=ip("2.3.4.5")|level="error"|addr!=ip("1.2.3.4")`, // chain label filters with ip matcher + exp: newPipelineExpr( + newMatcherExpr([]*labels.Matcher{mustNewMatcher(labels.MatchEqual, "foo", "bar")}), + MultiStageExpr{ + newLabelParserExpr(OpParserTypeLogfmt, ""), + newLabelFilterExpr(log.NewIPLabelFilter("remote_addr", log.LabelFilterNotEqual, "2.3.4.5")), + newLabelFilterExpr(log.NewStringLabelFilter(mustNewMatcher(labels.MatchEqual, "level", "error"))), + newLabelFilterExpr(log.NewIPLabelFilter("addr", log.LabelFilterNotEqual, "1.2.3.4")), + }, + ), + }, + { + in: `{ foo = "bar" }|logfmt|remote_addr=ip("2.3.4.5")|level="error"|addr!=ip("1.2.3.4")`, // chain label filters with ip matcher + exp: newPipelineExpr( + newMatcherExpr([]*labels.Matcher{mustNewMatcher(labels.MatchEqual, "foo", "bar")}), + MultiStageExpr{ + newLabelParserExpr(OpParserTypeLogfmt, ""), + newLabelFilterExpr(log.NewIPLabelFilter("remote_addr", log.LabelFilterEqual, "2.3.4.5")), + newLabelFilterExpr(log.NewStringLabelFilter(mustNewMatcher(labels.MatchEqual, "level", "error"))), + newLabelFilterExpr(log.NewIPLabelFilter("addr", log.LabelFilterNotEqual, "1.2.3.4")), }, ), }, - { in: `sum(3 ,count_over_time({ foo = "bar" }[5h]))`, err: logqlmodel.NewParseError("unsupported parameter for operation sum(3,", 0, 0), From c543730c0a0b87cc7ca8b7615f17886d9b5e9a2b Mon Sep 17 00:00:00 2001 From: Kaviraj Date: Tue, 13 Jul 2021 12:47:04 +0200 Subject: [PATCH 10/24] Make IP filter work as line filter 1. Add necessary interface implementation 2. More tests for parser and ast Signed-off-by: Kaviraj --- pkg/logql/ast.go | 342 ++++++++--------- pkg/logql/ast_test.go | 2 + pkg/logql/expr.y | 30 +- pkg/logql/expr.y.go | 555 +++++++++++++++------------- pkg/logql/log/ip.go | 39 +- pkg/logql/log/ip_test.go | 10 +- pkg/logql/parser_test.go | 405 +++++++++++++------- pkg/logql/shardmapper_test.go | 4 +- pkg/querier/http.go | 2 +- pkg/querier/queryrange/roundtrip.go | 2 +- 10 files changed, 767 insertions(+), 624 deletions(-) diff --git a/pkg/logql/ast.go b/pkg/logql/ast.go index 5818146039f42..c798b6698dc4f 100644 --- a/pkg/logql/ast.go +++ b/pkg/logql/ast.go @@ -22,7 +22,6 @@ import ( type Expr interface { logQLExpr() // ensure it's not implemented accidentally Shardable() bool // A recursive check on the AST to see if it's shardable. - Walkable fmt.Stringer } @@ -87,7 +86,7 @@ type Querier interface { // LogSelectorExpr is a LogQL expression filtering and returning logs. type LogSelectorExpr interface { Matchers() []*labels.Matcher - LogPipelineExpr + PipelineExpr HasFilter() bool Expr } @@ -98,8 +97,8 @@ type ( SampleExtractor = log.SampleExtractor ) -// LogPipelineExpr is an expression defining a log pipeline. -type LogPipelineExpr interface { +// PipelineExpr is an expression defining a log pipeline. +type PipelineExpr interface { Pipeline() (Pipeline, error) Expr } @@ -149,28 +148,22 @@ func (m MultiStageExpr) String() string { func (MultiStageExpr) logQLExpr() {} // nolint:unused -type MatchersExpr struct { +type matchersExpr struct { matchers []*labels.Matcher implicit } -func newMatcherExpr(matchers []*labels.Matcher) *MatchersExpr { - return &MatchersExpr{matchers: matchers} +func newMatcherExpr(matchers []*labels.Matcher) *matchersExpr { + return &matchersExpr{matchers: matchers} } -func (e *MatchersExpr) Matchers() []*labels.Matcher { +func (e *matchersExpr) Matchers() []*labels.Matcher { return e.matchers } -func (e *MatchersExpr) AppendMatchers(m []*labels.Matcher) { - e.matchers = append(e.matchers, m...) -} - -func (e *MatchersExpr) Shardable() bool { return true } - -func (e *MatchersExpr) Walk(f WalkFn) { f(e) } +func (e *matchersExpr) Shardable() bool { return true } -func (e *MatchersExpr) String() string { +func (e *matchersExpr) String() string { var sb strings.Builder sb.WriteString("{") for i, m := range e.matchers { @@ -183,28 +176,28 @@ func (e *MatchersExpr) String() string { return sb.String() } -func (e *MatchersExpr) Pipeline() (log.Pipeline, error) { +func (e *matchersExpr) Pipeline() (log.Pipeline, error) { return log.NewNoopPipeline(), nil } -func (e *MatchersExpr) HasFilter() bool { +func (e *matchersExpr) HasFilter() bool { return false } -type PipelineExpr struct { +type pipelineExpr struct { pipeline MultiStageExpr - left *MatchersExpr + left *matchersExpr implicit } -func newPipelineExpr(left *MatchersExpr, pipeline MultiStageExpr) LogSelectorExpr { - return &PipelineExpr{ +func newPipelineExpr(left *matchersExpr, pipeline MultiStageExpr) LogSelectorExpr { + return &pipelineExpr{ left: left, pipeline: pipeline, } } -func (e *PipelineExpr) Shardable() bool { +func (e *pipelineExpr) Shardable() bool { for _, p := range e.pipeline { if !p.Shardable() { return false @@ -213,26 +206,11 @@ func (e *PipelineExpr) Shardable() bool { return true } -func (e *PipelineExpr) Walk(f WalkFn) { - f(e) - - if e.left == nil { - return - } - - xs := make([]Walkable, 0, len(e.pipeline)+1) - xs = append(xs, e.left) - for _, p := range e.pipeline { - xs = append(xs, p) - } - walkAll(f, xs...) -} - -func (e *PipelineExpr) Matchers() []*labels.Matcher { +func (e *pipelineExpr) Matchers() []*labels.Matcher { return e.left.Matchers() } -func (e *PipelineExpr) String() string { +func (e *pipelineExpr) String() string { var sb strings.Builder sb.WriteString(e.left.String()) sb.WriteString(" ") @@ -240,15 +218,15 @@ func (e *PipelineExpr) String() string { return sb.String() } -func (e *PipelineExpr) Pipeline() (log.Pipeline, error) { +func (e *pipelineExpr) Pipeline() (log.Pipeline, error) { return e.pipeline.Pipeline() } // HasFilter returns true if the pipeline contains stage that can filter out lines. -func (e *PipelineExpr) HasFilter() bool { +func (e *pipelineExpr) HasFilter() bool { for _, p := range e.pipeline { switch p.(type) { - case *LineFilterExpr, *LabelFilterExpr: + case *lineFilterExpr, *labelFilterExpr: return true default: continue @@ -257,36 +235,38 @@ func (e *PipelineExpr) HasFilter() bool { return false } -type LineFilterExpr struct { - left *LineFilterExpr +type lineFilterExpr struct { + left *lineFilterExpr ty labels.MatchType match string + op string implicit } -func newLineFilterExpr(left *LineFilterExpr, ty labels.MatchType, match string) *LineFilterExpr { - return &LineFilterExpr{ - left: left, +func newLineFilterExpr(ty labels.MatchType, op, match string) *lineFilterExpr { + return &lineFilterExpr{ ty: ty, match: match, + op: op, } } -func (e *LineFilterExpr) Walk(f WalkFn) { - f(e) - if e.left == nil { - return +func newNestedLineFilterExpr(left *lineFilterExpr, right *lineFilterExpr) *lineFilterExpr { + return &lineFilterExpr{ + left: left, + ty: right.ty, + match: right.match, + op: right.op, } - e.left.Walk(f) } // AddFilterExpr adds a filter expression to a logselector expression. -func AddFilterExpr(expr LogSelectorExpr, ty labels.MatchType, match string) (LogSelectorExpr, error) { - filter := newLineFilterExpr(nil, ty, match) +func AddFilterExpr(expr LogSelectorExpr, ty labels.MatchType, op, match string) (LogSelectorExpr, error) { + filter := newLineFilterExpr(ty, op, match) switch e := expr.(type) { - case *MatchersExpr: + case *matchersExpr: return newPipelineExpr(e, MultiStageExpr{filter}), nil - case *PipelineExpr: + case *pipelineExpr: e.pipeline = append(e.pipeline, filter) return e, nil default: @@ -294,9 +274,9 @@ func AddFilterExpr(expr LogSelectorExpr, ty labels.MatchType, match string) (Log } } -func (e *LineFilterExpr) Shardable() bool { return true } +func (e *lineFilterExpr) Shardable() bool { return true } -func (e *LineFilterExpr) String() string { +func (e *lineFilterExpr) String() string { var sb strings.Builder if e.left != nil { sb.WriteString(e.left.String()) @@ -313,14 +293,35 @@ func (e *LineFilterExpr) String() string { sb.WriteString("!=") } sb.WriteString(" ") + if e.op == "" { + sb.WriteString(strconv.Quote(e.match)) + return sb.String() + } + sb.WriteString(e.op) + sb.WriteString("(") sb.WriteString(strconv.Quote(e.match)) + sb.WriteString(")") return sb.String() } -func (e *LineFilterExpr) Filter() (log.Filterer, error) { - f, err := log.NewFilter(e.match, e.ty) - if err != nil { - return nil, err +func (e *lineFilterExpr) Filter() (log.Filterer, error) { + var f log.Filterer + + if e.op == OpFilterIP { + // ip() supports only |= (matchEqual) and !=(matchNotEqual) matchtypes + switch e.ty { + case labels.MatchEqual, labels.MatchNotEqual: + default: + return nil, log.ErrIPFilterInvalidOperation + } + + f = log.NewIPLineFilter(e.match) + } else { + var err error // to avoid `f` being shadowed. + f, err = log.NewFilter(e.match, e.ty) + if err != nil { + return nil, err + } } if e.left != nil { nextFilter, err := e.left.Filter() @@ -335,7 +336,7 @@ func (e *LineFilterExpr) Filter() (log.Filterer, error) { return f, nil } -func (e *LineFilterExpr) Stage() (log.Stage, error) { +func (e *lineFilterExpr) Stage() (log.Stage, error) { f, err := e.Filter() if err != nil { return nil, err @@ -343,24 +344,22 @@ func (e *LineFilterExpr) Stage() (log.Stage, error) { return f.ToStage(), nil } -type LabelParserExpr struct { +type labelParserExpr struct { op string param string implicit } -func newLabelParserExpr(op, param string) *LabelParserExpr { - return &LabelParserExpr{ +func newLabelParserExpr(op, param string) *labelParserExpr { + return &labelParserExpr{ op: op, param: param, } } -func (e *LabelParserExpr) Shardable() bool { return true } - -func (e *LabelParserExpr) Walk(f WalkFn) { f(e) } +func (e *labelParserExpr) Shardable() bool { return true } -func (e *LabelParserExpr) Stage() (log.Stage, error) { +func (e *labelParserExpr) Stage() (log.Stage, error) { switch e.op { case OpParserTypeJSON: return log.NewJSONParser(), nil @@ -377,7 +376,7 @@ func (e *LabelParserExpr) Stage() (log.Stage, error) { } } -func (e *LabelParserExpr) String() string { +func (e *labelParserExpr) String() string { var sb strings.Builder sb.WriteString(OpPipe) sb.WriteString(" ") @@ -389,7 +388,7 @@ func (e *LabelParserExpr) String() string { return sb.String() } -type LabelFilterExpr struct { +type labelFilterExpr struct { log.LabelFilterer implicit } @@ -402,60 +401,54 @@ func newLabelFilterExpr(filterer log.LabelFilterer) *labelFilterExpr { func (e *labelFilterExpr) Shardable() bool { return true } -func (e *LabelFilterExpr) Walk(f WalkFn) { f(e) } - -func (e *LabelFilterExpr) Stage() (log.Stage, error) { +func (e *labelFilterExpr) Stage() (log.Stage, error) { return e.LabelFilterer, nil } -func (e *LabelFilterExpr) String() string { +func (e *labelFilterExpr) String() string { return fmt.Sprintf("%s %s", OpPipe, e.LabelFilterer.String()) } -type LineFmtExpr struct { +type lineFmtExpr struct { value string implicit } -func newLineFmtExpr(value string) *LineFmtExpr { - return &LineFmtExpr{ +func newLineFmtExpr(value string) *lineFmtExpr { + return &lineFmtExpr{ value: value, } } -func (e *LineFmtExpr) Shardable() bool { return true } +func (e *lineFmtExpr) Shardable() bool { return true } -func (e *LineFmtExpr) Walk(f WalkFn) { f(e) } - -func (e *LineFmtExpr) Stage() (log.Stage, error) { +func (e *lineFmtExpr) Stage() (log.Stage, error) { return log.NewFormatter(e.value) } -func (e *LineFmtExpr) String() string { +func (e *lineFmtExpr) String() string { return fmt.Sprintf("%s %s %s", OpPipe, OpFmtLine, strconv.Quote(e.value)) } -type LabelFmtExpr struct { +type labelFmtExpr struct { formats []log.LabelFmt implicit } -func newLabelFmtExpr(fmts []log.LabelFmt) *LabelFmtExpr { - return &LabelFmtExpr{ +func newLabelFmtExpr(fmts []log.LabelFmt) *labelFmtExpr { + return &labelFmtExpr{ formats: fmts, } } -func (e *LabelFmtExpr) Shardable() bool { return false } - -func (e *LabelFmtExpr) Walk(f WalkFn) { f(e) } +func (e *labelFmtExpr) Shardable() bool { return false } -func (e *LabelFmtExpr) Stage() (log.Stage, error) { +func (e *labelFmtExpr) Stage() (log.Stage, error) { return log.NewLabelsFormatter(e.formats) } -func (e *LabelFmtExpr) String() string { +func (e *labelFmtExpr) String() string { var sb strings.Builder sb.WriteString(fmt.Sprintf("%s %s ", OpPipe, OpFmtLabel)) for i, f := range e.formats { @@ -473,27 +466,25 @@ func (e *LabelFmtExpr) String() string { return sb.String() } -type JSONExpressionParser struct { +type jsonExpressionParser struct { expressions []log.JSONExpression implicit } -func newJSONExpressionParser(expressions []log.JSONExpression) *JSONExpressionParser { - return &JSONExpressionParser{ +func newJSONExpressionParser(expressions []log.JSONExpression) *jsonExpressionParser { + return &jsonExpressionParser{ expressions: expressions, } } -func (j *JSONExpressionParser) Shardable() bool { return true } - -func (j *JSONExpressionParser) Walk(f WalkFn) { f(j) } +func (j *jsonExpressionParser) Shardable() bool { return true } -func (j *JSONExpressionParser) Stage() (log.Stage, error) { +func (j *jsonExpressionParser) Stage() (log.Stage, error) { return log.NewJSONExpressionParser(j.expressions) } -func (j *JSONExpressionParser) String() string { +func (j *jsonExpressionParser) String() string { var sb strings.Builder sb.WriteString(fmt.Sprintf("%s %s ", OpPipe, OpParserTypeJSON)) for i, exp := range j.expressions { @@ -524,14 +515,14 @@ func mustNewFloat(s string) float64 { return n } -type UnwrapExpr struct { +type unwrapExpr struct { identifier string operation string postFilters []log.LabelFilterer } -func (u UnwrapExpr) String() string { +func (u unwrapExpr) String() string { var sb strings.Builder if u.operation != "" { sb.WriteString(fmt.Sprintf(" %s %s %s(%s)", OpPipe, OpUnwrap, u.operation, u.identifier)) @@ -544,25 +535,25 @@ func (u UnwrapExpr) String() string { return sb.String() } -func (u *UnwrapExpr) addPostFilter(f log.LabelFilterer) *UnwrapExpr { +func (u *unwrapExpr) addPostFilter(f log.LabelFilterer) *unwrapExpr { u.postFilters = append(u.postFilters, f) return u } -func newUnwrapExpr(id string, operation string) *UnwrapExpr { - return &UnwrapExpr{identifier: id, operation: operation} +func newUnwrapExpr(id string, operation string) *unwrapExpr { + return &unwrapExpr{identifier: id, operation: operation} } -type LogRange struct { +type logRange struct { left LogSelectorExpr interval time.Duration offset time.Duration - unwrap *UnwrapExpr + unwrap *unwrapExpr } // impls Stringer -func (r LogRange) String() string { +func (r logRange) String() string { var sb strings.Builder sb.WriteString(r.left.String()) if r.unwrap != nil { @@ -570,28 +561,20 @@ func (r LogRange) String() string { } sb.WriteString(fmt.Sprintf("[%v]", model.Duration(r.interval))) if r.offset != 0 { - offsetExpr := OffsetExpr{offset: r.offset} + offsetExpr := offsetExpr{offset: r.offset} sb.WriteString(offsetExpr.String()) } return sb.String() } -func (r *LogRange) Shardable() bool { return r.left.Shardable() } - -func (r *LogRange) Walk(f WalkFn) { - f(r) - if r.left == nil { - return - } - r.left.Walk(f) -} +func (r *logRange) Shardable() bool { return r.left.Shardable() } -func newLogRange(left LogSelectorExpr, interval time.Duration, u *UnwrapExpr, o *OffsetExpr) *LogRange { +func newLogRange(left LogSelectorExpr, interval time.Duration, u *unwrapExpr, o *offsetExpr) *logRange { var offset time.Duration if o != nil { offset = o.offset } - return &LogRange{ + return &logRange{ left: left, interval: interval, unwrap: u, @@ -599,18 +582,18 @@ func newLogRange(left LogSelectorExpr, interval time.Duration, u *UnwrapExpr, o } } -type OffsetExpr struct { +type offsetExpr struct { offset time.Duration } -func (o *OffsetExpr) String() string { +func (o *offsetExpr) String() string { var sb strings.Builder sb.WriteString(fmt.Sprintf(" %s %s", OpOffset, o.offset.String())) return sb.String() } -func newOffsetExpr(offset time.Duration) *OffsetExpr { - return &OffsetExpr{ +func newOffsetExpr(offset time.Duration) *offsetExpr { + return &offsetExpr{ offset: offset, } } @@ -716,8 +699,8 @@ type SampleExpr interface { Expr } -type RangeAggregationExpr struct { - left *LogRange +type rangeAggregationExpr struct { + left *logRange operation string params *float64 @@ -725,7 +708,7 @@ type RangeAggregationExpr struct { implicit } -func newRangeAggregationExpr(left *LogRange, operation string, gr *grouping, stringParams *string) SampleExpr { +func newRangeAggregationExpr(left *logRange, operation string, gr *grouping, stringParams *string) SampleExpr { var params *float64 if stringParams != nil { if operation != OpRangeTypeQuantile { @@ -743,7 +726,7 @@ func newRangeAggregationExpr(left *LogRange, operation string, gr *grouping, str panic(logqlmodel.NewParseError(fmt.Sprintf("parameter required for operation %s", operation), 0, 0)) } } - e := &RangeAggregationExpr{ + e := &rangeAggregationExpr{ left: left, operation: operation, grouping: gr, @@ -755,11 +738,11 @@ func newRangeAggregationExpr(left *LogRange, operation string, gr *grouping, str return e } -func (e *RangeAggregationExpr) Selector() LogSelectorExpr { +func (e *rangeAggregationExpr) Selector() LogSelectorExpr { return e.left.left } -func (e RangeAggregationExpr) validate() error { +func (e rangeAggregationExpr) validate() error { if e.grouping != nil { switch e.operation { case OpRangeTypeAvg, OpRangeTypeStddev, OpRangeTypeStdvar, OpRangeTypeQuantile, OpRangeTypeMax, OpRangeTypeMin, OpRangeTypeFirst, OpRangeTypeLast: @@ -784,7 +767,7 @@ func (e RangeAggregationExpr) validate() error { } // impls Stringer -func (e *RangeAggregationExpr) String() string { +func (e *rangeAggregationExpr) String() string { var sb strings.Builder sb.WriteString(e.operation) sb.WriteString("(") @@ -801,18 +784,10 @@ func (e *RangeAggregationExpr) String() string { } // impl SampleExpr -func (e *RangeAggregationExpr) Shardable() bool { +func (e *rangeAggregationExpr) Shardable() bool { return shardableOps[e.operation] && e.left.Shardable() } -func (e *RangeAggregationExpr) Walk(f WalkFn) { - f(e) - if e.left == nil { - return - } - e.left.Walk(f) -} - type grouping struct { groups []string without bool @@ -836,7 +811,7 @@ func (g grouping) String() string { return sb.String() } -type VectorAggregationExpr struct { +type vectorAggregationExpr struct { left SampleExpr grouping *grouping @@ -865,7 +840,7 @@ func mustNewVectorAggregationExpr(left SampleExpr, operation string, gr *groupin if gr == nil { gr = &grouping{} } - return &VectorAggregationExpr{ + return &vectorAggregationExpr{ left: left, operation: operation, grouping: gr, @@ -873,14 +848,14 @@ func mustNewVectorAggregationExpr(left SampleExpr, operation string, gr *groupin } } -func (e *VectorAggregationExpr) Selector() LogSelectorExpr { +func (e *vectorAggregationExpr) Selector() LogSelectorExpr { return e.left.Selector() } -func (e *VectorAggregationExpr) Extractor() (log.SampleExtractor, error) { +func (e *vectorAggregationExpr) Extractor() (log.SampleExtractor, error) { // inject in the range vector extractor the outer groups to improve performance. // This is only possible if the operation is a sum. Anything else needs all labels. - if r, ok := e.left.(*RangeAggregationExpr); ok && canInjectVectorGrouping(e.operation, r.operation) { + if r, ok := e.left.(*rangeAggregationExpr); ok && canInjectVectorGrouping(e.operation, r.operation) { // if the range vec operation has no grouping we can push down the vec one. if r.grouping == nil { return r.extractor(e.grouping) @@ -902,7 +877,7 @@ func canInjectVectorGrouping(vecOp, rangeOp string) bool { } } -func (e *VectorAggregationExpr) String() string { +func (e *vectorAggregationExpr) String() string { var params []string if e.params != 0 { params = []string{fmt.Sprintf("%d", e.params), e.left.String()} @@ -913,30 +888,22 @@ func (e *VectorAggregationExpr) String() string { } // impl SampleExpr -func (e *VectorAggregationExpr) Shardable() bool { +func (e *vectorAggregationExpr) Shardable() bool { return shardableOps[e.operation] && e.left.Shardable() } -func (e *VectorAggregationExpr) Walk(f WalkFn) { - f(e) - if e.left == nil { - return - } - e.left.Walk(f) -} - type BinOpOptions struct { ReturnBool bool } -type BinOpExpr struct { +type binOpExpr struct { SampleExpr RHS SampleExpr op string opts BinOpOptions } -func (e *BinOpExpr) String() string { +func (e *binOpExpr) String() string { if e.opts.ReturnBool { return fmt.Sprintf("(%s %s bool %s)", e.SampleExpr.String(), e.op, e.RHS.String()) } @@ -944,14 +911,10 @@ func (e *BinOpExpr) String() string { } // impl SampleExpr -func (e *BinOpExpr) Shardable() bool { +func (e *binOpExpr) Shardable() bool { return shardableOps[e.op] && e.SampleExpr.Shardable() && e.RHS.Shardable() } -func (e *BinOpExpr) Walk(f WalkFn) { - walkAll(f, e.SampleExpr, e.RHS) -} - func mustNewBinOpExpr(op string, opts BinOpOptions, lhs, rhs Expr) SampleExpr { left, ok := lhs.(SampleExpr) if !ok { @@ -971,8 +934,8 @@ func mustNewBinOpExpr(op string, opts BinOpOptions, lhs, rhs Expr) SampleExpr { ), 0, 0)) } - leftLit, lOk := left.(*LiteralExpr) - rightLit, rOk := right.(*LiteralExpr) + leftLit, lOk := left.(*literalExpr) + rightLit, rOk := right.(*literalExpr) if IsLogicalBinOp(op) { if lOk { @@ -997,7 +960,7 @@ func mustNewBinOpExpr(op string, opts BinOpOptions, lhs, rhs Expr) SampleExpr { return reduceBinOp(op, leftLit, rightLit) } - return &BinOpExpr{ + return &binOpExpr{ SampleExpr: left, RHS: right, op: op, @@ -1008,7 +971,7 @@ func mustNewBinOpExpr(op string, opts BinOpOptions, lhs, rhs Expr) SampleExpr { // Reduces a binary operation expression. A binop is reducible if both of its legs are literal expressions. // This is because literals need match all labels, which is currently difficult to encode into StepEvaluators. // Therefore, we ensure a binop can be reduced/simplified, maintaining the invariant that it does not have two literal legs. -func reduceBinOp(op string, left, right *LiteralExpr) *LiteralExpr { +func reduceBinOp(op string, left, right *literalExpr) *literalExpr { merged := mergeBinOp( op, &promql.Sample{Point: promql.Point{V: left.value}}, @@ -1016,15 +979,15 @@ func reduceBinOp(op string, left, right *LiteralExpr) *LiteralExpr { false, false, ) - return &LiteralExpr{value: merged.V} + return &literalExpr{value: merged.V} } -type LiteralExpr struct { +type literalExpr struct { value float64 implicit } -func mustNewLiteralExpr(s string, invert bool) *LiteralExpr { +func mustNewLiteralExpr(s string, invert bool) *literalExpr { n, err := strconv.ParseFloat(s, 64) if err != nil { panic(logqlmodel.NewParseError(fmt.Sprintf("unable to parse literal as a float: %s", err.Error()), 0, 0)) @@ -1034,25 +997,24 @@ func mustNewLiteralExpr(s string, invert bool) *LiteralExpr { n = -n } - return &LiteralExpr{ + return &literalExpr{ value: n, } } -func (e *LiteralExpr) String() string { +func (e *literalExpr) String() string { return fmt.Sprint(e.value) } // literlExpr impls SampleExpr & LogSelectorExpr mainly to reduce the need for more complicated typings // to facilitate sum types. We'll be type switching when evaluating them anyways // and they will only be present in binary operation legs. -func (e *LiteralExpr) Selector() LogSelectorExpr { return e } -func (e *LiteralExpr) HasFilter() bool { return false } -func (e *LiteralExpr) Shardable() bool { return true } -func (e *LiteralExpr) Walk(f WalkFn) { f(e) } -func (e *LiteralExpr) Pipeline() (log.Pipeline, error) { return log.NewNoopPipeline(), nil } -func (e *LiteralExpr) Matchers() []*labels.Matcher { return nil } -func (e *LiteralExpr) Extractor() (log.SampleExtractor, error) { return nil, nil } +func (e *literalExpr) Selector() LogSelectorExpr { return e } +func (e *literalExpr) HasFilter() bool { return false } +func (e *literalExpr) Shardable() bool { return true } +func (e *literalExpr) Pipeline() (log.Pipeline, error) { return log.NewNoopPipeline(), nil } +func (e *literalExpr) Matchers() []*labels.Matcher { return nil } +func (e *literalExpr) Extractor() (log.SampleExtractor, error) { return nil, nil } // helper used to impl Stringer for vector and range aggregations // nolint:interfacer @@ -1075,7 +1037,7 @@ func formatOperation(op string, grouping *grouping, params ...string) string { return sb.String() } -type LabelReplaceExpr struct { +type labelReplaceExpr struct { left SampleExpr dst string replacement string @@ -1086,12 +1048,12 @@ type LabelReplaceExpr struct { implicit } -func mustNewLabelReplaceExpr(left SampleExpr, dst, replacement, src, regex string) *LabelReplaceExpr { +func mustNewLabelReplaceExpr(left SampleExpr, dst, replacement, src, regex string) *labelReplaceExpr { re, err := regexp.Compile("^(?:" + regex + ")$") if err != nil { panic(logqlmodel.NewParseError(fmt.Sprintf("invalid regex in label_replace: %s", err.Error()), 0, 0)) } - return &LabelReplaceExpr{ + return &labelReplaceExpr{ left: left, dst: dst, replacement: replacement, @@ -1101,27 +1063,19 @@ func mustNewLabelReplaceExpr(left SampleExpr, dst, replacement, src, regex strin } } -func (e *LabelReplaceExpr) Selector() LogSelectorExpr { +func (e *labelReplaceExpr) Selector() LogSelectorExpr { return e.left.Selector() } -func (e *LabelReplaceExpr) Extractor() (SampleExtractor, error) { +func (e *labelReplaceExpr) Extractor() (SampleExtractor, error) { return e.left.Extractor() } -func (e *LabelReplaceExpr) Shardable() bool { +func (e *labelReplaceExpr) Shardable() bool { return false } -func (e *LabelReplaceExpr) Walk(f WalkFn) { - f(e) - if e.left == nil { - return - } - e.left.Walk(f) -} - -func (e *LabelReplaceExpr) String() string { +func (e *labelReplaceExpr) String() string { var sb strings.Builder sb.WriteString(OpLabelReplace) sb.WriteString("(") diff --git a/pkg/logql/ast_test.go b/pkg/logql/ast_test.go index 750546d6391ed..e72731e380f0e 100644 --- a/pkg/logql/ast_test.go +++ b/pkg/logql/ast_test.go @@ -21,6 +21,8 @@ func Test_logSelectorExpr_String(t *testing.T) { {`{foo="bar", bar!="baz"} != "bip" !~ ".+bop"`, true}, {`{foo="bar"} |= "baz" |~ "blip" != "flip" !~ "flap"`, true}, {`{foo="bar", bar!="baz"} |= ""`, false}, + {`{foo="bar", bar!="baz"} |= "" |= ip("::1")`, true}, + {`{foo="bar", bar!="baz"} |= "" != ip("127.0.0.1")`, true}, {`{foo="bar", bar!="baz"} |~ ""`, false}, {`{foo="bar", bar!="baz"} |~ ".*"`, false}, {`{foo="bar", bar!="baz"} |= "" |= ""`, false}, diff --git a/pkg/logql/expr.y b/pkg/logql/expr.y index e48232b982597..a210da7f0b21c 100644 --- a/pkg/logql/expr.y +++ b/pkg/logql/expr.y @@ -15,7 +15,7 @@ import ( Grouping *grouping Labels []string LogExpr LogSelectorExpr - LogRangeExpr *LogRange + LogRangeExpr *logRange Matcher *labels.Matcher Matchers []*labels.Matcher RangeAggregationExpr SampleExpr @@ -25,16 +25,18 @@ import ( VectorAggregationExpr SampleExpr MetricExpr SampleExpr VectorOp string + FilterOp string BinOpExpr SampleExpr LabelReplaceExpr SampleExpr binOp string bytes uint64 str string duration time.Duration - LiteralExpr *LiteralExpr + LiteralExpr *literalExpr BinOpModifier BinOpOptions - LabelParser *LabelParserExpr - LineFilters *LineFilterExpr + LabelParser *labelParserExpr + LineFilters *lineFilterExpr + LineFilter *lineFilterExpr PipelineExpr MultiStageExpr PipelineStage StageExpr BytesFilter log.LabelFilterer @@ -50,8 +52,8 @@ import ( JSONExpressionParser *jsonExpressionParser JSONExpression log.JSONExpression JSONExpressionList []log.JSONExpression - UnwrapExpr *UnwrapExpr - OffsetExpr *OffsetExpr + UnwrapExpr *unwrapExpr + OffsetExpr *offsetExpr } %start root @@ -71,6 +73,7 @@ import ( %type selector %type vectorAggregationExpr %type vectorOp +%type filterOp %type binOpExpr %type literalExpr %type labelReplaceExpr @@ -83,6 +86,7 @@ import ( %type durationFilter %type labelFilter %type lineFilters +%type lineFilter %type lineFormatExpr %type labelFormatExpr %type labelFormat @@ -234,14 +238,22 @@ pipelineStage: lineFilters { $$ = $1 } | PIPE labelParser { $$ = $2 } | PIPE jsonExpressionParser { $$ = $2 } - | PIPE labelFilter { $$ = &LabelFilterExpr{LabelFilterer: $2 }} + | PIPE labelFilter { $$ = &labelFilterExpr{LabelFilterer: $2 }} | PIPE lineFormatExpr { $$ = $2 } | PIPE labelFormatExpr { $$ = $2 } ; +filterOp: + IP { $$ = OpFilterIP } + ; + +lineFilter: + filter STRING { $$ = newLineFilterExpr($1, "", $2,) } + | filter filterOp OPEN_PARENTHESIS STRING CLOSE_PARENTHESIS { $$ = newLineFilterExpr($1, $2, $4) } + lineFilters: - filter STRING { $$ = newLineFilterExpr(nil, $1, $2 ) } - | lineFilters filter STRING { $$ = newLineFilterExpr($1, $2, $3 ) } + lineFilter { $$ = $1 } + | lineFilters lineFilter { $$ = newNestedLineFilterExpr($1, $2) } labelParser: JSON { $$ = newLabelParserExpr(OpParserTypeJSON, "") } diff --git a/pkg/logql/expr.y.go b/pkg/logql/expr.y.go index 7ece14922eef7..9110b927da334 100644 --- a/pkg/logql/expr.y.go +++ b/pkg/logql/expr.y.go @@ -28,6 +28,7 @@ type exprSymType struct { VectorAggregationExpr SampleExpr MetricExpr SampleExpr VectorOp string + FilterOp string BinOpExpr SampleExpr LabelReplaceExpr SampleExpr binOp string @@ -38,6 +39,7 @@ type exprSymType struct { BinOpModifier BinOpOptions LabelParser *labelParserExpr LineFilters *lineFilterExpr + LineFilter *lineFilterExpr PipelineExpr MultiStageExpr PipelineStage StageExpr BytesFilter log.LabelFilterer @@ -232,123 +234,126 @@ var exprExca = [...]int{ const exprPrivate = 57344 -const exprLast = 500 +const exprLast = 502 var exprAct = [...]int{ - 234, 186, 75, 4, 168, 58, 156, 5, 161, 195, - 66, 109, 50, 57, 237, 130, 68, 2, 45, 46, - 47, 48, 49, 50, 71, 42, 43, 44, 51, 52, + 238, 189, 76, 4, 170, 58, 158, 5, 163, 198, + 67, 110, 50, 57, 241, 133, 69, 2, 45, 46, + 47, 48, 49, 50, 72, 42, 43, 44, 51, 52, 55, 56, 53, 54, 45, 46, 47, 48, 49, 50, 43, 44, 51, 52, 55, 56, 53, 54, 45, 46, - 47, 48, 49, 50, 47, 48, 49, 50, 126, 128, - 129, 64, 242, 97, 170, 128, 129, 101, 62, 63, - 215, 239, 179, 216, 214, 293, 290, 82, 134, 313, - 293, 132, 61, 271, 139, 51, 52, 55, 56, 53, - 54, 45, 46, 47, 48, 49, 50, 116, 140, 237, - 141, 142, 143, 144, 145, 146, 147, 148, 149, 150, - 151, 152, 153, 154, 116, 113, 127, 65, 239, 308, - 165, 176, 171, 174, 175, 172, 173, 74, 158, 76, - 77, 213, 113, 104, 106, 105, 177, 114, 115, 242, - 193, 189, 98, 76, 77, 240, 187, 238, 198, 190, - 64, 301, 300, 271, 107, 116, 108, 62, 63, 272, - 279, 182, 185, 298, 201, 202, 203, 64, 296, 158, - 238, 64, 281, 113, 62, 63, 157, 243, 62, 63, - 188, 262, 239, 263, 249, 278, 232, 235, 239, 241, - 116, 244, 132, 97, 247, 101, 248, 188, 191, 236, - 233, 188, 240, 245, 158, 239, 65, 64, 113, 274, - 275, 276, 257, 255, 62, 63, 159, 157, 182, 251, - 237, 185, 182, 65, 253, 251, 64, 65, 121, 120, - 252, 261, 138, 62, 63, 125, 264, 188, 266, 268, - 246, 270, 97, 197, 183, 64, 269, 280, 265, 197, - 64, 97, 62, 63, 282, 116, 188, 62, 63, 311, - 137, 131, 199, 65, 136, 80, 287, 288, 196, 12, - 73, 97, 289, 113, 123, 188, 307, 133, 291, 292, - 60, 284, 65, 211, 297, 178, 212, 210, 122, 15, - 12, 124, 250, 303, 267, 304, 305, 12, 133, 207, - 204, 65, 200, 192, 184, 6, 65, 309, 208, 19, - 20, 33, 34, 36, 37, 35, 38, 39, 40, 41, - 21, 22, 205, 230, 306, 227, 231, 229, 228, 226, - 23, 24, 25, 26, 27, 28, 29, 295, 294, 194, - 30, 31, 32, 18, 209, 224, 277, 12, 225, 223, - 221, 79, 78, 222, 220, 6, 312, 16, 17, 19, - 20, 33, 34, 36, 37, 35, 38, 39, 40, 41, - 21, 22, 218, 310, 302, 219, 217, 259, 260, 299, - 23, 24, 25, 26, 27, 28, 29, 3, 116, 135, - 30, 31, 32, 18, 67, 286, 285, 12, 258, 256, - 254, 169, 110, 181, 180, 6, 113, 16, 17, 19, - 20, 33, 34, 36, 37, 35, 38, 39, 40, 41, - 21, 22, 81, 179, 104, 106, 105, 178, 114, 115, - 23, 24, 25, 26, 27, 28, 29, 116, 111, 166, - 30, 31, 32, 18, 164, 107, 163, 108, 155, 119, - 70, 158, 283, 72, 162, 113, 206, 16, 17, 72, - 169, 160, 100, 167, 103, 102, 83, 84, 85, 86, - 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, - 59, 117, 112, 118, 99, 11, 10, 9, 14, 8, - 273, 13, 7, 69, 1, 0, 0, 0, 159, 157, + 47, 48, 49, 50, 47, 48, 49, 50, 172, 131, + 132, 246, 298, 98, 243, 244, 120, 102, 65, 298, + 65, 129, 131, 132, 83, 63, 64, 63, 64, 137, + 284, 241, 135, 77, 78, 142, 51, 52, 55, 56, + 53, 54, 45, 46, 47, 48, 49, 50, 191, 143, + 191, 144, 145, 146, 147, 148, 149, 150, 151, 152, + 153, 154, 155, 156, 157, 178, 173, 176, 177, 174, + 175, 167, 188, 65, 66, 122, 66, 65, 117, 130, + 63, 64, 185, 61, 63, 64, 276, 247, 242, 180, + 318, 313, 160, 196, 192, 75, 114, 77, 78, 190, + 244, 201, 193, 191, 268, 65, 218, 191, 182, 219, + 217, 306, 63, 64, 255, 188, 204, 205, 206, 257, + 65, 243, 241, 243, 65, 305, 12, 63, 64, 66, + 65, 63, 64, 66, 136, 191, 185, 63, 64, 236, + 239, 117, 245, 99, 248, 135, 98, 251, 102, 252, + 191, 200, 240, 237, 60, 160, 249, 117, 250, 114, + 295, 66, 185, 117, 276, 261, 259, 216, 303, 255, + 202, 160, 286, 117, 256, 114, 66, 160, 200, 301, + 66, 114, 209, 267, 186, 214, 66, 181, 215, 213, + 269, 114, 271, 273, 253, 275, 98, 199, 117, 243, + 274, 285, 270, 159, 277, 98, 194, 124, 287, 105, + 107, 106, 134, 115, 116, 246, 114, 266, 161, 159, + 12, 292, 293, 242, 161, 159, 98, 294, 136, 265, + 108, 123, 109, 296, 297, 179, 141, 128, 283, 302, + 140, 139, 81, 74, 15, 316, 212, 312, 308, 289, + 309, 310, 12, 254, 279, 280, 281, 210, 243, 207, + 6, 203, 314, 126, 19, 20, 33, 34, 36, 37, + 35, 38, 39, 40, 41, 21, 22, 125, 195, 187, + 127, 211, 208, 311, 300, 23, 24, 25, 26, 27, + 28, 29, 299, 282, 197, 30, 31, 32, 18, 272, + 233, 80, 12, 234, 232, 230, 79, 307, 231, 229, + 6, 317, 16, 17, 19, 20, 33, 34, 36, 37, + 35, 38, 39, 40, 41, 21, 22, 227, 315, 224, + 228, 226, 225, 223, 304, 23, 24, 25, 26, 27, + 28, 29, 3, 117, 138, 30, 31, 32, 18, 68, + 221, 291, 12, 222, 220, 263, 264, 290, 260, 258, + 6, 114, 16, 17, 19, 20, 33, 34, 36, 37, + 35, 38, 39, 40, 41, 21, 22, 82, 235, 105, + 107, 106, 184, 115, 116, 23, 24, 25, 26, 27, + 28, 29, 183, 182, 181, 30, 31, 32, 18, 168, + 108, 262, 109, 166, 171, 71, 165, 288, 73, 164, + 73, 171, 16, 17, 111, 112, 162, 101, 169, 104, + 103, 84, 85, 86, 87, 88, 89, 90, 91, 92, + 93, 94, 95, 96, 97, 59, 118, 113, 119, 100, + 11, 10, 9, 121, 14, 8, 278, 13, 7, 70, + 62, 1, } var exprPact = [...]int{ - 282, -1000, -41, -1000, -1000, 236, 282, -1000, -1000, -1000, - -1000, -1000, 448, 247, 104, -1000, 345, 344, 242, -1000, + 287, -1000, -41, -1000, -1000, 160, 287, -1000, -1000, -1000, + -1000, -1000, 453, 270, 122, -1000, 349, 344, 269, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, - -1000, -1000, 37, 37, 37, 37, 37, 37, 37, 37, - 37, 37, 37, 37, 37, 37, 37, 236, -1000, 47, - 383, 443, -1000, -1000, -1000, -1000, 205, 204, -41, 272, - 219, -1000, 46, 254, 382, 241, 237, 209, -1000, -1000, - 282, 282, -1000, 282, 282, 282, 282, 282, 282, 282, - 282, 282, 282, 282, 282, 282, 282, -1000, 442, -1000, - -1000, 150, -1000, -1000, 449, -1000, 440, -1000, 438, -1000, - -1000, -1000, -1000, 250, 433, 455, 52, -1000, -1000, -1000, - -1000, -1000, -1000, -1000, 454, -1000, 421, 417, 398, 397, - 220, 285, 212, 275, 174, 284, 332, 244, 238, 283, - -27, 16, 16, -23, -23, -68, -68, -68, -68, -57, - -57, -57, -57, -57, -57, -1000, 150, 250, 250, 250, - 281, -1000, 310, -1000, -1000, 432, -1000, 280, -1000, 296, - 279, 66, 368, 346, 341, 321, 319, -1000, -1000, -1000, - -1000, -1000, -1000, 118, 275, 157, 138, 193, 92, 153, - 216, 118, 282, 160, 273, 206, -1000, -1000, 200, -1000, - 394, 185, 150, 109, 449, 393, -1000, 396, 372, 208, - -1000, -1000, -1000, 158, -1000, -1000, -1000, -1000, -1000, -1000, + -1000, -1000, 34, 34, 34, 34, 34, 34, 34, 34, + 34, 34, 34, 34, 34, 34, 34, 160, -1000, 166, + 388, -1000, 60, -1000, -1000, -1000, -1000, 257, 233, -41, + 311, 271, -1000, 59, 255, 387, 268, 267, 263, -1000, + -1000, 287, 287, -1000, 287, 287, 287, 287, 287, 287, + 287, 287, 287, 287, 287, 287, 287, 287, -1000, -1000, + -1000, -1000, 202, -1000, -1000, 454, -1000, 450, -1000, 447, + -1000, -1000, -1000, -1000, 243, 443, 456, 46, -1000, -1000, + -1000, 262, -1000, -1000, -1000, -1000, -1000, 455, -1000, 438, + 437, 436, 426, 210, 310, 156, 161, 232, 309, 337, + 223, 196, 292, -27, 17, 17, -23, -23, -68, -68, + -68, -68, -57, -57, -57, -57, -57, -57, 202, 243, + 243, 243, 290, -1000, 320, -1000, -1000, 208, -1000, 288, + -1000, 319, 231, 152, 396, 375, 373, 351, 346, 422, + -1000, -1000, -1000, -1000, -1000, -1000, 58, 161, 109, 129, + 141, 218, 113, 184, 58, 287, 220, 284, 200, -1000, + -1000, 145, -1000, 403, 123, 202, 186, 454, 402, -1000, + 449, 400, 256, -1000, -1000, -1000, 244, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, -1000, - -1000, -1000, -1000, 159, 231, 27, 231, 286, -49, 250, - -49, 74, 154, 337, 161, 136, -1000, -1000, 148, -1000, - 282, 447, -1000, -1000, 262, -1000, -1000, -1000, -1000, -1000, - -1000, 390, 389, 118, 27, 231, 27, -1000, -1000, 150, - -1000, -49, -1000, 53, -1000, -1000, -1000, 36, 329, 328, - 144, 118, 139, -1000, 373, 128, 127, -1000, 27, -1000, - 369, 31, 27, 15, -49, -49, 315, -1000, -1000, 257, - -1000, -1000, 95, 27, -1000, -1000, -49, 367, -1000, -1000, - 240, 350, 55, -1000, + -1000, -1000, -1000, -1000, -1000, 209, -1000, 130, 54, 20, + 54, 341, -49, 243, -49, 127, 249, 334, 264, 56, + -1000, -1000, 198, -1000, 287, 452, -1000, -1000, 280, -1000, + -1000, -1000, -1000, -1000, -1000, 401, 395, -1000, 58, 20, + 54, 20, -1000, -1000, 202, -1000, -49, -1000, 187, -1000, + -1000, -1000, 18, 333, 325, 205, 58, 194, -1000, 378, + 151, 137, -1000, 20, -1000, 352, 25, 20, 14, -49, + -49, 324, -1000, -1000, 278, -1000, -1000, 117, 20, -1000, + -1000, -49, 372, -1000, -1000, 276, 355, 116, -1000, } var exprPgo = [...]int{ - 0, 494, 16, 82, 2, 9, 387, 3, 15, 11, - 493, 492, 491, 490, 7, 489, 488, 487, 486, 485, - 422, 484, 13, 5, 483, 482, 481, 6, 480, 465, - 464, 4, 463, 462, 8, 461, 1, 438, 402, 0, + 0, 501, 16, 500, 2, 9, 392, 3, 15, 11, + 499, 498, 497, 496, 7, 495, 494, 493, 492, 491, + 490, 427, 489, 13, 5, 488, 487, 486, 6, 485, + 133, 470, 469, 4, 468, 467, 8, 466, 1, 465, + 464, 0, } var exprR1 = [...]int{ 0, 1, 2, 2, 7, 7, 7, 7, 7, 7, 6, 6, 6, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, 8, - 8, 8, 8, 8, 8, 8, 8, 8, 8, 36, - 36, 36, 13, 13, 13, 11, 11, 11, 11, 15, - 15, 15, 15, 15, 15, 19, 3, 3, 3, 3, - 14, 14, 14, 10, 10, 9, 9, 9, 9, 22, - 22, 23, 23, 23, 23, 23, 23, 28, 28, 21, - 21, 21, 21, 21, 33, 29, 31, 31, 32, 32, - 32, 30, 27, 27, 27, 27, 27, 27, 27, 27, - 27, 34, 35, 35, 38, 38, 37, 37, 26, 26, - 26, 26, 26, 26, 26, 24, 24, 24, 24, 24, - 24, 24, 25, 25, 25, 25, 25, 25, 25, 17, - 17, 17, 17, 17, 17, 17, 17, 17, 17, 17, - 17, 17, 17, 17, 20, 20, 18, 18, 18, 16, - 16, 16, 16, 16, 16, 16, 16, 16, 12, 12, - 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, - 12, 12, 39, 5, 5, 4, 4, 4, 4, + 8, 8, 8, 8, 8, 8, 8, 8, 8, 38, + 38, 38, 13, 13, 13, 11, 11, 11, 11, 15, + 15, 15, 15, 15, 15, 20, 3, 3, 3, 3, + 14, 14, 14, 10, 10, 9, 9, 9, 9, 23, + 23, 24, 24, 24, 24, 24, 24, 17, 30, 30, + 29, 29, 22, 22, 22, 22, 22, 35, 31, 33, + 33, 34, 34, 34, 32, 28, 28, 28, 28, 28, + 28, 28, 28, 28, 36, 37, 37, 40, 40, 39, + 39, 27, 27, 27, 27, 27, 27, 27, 25, 25, + 25, 25, 25, 25, 25, 26, 26, 26, 26, 26, + 26, 26, 18, 18, 18, 18, 18, 18, 18, 18, + 18, 18, 18, 18, 18, 18, 18, 21, 21, 19, + 19, 19, 16, 16, 16, 16, 16, 16, 16, 16, + 16, 12, 12, 12, 12, 12, 12, 12, 12, 12, + 12, 12, 12, 12, 12, 41, 5, 5, 4, 4, + 4, 4, } var exprR2 = [...]int{ @@ -359,87 +364,88 @@ var exprR2 = [...]int{ 6, 3, 1, 1, 1, 4, 6, 5, 7, 4, 5, 5, 6, 7, 7, 12, 1, 1, 1, 1, 3, 3, 3, 1, 3, 3, 3, 3, 3, 1, - 2, 1, 2, 2, 2, 2, 2, 2, 3, 1, - 1, 2, 1, 2, 2, 2, 3, 3, 1, 3, - 3, 2, 1, 1, 1, 1, 3, 2, 3, 3, - 3, 3, 1, 3, 6, 6, 1, 1, 3, 3, + 2, 1, 2, 2, 2, 2, 2, 1, 2, 5, + 1, 2, 1, 1, 2, 1, 2, 2, 2, 3, + 3, 1, 3, 3, 2, 1, 1, 1, 1, 3, + 2, 3, 3, 3, 3, 1, 3, 6, 6, 1, + 1, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, - 3, 3, 3, 3, 3, 3, 3, 3, 3, 4, - 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, - 4, 4, 4, 4, 0, 1, 1, 2, 2, 1, + 3, 3, 4, 4, 4, 4, 4, 4, 4, 4, + 4, 4, 4, 4, 4, 4, 4, 0, 1, 1, + 2, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, - 1, 1, 2, 1, 3, 4, 4, 3, 3, + 1, 1, 1, 1, 1, 2, 1, 3, 4, 4, + 3, 3, } var exprChk = [...]int{ - -1000, -1, -2, -6, -7, -14, 23, -11, -15, -17, - -18, -19, 15, -12, -16, 7, 75, 76, 61, 27, + -1000, -1, -2, -6, -7, -14, 23, -11, -15, -18, + -19, -20, 15, -12, -16, 7, 75, 76, 61, 27, 28, 38, 39, 48, 49, 50, 51, 52, 53, 54, 58, 59, 60, 29, 30, 33, 31, 32, 34, 35, 36, 37, 66, 67, 68, 75, 76, 77, 78, 79, - 80, 69, 70, 73, 74, 71, 72, -22, -23, -28, - 44, -3, 21, 22, 14, 70, -7, -6, -2, -10, - 2, -9, 5, 23, 23, -4, 25, 26, 7, 7, - 23, -20, 40, -20, -20, -20, -20, -20, -20, -20, - -20, -20, -20, -20, -20, -20, -20, -23, -3, -21, - -33, -27, -29, -30, 41, 43, 42, 62, 64, -9, - -38, -37, -25, 23, 45, 46, 5, -26, -24, 6, - 24, 24, 16, 2, 19, 16, 12, 70, 13, 14, - -8, 7, -14, 23, -7, 7, 23, 23, 23, -7, - -2, -2, -2, -2, -2, -2, -2, -2, -2, -2, - -2, -2, -2, -2, -2, 6, -27, 67, 19, 66, - -35, -34, 5, 6, 6, -27, 6, -32, -31, 5, - 12, 70, 73, 74, 71, 72, 69, -9, 6, 6, - 6, 6, 2, 24, 19, 9, -36, -22, 44, -14, - -8, 24, 19, -7, 7, -5, 24, 5, -5, 24, - 19, -27, -27, -27, 19, 12, 24, 19, 12, 65, - 8, 4, 7, 65, 8, 4, 7, 8, 4, 7, + 80, 69, 70, 73, 74, 71, 72, -23, -24, -29, + 44, -30, -3, 21, 22, 14, 70, -7, -6, -2, + -10, 2, -9, 5, 23, 23, -4, 25, 26, 7, + 7, 23, -21, 40, -21, -21, -21, -21, -21, -21, + -21, -21, -21, -21, -21, -21, -21, -21, -24, -30, + -22, -35, -28, -31, -32, 41, 43, 42, 62, 64, + -9, -40, -39, -26, 23, 45, 46, 5, -27, -25, + 6, -17, 65, 24, 24, 16, 2, 19, 16, 12, + 70, 13, 14, -8, 7, -14, 23, -7, 7, 23, + 23, 23, -7, -2, -2, -2, -2, -2, -2, -2, + -2, -2, -2, -2, -2, -2, -2, -2, -28, 67, + 19, 66, -37, -36, 5, 6, 6, -28, 6, -34, + -33, 5, 12, 70, 73, 74, 71, 72, 69, 23, + -9, 6, 6, 6, 6, 2, 24, 19, 9, -38, + -23, 44, -14, -8, 24, 19, -7, 7, -5, 24, + 5, -5, 24, 19, -28, -28, -28, 19, 12, 24, + 19, 12, 65, 8, 4, 7, 65, 8, 4, 7, 8, 4, 7, 8, 4, 7, 8, 4, 7, 8, - 4, 7, -4, -8, -39, -36, -22, 63, 9, 44, - 9, -36, 47, 24, -36, -22, 24, -4, -7, 24, - 19, 19, 24, 24, 6, -34, 6, -31, 2, 5, - 6, 23, 23, 24, -36, -22, -36, 8, -39, -27, - -39, 9, 5, -13, 55, 56, 57, 9, 24, 24, - -36, 24, -7, 5, 19, 6, 6, -4, -36, -39, - 23, -39, -36, 44, 9, 9, 24, -4, 24, 6, - 24, 24, 5, -36, -39, -39, 9, 19, 24, -39, - 6, 19, 6, 24, + 4, 7, 8, 4, 7, 6, -4, -8, -41, -38, + -23, 63, 9, 44, 9, -38, 47, 24, -38, -23, + 24, -4, -7, 24, 19, 19, 24, 24, 6, -36, + 6, -33, 2, 5, 6, 23, 23, 24, 24, -38, + -23, -38, 8, -41, -28, -41, 9, 5, -13, 55, + 56, 57, 9, 24, 24, -38, 24, -7, 5, 19, + 6, 6, -4, -38, -41, 23, -41, -38, 44, 9, + 9, 24, -4, 24, 6, 24, 24, 5, -38, -41, + -41, 9, 19, 24, -41, 6, 19, 6, 24, } var exprDef = [...]int{ 0, -2, 1, 2, 3, 10, 0, 4, 5, 6, - 7, 8, 0, 0, 0, 146, 0, 0, 0, 158, - 159, 160, 161, 162, 163, 164, 165, 166, 167, 168, - 169, 170, 171, 149, 150, 151, 152, 153, 154, 155, - 156, 157, 144, 144, 144, 144, 144, 144, 144, 144, - 144, 144, 144, 144, 144, 144, 144, 11, 69, 71, - 0, 0, 56, 57, 58, 59, 3, 2, 0, 0, - 0, 63, 0, 0, 0, 0, 0, 0, 147, 148, - 0, 0, 145, 0, 0, 0, 0, 0, 0, 0, - 0, 0, 0, 0, 0, 0, 0, 70, 0, 72, - 73, 74, 75, 76, 79, 80, 0, 82, 0, 92, - 93, 94, 95, 0, 0, 0, 0, 106, 107, 77, - 9, 12, 60, 61, 0, 62, 0, 0, 0, 0, - 0, 0, 0, 0, 3, 146, 0, 0, 0, 3, - 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, - 139, 140, 141, 142, 143, 78, 97, 0, 0, 0, - 84, 102, 0, 81, 83, 0, 85, 91, 88, 0, - 0, 0, 0, 0, 0, 0, 0, 64, 65, 66, - 67, 68, 38, 45, 0, 13, 0, 0, 0, 0, - 0, 49, 0, 3, 146, 0, 177, 173, 0, 178, - 0, 98, 99, 100, 0, 0, 96, 0, 0, 0, - 113, 120, 127, 0, 112, 119, 126, 108, 115, 122, - 109, 116, 123, 110, 117, 124, 111, 118, 125, 114, - 121, 128, 47, 0, 14, 17, 33, 0, 21, 0, - 25, 0, 0, 0, 0, 0, 37, 51, 3, 50, - 0, 0, 175, 176, 0, 103, 101, 89, 90, 86, - 87, 0, 0, 46, 18, 34, 35, 172, 22, 41, - 26, 29, 39, 0, 42, 43, 44, 15, 0, 0, - 0, 52, 3, 174, 0, 0, 0, 48, 36, 30, - 0, 16, 19, 0, 23, 27, 0, 53, 54, 0, - 104, 105, 0, 20, 24, 28, 31, 0, 40, 32, - 0, 0, 0, 55, + 7, 8, 0, 0, 0, 149, 0, 0, 0, 161, + 162, 163, 164, 165, 166, 167, 168, 169, 170, 171, + 172, 173, 174, 152, 153, 154, 155, 156, 157, 158, + 159, 160, 147, 147, 147, 147, 147, 147, 147, 147, + 147, 147, 147, 147, 147, 147, 147, 11, 69, 71, + 0, 80, 0, 56, 57, 58, 59, 3, 2, 0, + 0, 0, 63, 0, 0, 0, 0, 0, 0, 150, + 151, 0, 0, 148, 0, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 70, 81, + 72, 73, 74, 75, 76, 82, 83, 0, 85, 0, + 95, 96, 97, 98, 0, 0, 0, 0, 109, 110, + 78, 0, 77, 9, 12, 60, 61, 0, 62, 0, + 0, 0, 0, 0, 0, 0, 0, 3, 149, 0, + 0, 0, 3, 132, 133, 134, 135, 136, 137, 138, + 139, 140, 141, 142, 143, 144, 145, 146, 100, 0, + 0, 0, 87, 105, 0, 84, 86, 0, 88, 94, + 91, 0, 0, 0, 0, 0, 0, 0, 0, 0, + 64, 65, 66, 67, 68, 38, 45, 0, 13, 0, + 0, 0, 0, 0, 49, 0, 3, 149, 0, 180, + 176, 0, 181, 0, 101, 102, 103, 0, 0, 99, + 0, 0, 0, 116, 123, 130, 0, 115, 122, 129, + 111, 118, 125, 112, 119, 126, 113, 120, 127, 114, + 121, 128, 117, 124, 131, 0, 47, 0, 14, 17, + 33, 0, 21, 0, 25, 0, 0, 0, 0, 0, + 37, 51, 3, 50, 0, 0, 178, 179, 0, 106, + 104, 92, 93, 89, 90, 0, 0, 79, 46, 18, + 34, 35, 175, 22, 41, 26, 29, 39, 0, 42, + 43, 44, 15, 0, 0, 0, 52, 3, 177, 0, + 0, 0, 48, 36, 30, 0, 16, 19, 0, 23, + 27, 0, 53, 54, 0, 107, 108, 0, 20, 24, + 28, 31, 0, 40, 32, 0, 0, 0, 55, } var exprTok1 = [...]int{ @@ -1171,506 +1177,521 @@ exprdefault: exprVAL.PipelineStage = exprDollar[2].LabelFormatExpr } case 77: - exprDollar = exprS[exprpt-2 : exprpt+1] + exprDollar = exprS[exprpt-1 : exprpt+1] { - exprVAL.LineFilters = newLineFilterExpr(nil, exprDollar[1].Filter, exprDollar[2].str) + exprVAL.FilterOp = OpFilterIP } case 78: - exprDollar = exprS[exprpt-3 : exprpt+1] + exprDollar = exprS[exprpt-2 : exprpt+1] { - exprVAL.LineFilters = newLineFilterExpr(exprDollar[1].LineFilters, exprDollar[2].Filter, exprDollar[3].str) + exprVAL.LineFilter = newLineFilterExpr(exprDollar[1].Filter, "", exprDollar[2].str) } case 79: + exprDollar = exprS[exprpt-5 : exprpt+1] + { + exprVAL.LineFilter = newLineFilterExpr(exprDollar[1].Filter, exprDollar[2].FilterOp, exprDollar[4].str) + } + case 80: + exprDollar = exprS[exprpt-1 : exprpt+1] + { + exprVAL.LineFilters = exprDollar[1].LineFilter + } + case 81: + exprDollar = exprS[exprpt-2 : exprpt+1] + { + exprVAL.LineFilters = newNestedLineFilterExpr(exprDollar[1].LineFilters, exprDollar[2].LineFilter) + } + case 82: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.LabelParser = newLabelParserExpr(OpParserTypeJSON, "") } - case 80: + case 83: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.LabelParser = newLabelParserExpr(OpParserTypeLogfmt, "") } - case 81: + case 84: exprDollar = exprS[exprpt-2 : exprpt+1] { exprVAL.LabelParser = newLabelParserExpr(OpParserTypeRegexp, exprDollar[2].str) } - case 82: + case 85: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.LabelParser = newLabelParserExpr(OpParserTypeUnpack, "") } - case 83: + case 86: exprDollar = exprS[exprpt-2 : exprpt+1] { exprVAL.LabelParser = newLabelParserExpr(OpParserTypePattern, exprDollar[2].str) } - case 84: + case 87: exprDollar = exprS[exprpt-2 : exprpt+1] { exprVAL.JSONExpressionParser = newJSONExpressionParser(exprDollar[2].JSONExpressionList) } - case 85: + case 88: exprDollar = exprS[exprpt-2 : exprpt+1] { exprVAL.LineFormatExpr = newLineFmtExpr(exprDollar[2].str) } - case 86: + case 89: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.LabelFormat = log.NewRenameLabelFmt(exprDollar[1].str, exprDollar[3].str) } - case 87: + case 90: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.LabelFormat = log.NewTemplateLabelFmt(exprDollar[1].str, exprDollar[3].str) } - case 88: + case 91: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.LabelsFormat = []log.LabelFmt{exprDollar[1].LabelFormat} } - case 89: + case 92: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.LabelsFormat = append(exprDollar[1].LabelsFormat, exprDollar[3].LabelFormat) } - case 91: + case 94: exprDollar = exprS[exprpt-2 : exprpt+1] { exprVAL.LabelFormatExpr = newLabelFmtExpr(exprDollar[2].LabelsFormat) } - case 92: + case 95: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.LabelFilter = log.NewStringLabelFilter(exprDollar[1].Matcher) } - case 93: + case 96: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.LabelFilter = exprDollar[1].IPLabelFilter } - case 94: + case 97: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.LabelFilter = exprDollar[1].UnitFilter } - case 95: + case 98: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.LabelFilter = exprDollar[1].NumberFilter } - case 96: + case 99: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.LabelFilter = exprDollar[2].LabelFilter } - case 97: + case 100: exprDollar = exprS[exprpt-2 : exprpt+1] { exprVAL.LabelFilter = log.NewAndLabelFilter(exprDollar[1].LabelFilter, exprDollar[2].LabelFilter) } - case 98: + case 101: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.LabelFilter = log.NewAndLabelFilter(exprDollar[1].LabelFilter, exprDollar[3].LabelFilter) } - case 99: + case 102: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.LabelFilter = log.NewAndLabelFilter(exprDollar[1].LabelFilter, exprDollar[3].LabelFilter) } - case 100: + case 103: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.LabelFilter = log.NewOrLabelFilter(exprDollar[1].LabelFilter, exprDollar[3].LabelFilter) } - case 101: + case 104: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.JSONExpression = log.NewJSONExpr(exprDollar[1].str, exprDollar[3].str) } - case 102: + case 105: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.JSONExpressionList = []log.JSONExpression{exprDollar[1].JSONExpression} } - case 103: + case 106: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.JSONExpressionList = append(exprDollar[1].JSONExpressionList, exprDollar[3].JSONExpression) } - case 104: + case 107: exprDollar = exprS[exprpt-6 : exprpt+1] { exprVAL.IPLabelFilter = log.NewIPLabelFilter(exprDollar[1].str, log.LabelFilterEqual, exprDollar[5].str) } - case 105: + case 108: exprDollar = exprS[exprpt-6 : exprpt+1] { exprVAL.IPLabelFilter = log.NewIPLabelFilter(exprDollar[1].str, log.LabelFilterNotEqual, exprDollar[5].str) } - case 106: + case 109: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.UnitFilter = exprDollar[1].DurationFilter } - case 107: + case 110: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.UnitFilter = exprDollar[1].BytesFilter } - case 108: + case 111: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.DurationFilter = log.NewDurationLabelFilter(log.LabelFilterGreaterThan, exprDollar[1].str, exprDollar[3].duration) } - case 109: + case 112: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.DurationFilter = log.NewDurationLabelFilter(log.LabelFilterGreaterThanOrEqual, exprDollar[1].str, exprDollar[3].duration) } - case 110: + case 113: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.DurationFilter = log.NewDurationLabelFilter(log.LabelFilterLesserThan, exprDollar[1].str, exprDollar[3].duration) } - case 111: + case 114: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.DurationFilter = log.NewDurationLabelFilter(log.LabelFilterLesserThanOrEqual, exprDollar[1].str, exprDollar[3].duration) } - case 112: + case 115: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.DurationFilter = log.NewDurationLabelFilter(log.LabelFilterNotEqual, exprDollar[1].str, exprDollar[3].duration) } - case 113: + case 116: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.DurationFilter = log.NewDurationLabelFilter(log.LabelFilterEqual, exprDollar[1].str, exprDollar[3].duration) } - case 114: + case 117: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.DurationFilter = log.NewDurationLabelFilter(log.LabelFilterEqual, exprDollar[1].str, exprDollar[3].duration) } - case 115: + case 118: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.BytesFilter = log.NewBytesLabelFilter(log.LabelFilterGreaterThan, exprDollar[1].str, exprDollar[3].bytes) } - case 116: + case 119: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.BytesFilter = log.NewBytesLabelFilter(log.LabelFilterGreaterThanOrEqual, exprDollar[1].str, exprDollar[3].bytes) } - case 117: + case 120: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.BytesFilter = log.NewBytesLabelFilter(log.LabelFilterLesserThan, exprDollar[1].str, exprDollar[3].bytes) } - case 118: + case 121: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.BytesFilter = log.NewBytesLabelFilter(log.LabelFilterLesserThanOrEqual, exprDollar[1].str, exprDollar[3].bytes) } - case 119: + case 122: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.BytesFilter = log.NewBytesLabelFilter(log.LabelFilterNotEqual, exprDollar[1].str, exprDollar[3].bytes) } - case 120: + case 123: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.BytesFilter = log.NewBytesLabelFilter(log.LabelFilterEqual, exprDollar[1].str, exprDollar[3].bytes) } - case 121: + case 124: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.BytesFilter = log.NewBytesLabelFilter(log.LabelFilterEqual, exprDollar[1].str, exprDollar[3].bytes) } - case 122: + case 125: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.NumberFilter = log.NewNumericLabelFilter(log.LabelFilterGreaterThan, exprDollar[1].str, mustNewFloat(exprDollar[3].str)) } - case 123: + case 126: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.NumberFilter = log.NewNumericLabelFilter(log.LabelFilterGreaterThanOrEqual, exprDollar[1].str, mustNewFloat(exprDollar[3].str)) } - case 124: + case 127: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.NumberFilter = log.NewNumericLabelFilter(log.LabelFilterLesserThan, exprDollar[1].str, mustNewFloat(exprDollar[3].str)) } - case 125: + case 128: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.NumberFilter = log.NewNumericLabelFilter(log.LabelFilterLesserThanOrEqual, exprDollar[1].str, mustNewFloat(exprDollar[3].str)) } - case 126: + case 129: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.NumberFilter = log.NewNumericLabelFilter(log.LabelFilterNotEqual, exprDollar[1].str, mustNewFloat(exprDollar[3].str)) } - case 127: + case 130: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.NumberFilter = log.NewNumericLabelFilter(log.LabelFilterEqual, exprDollar[1].str, mustNewFloat(exprDollar[3].str)) } - case 128: + case 131: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.NumberFilter = log.NewNumericLabelFilter(log.LabelFilterEqual, exprDollar[1].str, mustNewFloat(exprDollar[3].str)) } - case 129: + case 132: exprDollar = exprS[exprpt-4 : exprpt+1] { exprVAL.BinOpExpr = mustNewBinOpExpr("or", exprDollar[3].BinOpModifier, exprDollar[1].Expr, exprDollar[4].Expr) } - case 130: + case 133: exprDollar = exprS[exprpt-4 : exprpt+1] { exprVAL.BinOpExpr = mustNewBinOpExpr("and", exprDollar[3].BinOpModifier, exprDollar[1].Expr, exprDollar[4].Expr) } - case 131: + case 134: exprDollar = exprS[exprpt-4 : exprpt+1] { exprVAL.BinOpExpr = mustNewBinOpExpr("unless", exprDollar[3].BinOpModifier, exprDollar[1].Expr, exprDollar[4].Expr) } - case 132: + case 135: exprDollar = exprS[exprpt-4 : exprpt+1] { exprVAL.BinOpExpr = mustNewBinOpExpr("+", exprDollar[3].BinOpModifier, exprDollar[1].Expr, exprDollar[4].Expr) } - case 133: + case 136: exprDollar = exprS[exprpt-4 : exprpt+1] { exprVAL.BinOpExpr = mustNewBinOpExpr("-", exprDollar[3].BinOpModifier, exprDollar[1].Expr, exprDollar[4].Expr) } - case 134: + case 137: exprDollar = exprS[exprpt-4 : exprpt+1] { exprVAL.BinOpExpr = mustNewBinOpExpr("*", exprDollar[3].BinOpModifier, exprDollar[1].Expr, exprDollar[4].Expr) } - case 135: + case 138: exprDollar = exprS[exprpt-4 : exprpt+1] { exprVAL.BinOpExpr = mustNewBinOpExpr("/", exprDollar[3].BinOpModifier, exprDollar[1].Expr, exprDollar[4].Expr) } - case 136: + case 139: exprDollar = exprS[exprpt-4 : exprpt+1] { exprVAL.BinOpExpr = mustNewBinOpExpr("%", exprDollar[3].BinOpModifier, exprDollar[1].Expr, exprDollar[4].Expr) } - case 137: + case 140: exprDollar = exprS[exprpt-4 : exprpt+1] { exprVAL.BinOpExpr = mustNewBinOpExpr("^", exprDollar[3].BinOpModifier, exprDollar[1].Expr, exprDollar[4].Expr) } - case 138: + case 141: exprDollar = exprS[exprpt-4 : exprpt+1] { exprVAL.BinOpExpr = mustNewBinOpExpr("==", exprDollar[3].BinOpModifier, exprDollar[1].Expr, exprDollar[4].Expr) } - case 139: + case 142: exprDollar = exprS[exprpt-4 : exprpt+1] { exprVAL.BinOpExpr = mustNewBinOpExpr("!=", exprDollar[3].BinOpModifier, exprDollar[1].Expr, exprDollar[4].Expr) } - case 140: + case 143: exprDollar = exprS[exprpt-4 : exprpt+1] { exprVAL.BinOpExpr = mustNewBinOpExpr(">", exprDollar[3].BinOpModifier, exprDollar[1].Expr, exprDollar[4].Expr) } - case 141: + case 144: exprDollar = exprS[exprpt-4 : exprpt+1] { exprVAL.BinOpExpr = mustNewBinOpExpr(">=", exprDollar[3].BinOpModifier, exprDollar[1].Expr, exprDollar[4].Expr) } - case 142: + case 145: exprDollar = exprS[exprpt-4 : exprpt+1] { exprVAL.BinOpExpr = mustNewBinOpExpr("<", exprDollar[3].BinOpModifier, exprDollar[1].Expr, exprDollar[4].Expr) } - case 143: + case 146: exprDollar = exprS[exprpt-4 : exprpt+1] { exprVAL.BinOpExpr = mustNewBinOpExpr("<=", exprDollar[3].BinOpModifier, exprDollar[1].Expr, exprDollar[4].Expr) } - case 144: + case 147: exprDollar = exprS[exprpt-0 : exprpt+1] { exprVAL.BinOpModifier = BinOpOptions{} } - case 145: + case 148: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.BinOpModifier = BinOpOptions{ReturnBool: true} } - case 146: + case 149: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.LiteralExpr = mustNewLiteralExpr(exprDollar[1].str, false) } - case 147: + case 150: exprDollar = exprS[exprpt-2 : exprpt+1] { exprVAL.LiteralExpr = mustNewLiteralExpr(exprDollar[2].str, false) } - case 148: + case 151: exprDollar = exprS[exprpt-2 : exprpt+1] { exprVAL.LiteralExpr = mustNewLiteralExpr(exprDollar[2].str, true) } - case 149: + case 152: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.VectorOp = OpTypeSum } - case 150: + case 153: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.VectorOp = OpTypeAvg } - case 151: + case 154: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.VectorOp = OpTypeCount } - case 152: + case 155: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.VectorOp = OpTypeMax } - case 153: + case 156: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.VectorOp = OpTypeMin } - case 154: + case 157: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.VectorOp = OpTypeStddev } - case 155: + case 158: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.VectorOp = OpTypeStdvar } - case 156: + case 159: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.VectorOp = OpTypeBottomK } - case 157: + case 160: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.VectorOp = OpTypeTopK } - case 158: + case 161: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.RangeOp = OpRangeTypeCount } - case 159: + case 162: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.RangeOp = OpRangeTypeRate } - case 160: + case 163: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.RangeOp = OpRangeTypeBytes } - case 161: + case 164: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.RangeOp = OpRangeTypeBytesRate } - case 162: + case 165: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.RangeOp = OpRangeTypeAvg } - case 163: + case 166: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.RangeOp = OpRangeTypeSum } - case 164: + case 167: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.RangeOp = OpRangeTypeMin } - case 165: + case 168: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.RangeOp = OpRangeTypeMax } - case 166: + case 169: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.RangeOp = OpRangeTypeStdvar } - case 167: + case 170: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.RangeOp = OpRangeTypeStddev } - case 168: + case 171: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.RangeOp = OpRangeTypeQuantile } - case 169: + case 172: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.RangeOp = OpRangeTypeFirst } - case 170: + case 173: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.RangeOp = OpRangeTypeLast } - case 171: + case 174: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.RangeOp = OpRangeTypeAbsent } - case 172: + case 175: exprDollar = exprS[exprpt-2 : exprpt+1] { exprVAL.OffsetExpr = newOffsetExpr(exprDollar[2].duration) } - case 173: + case 176: exprDollar = exprS[exprpt-1 : exprpt+1] { exprVAL.Labels = []string{exprDollar[1].str} } - case 174: + case 177: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.Labels = append(exprDollar[1].Labels, exprDollar[3].str) } - case 175: + case 178: exprDollar = exprS[exprpt-4 : exprpt+1] { exprVAL.Grouping = &grouping{without: false, groups: exprDollar[3].Labels} } - case 176: + case 179: exprDollar = exprS[exprpt-4 : exprpt+1] { exprVAL.Grouping = &grouping{without: true, groups: exprDollar[3].Labels} } - case 177: + case 180: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.Grouping = &grouping{without: false, groups: nil} } - case 178: + case 181: exprDollar = exprS[exprpt-3 : exprpt+1] { exprVAL.Grouping = &grouping{without: true, groups: nil} diff --git a/pkg/logql/log/ip.go b/pkg/logql/log/ip.go index c0036b250961d..87011c6314172 100644 --- a/pkg/logql/log/ip.go +++ b/pkg/logql/log/ip.go @@ -9,8 +9,8 @@ import ( ) var ( - errIPFilterInvalidPattern = errors.New("ip: invalid pattern") - errIPFilterInvalidOperation = errors.New("ip: invalid operation") + ErrIPFilterInvalidPattern = errors.New("ip: invalid pattern") + ErrIPFilterInvalidOperation = errors.New("ip: invalid operation") ) type IPMatchType int @@ -26,6 +26,13 @@ const ( // Should be one of the netaddr.IP, netaddr.IPRange, netadd.IPPrefix. type IPMatcher interface{} +// IPFilter search for IP addresses of given `pattern` in the given `line`. +// It returns true if pattern is matched with at least one IP in the `line` + +// pattern - can be of the following form for both IPv4 and IPv6. +// 1. SINGLE-IP - "192.168.0.1" +// 2. IP RANGE - "192.168.0.1-192.168.0.23" +// 3. CIDR - "192.168.0.0/16" type IPFilter struct { pattern string matcher IPMatcher @@ -42,13 +49,6 @@ type IPFilter struct { labelOp LabelFilterType } -// IPFilter search for IP addresses of given `pattern` in the given `line`. -// It returns true if pattern is matched with at least one IP in the `line` - -// pattern - can be of the following form for both IPv4 and IPv6. -// 1. SINGLE-IP - "192.168.0.1" -// 2. IP RANGE - "192.168.0.1-192.168.0.23" -// 3. CIDR - "192.168.0.0/16" func newIPFilter(pattern string) *IPFilter { filter := &IPFilter{pattern: pattern} @@ -59,6 +59,12 @@ func newIPFilter(pattern string) *IPFilter { return filter } +// NewIPLineFilter is used to construct ip filter as a `LineFilter` +func NewIPLineFilter(pattern string) *IPFilter { + return newIPFilter(pattern) +} + +// NewIPLabelFilter is used to construct ip filter as label filter for the given `label`. func NewIPLabelFilter(label string, op LabelFilterType, pattern string) *IPFilter { filter := newIPFilter(pattern) filter.labelName = label @@ -68,6 +74,7 @@ func NewIPLabelFilter(label string, op LabelFilterType, pattern string) *IPFilte } // filter does the heavy lifting finding ip `pattern` in the givin `line`. +// This is the function if you want to understand how the core logic how ip filter works! func (ipf *IPFilter) filter(line []byte) bool { if len(line) == 0 { return false @@ -113,6 +120,16 @@ func (ipf *IPFilter) filter(line []byte) bool { return false } +// Filter implement `Filterer` interface. +func (ipf *IPFilter) Filter(line []byte) bool { + return ipf.filter(line) +} + +// ToStage implements `Filterer` interface. +func (ipf *IPFilter) ToStage() Stage { + return ipf +} + // `Process` implements `Stage` interface func (ipf *IPFilter) Process(line []byte, lbs *LabelsBuilder) ([]byte, bool) { @@ -136,7 +153,7 @@ func (ipf *IPFilter) Process(line []byte, lbs *LabelsBuilder) ([]byte, bool) { case LabelFilterNotEqual: return line, !ipf.filter(input) default: - lbs.SetErr(errIPFilterInvalidOperation.Error()) + lbs.SetErr(ErrIPFilterInvalidOperation.Error()) } } @@ -188,7 +205,7 @@ func getMatcher(pattern string) (IPMatcher, error) { return matcher, nil } - return nil, fmt.Errorf("%w: %q", errIPFilterInvalidPattern, pattern) + return nil, fmt.Errorf("%w: %q", ErrIPFilterInvalidPattern, pattern) } func ipv4Hint(prefix [4]byte) bool { diff --git a/pkg/logql/log/ip_test.go b/pkg/logql/log/ip_test.go index dd5e49146c27a..c6318df7481fc 100644 --- a/pkg/logql/log/ip_test.go +++ b/pkg/logql/log/ip_test.go @@ -77,7 +77,7 @@ func Test_IPFilter(t *testing.T) { name: "wrong IP range syntax extra space", // NOTE(kavi): Should we handle this as normal valid range pattern? pat: "192.168.0.1 - 192.189.10.12", fail: true, - err: errIPFilterInvalidPattern, + err: ErrIPFilterInvalidPattern, }, { name: "CIDR", @@ -164,7 +164,7 @@ func Test_IPLabelFilterOp(t *testing.T) { label: "addr", val: []byte("192.168.0.1"), fail: true, - err: errIPFilterInvalidOperation.Error(), + err: ErrIPFilterInvalidOperation.Error(), }, { name: "great than or equal to operator-not-supported", @@ -173,7 +173,7 @@ func Test_IPLabelFilterOp(t *testing.T) { label: "addr", val: []byte("192.168.0.1"), fail: true, - err: errIPFilterInvalidOperation.Error(), + err: ErrIPFilterInvalidOperation.Error(), }, { name: "less than operator-not-supported", @@ -182,7 +182,7 @@ func Test_IPLabelFilterOp(t *testing.T) { label: "addr", val: []byte("192.168.0.1"), fail: true, - err: errIPFilterInvalidOperation.Error(), + err: ErrIPFilterInvalidOperation.Error(), }, { name: "less than or equal to operator-not-supported", @@ -191,7 +191,7 @@ func Test_IPLabelFilterOp(t *testing.T) { label: "addr", val: []byte("192.168.0.1"), fail: true, - err: errIPFilterInvalidOperation.Error(), + err: ErrIPFilterInvalidOperation.Error(), }, } diff --git a/pkg/logql/parser_test.go b/pkg/logql/parser_test.go index 7283c445b6b9c..e37e004f56826 100644 --- a/pkg/logql/parser_test.go +++ b/pkg/logql/parser_test.go @@ -31,7 +31,7 @@ func TestParse(t *testing.T) { left: &LogRange{ left: &PipelineExpr{ pipeline: MultiStageExpr{ - newLineFilterExpr(nil, labels.MatchRegexp, "error\\"), + newLineFilterExpr(labels.MatchRegexp, "", "error\\"), }, left: &MatchersExpr{ matchers: []*labels.Matcher{ @@ -52,7 +52,7 @@ func TestParse(t *testing.T) { left: newPipelineExpr( newMatcherExpr([]*labels.Matcher{{Type: labels.MatchEqual, Name: "foo", Value: "bar"}}), MultiStageExpr{ - newLineFilterExpr(nil, labels.MatchEqual, "error"), + newLineFilterExpr(labels.MatchEqual, "", "error"), }, ), interval: 12 * time.Hour, @@ -67,7 +67,7 @@ func TestParse(t *testing.T) { left: &LogRange{ left: newPipelineExpr( newMatcherExpr([]*labels.Matcher{{Type: labels.MatchEqual, Name: "foo", Value: "bar"}}), - MultiStageExpr{newLineFilterExpr(nil, labels.MatchEqual, "error")}, + MultiStageExpr{newLineFilterExpr(labels.MatchEqual, "", "error")}, ), interval: 12 * time.Hour, }, @@ -338,6 +338,104 @@ func TestParse(t *testing.T) { in: `min({ foo = "bar" }[5m])`, err: logqlmodel.NewParseError("syntax error: unexpected RANGE", 0, 20), }, + // line filter for ip-matcher + { + in: `{foo="bar"} |= "baz" |= ip("123.123.123.123")`, + exp: newPipelineExpr( + newMatcherExpr([]*labels.Matcher{mustNewMatcher(labels.MatchEqual, "foo", "bar")}), + MultiStageExpr{ + newNestedLineFilterExpr( + newLineFilterExpr(labels.MatchEqual, "", "baz"), + newLineFilterExpr(labels.MatchEqual, OpFilterIP, "123.123.123.123"), + ), + }, + ), + }, + { + in: `{foo="bar"} |= ip("123.123.123.123")`, + exp: newPipelineExpr( + newMatcherExpr([]*labels.Matcher{mustNewMatcher(labels.MatchEqual, "foo", "bar")}), + MultiStageExpr{ + newLineFilterExpr(labels.MatchEqual, OpFilterIP, "123.123.123.123"), + }, + ), + }, + { + in: `{foo="bar"} |= ip("123.123.123.123")|= "baz"`, + exp: newPipelineExpr( + newMatcherExpr([]*labels.Matcher{mustNewMatcher(labels.MatchEqual, "foo", "bar")}), + MultiStageExpr{ + newNestedLineFilterExpr( + newLineFilterExpr(labels.MatchEqual, OpFilterIP, "123.123.123.123"), + newLineFilterExpr(labels.MatchEqual, "", "baz"), + ), + }, + ), + }, + { + in: `{foo="bar"} |= ip("123.123.123.123")|= "baz" |=ip("123.123.123.123")`, + exp: newPipelineExpr( + newMatcherExpr([]*labels.Matcher{mustNewMatcher(labels.MatchEqual, "foo", "bar")}), + MultiStageExpr{ + newNestedLineFilterExpr( + newNestedLineFilterExpr( + newLineFilterExpr(labels.MatchEqual, OpFilterIP, "123.123.123.123"), + newLineFilterExpr(labels.MatchEqual, "", "baz"), + ), + newLineFilterExpr(labels.MatchEqual, OpFilterIP, "123.123.123.123"), + ), + }, + ), + }, + { + in: `{foo="bar"} |= "baz" |= ip("123.123.123.123")`, + exp: newPipelineExpr( + newMatcherExpr([]*labels.Matcher{mustNewMatcher(labels.MatchEqual, "foo", "bar")}), + MultiStageExpr{ + newNestedLineFilterExpr( + newLineFilterExpr(labels.MatchEqual, "", "baz"), + newLineFilterExpr(labels.MatchEqual, OpFilterIP, "123.123.123.123"), + ), + }, + ), + }, + { + in: `{foo="bar"} != ip("123.123.123.123")`, + exp: newPipelineExpr( + newMatcherExpr([]*labels.Matcher{mustNewMatcher(labels.MatchEqual, "foo", "bar")}), + MultiStageExpr{ + newLineFilterExpr(labels.MatchNotEqual, OpFilterIP, "123.123.123.123"), + }, + ), + }, + { + in: `{foo="bar"} != ip("123.123.123.123")|= "baz"`, + exp: newPipelineExpr( + newMatcherExpr([]*labels.Matcher{mustNewMatcher(labels.MatchEqual, "foo", "bar")}), + MultiStageExpr{ + newNestedLineFilterExpr( + newLineFilterExpr(labels.MatchNotEqual, OpFilterIP, "123.123.123.123"), + newLineFilterExpr(labels.MatchEqual, "", "baz"), + ), + }, + ), + }, + { + in: `{foo="bar"} != ip("123.123.123.123")|= "baz" !=ip("123.123.123.123")`, + exp: newPipelineExpr( + newMatcherExpr([]*labels.Matcher{mustNewMatcher(labels.MatchEqual, "foo", "bar")}), + MultiStageExpr{ + newNestedLineFilterExpr( + newNestedLineFilterExpr( + newLineFilterExpr(labels.MatchNotEqual, OpFilterIP, "123.123.123.123"), + newLineFilterExpr(labels.MatchEqual, "", "baz"), + ), + newLineFilterExpr(labels.MatchNotEqual, OpFilterIP, "123.123.123.123"), + ), + }, + ), + }, + // label filter for ip-matcher { in: `{ foo = "bar" }|logfmt|addr>=ip("1.2.3.4")`, err: logqlmodel.NewParseError("syntax error: unexpected ip, expecting BYTES or NUMBER or DURATION", 1, 30), @@ -457,7 +555,7 @@ func TestParse(t *testing.T) { in: `{foo="bar"} |= "baz"`, exp: newPipelineExpr( newMatcherExpr([]*labels.Matcher{mustNewMatcher(labels.MatchEqual, "foo", "bar")}), - MultiStageExpr{newLineFilterExpr(nil, labels.MatchEqual, "baz")}, + MultiStageExpr{newLineFilterExpr(labels.MatchEqual, "", "baz")}, ), }, { @@ -465,13 +563,16 @@ func TestParse(t *testing.T) { exp: newPipelineExpr( newMatcherExpr([]*labels.Matcher{mustNewMatcher(labels.MatchEqual, "foo", "bar")}), MultiStageExpr{ - newLineFilterExpr( - newLineFilterExpr( - newLineFilterExpr( - newLineFilterExpr(nil, labels.MatchEqual, "baz"), - labels.MatchRegexp, "blip"), - labels.MatchNotEqual, "flip"), - labels.MatchNotRegexp, "flap"), + newNestedLineFilterExpr( + newNestedLineFilterExpr( + newNestedLineFilterExpr( + newLineFilterExpr(labels.MatchEqual, "", "baz"), + newLineFilterExpr(labels.MatchRegexp, "", "blip"), + ), + newLineFilterExpr(labels.MatchNotEqual, "", "flip"), + ), + newLineFilterExpr(labels.MatchNotRegexp, "", "flap"), + ), }, ), }, @@ -482,13 +583,16 @@ func TestParse(t *testing.T) { left: newPipelineExpr( newMatcherExpr([]*labels.Matcher{mustNewMatcher(labels.MatchEqual, "foo", "bar")}), MultiStageExpr{ - newLineFilterExpr( - newLineFilterExpr( - newLineFilterExpr( - newLineFilterExpr(nil, labels.MatchEqual, "baz"), - labels.MatchRegexp, "blip"), - labels.MatchNotEqual, "flip"), - labels.MatchNotRegexp, "flap"), + newNestedLineFilterExpr( + newNestedLineFilterExpr( + newNestedLineFilterExpr( + newLineFilterExpr(labels.MatchEqual, "", "baz"), + newLineFilterExpr(labels.MatchRegexp, "", "blip"), + ), + newLineFilterExpr(labels.MatchNotEqual, "", "flip"), + ), + newLineFilterExpr(labels.MatchNotRegexp, "", "flap"), + ), }, ), interval: 5 * time.Minute, @@ -501,13 +605,16 @@ func TestParse(t *testing.T) { left: newPipelineExpr( newMatcherExpr([]*labels.Matcher{mustNewMatcher(labels.MatchEqual, "foo", "bar")}), MultiStageExpr{ - newLineFilterExpr( - newLineFilterExpr( - newLineFilterExpr( - newLineFilterExpr(nil, labels.MatchEqual, "baz"), - labels.MatchRegexp, "blip"), - labels.MatchNotEqual, "flip"), - labels.MatchNotRegexp, "flap"), + newNestedLineFilterExpr( + newNestedLineFilterExpr( + newNestedLineFilterExpr( + newLineFilterExpr(labels.MatchEqual, "", "baz"), + newLineFilterExpr(labels.MatchRegexp, "", "blip"), + ), + newLineFilterExpr(labels.MatchNotEqual, "", "flip"), + ), + newLineFilterExpr(labels.MatchNotRegexp, "", "flap"), + ), }, ), interval: 5 * time.Minute, @@ -520,13 +627,16 @@ func TestParse(t *testing.T) { left: newPipelineExpr( newMatcherExpr([]*labels.Matcher{mustNewMatcher(labels.MatchEqual, "foo", "bar")}), MultiStageExpr{ - newLineFilterExpr( - newLineFilterExpr( - newLineFilterExpr( - newLineFilterExpr(nil, labels.MatchEqual, "baz"), - labels.MatchRegexp, "blip"), - labels.MatchNotEqual, "flip"), - labels.MatchNotRegexp, "flap"), + newNestedLineFilterExpr( + newNestedLineFilterExpr( + newNestedLineFilterExpr( + newLineFilterExpr(labels.MatchEqual, "", "baz"), + newLineFilterExpr(labels.MatchRegexp, "", "blip"), + ), + newLineFilterExpr(labels.MatchNotEqual, "", "flip"), + ), + newLineFilterExpr(labels.MatchNotRegexp, "", "flap"), + ), newLabelParserExpr(OpParserTypeUnpack, ""), }, ), @@ -549,13 +659,16 @@ func TestParse(t *testing.T) { left: newPipelineExpr( newMatcherExpr([]*labels.Matcher{mustNewMatcher(labels.MatchEqual, "foo", "bar")}), MultiStageExpr{ - newLineFilterExpr( - newLineFilterExpr( - newLineFilterExpr( - newLineFilterExpr(nil, labels.MatchEqual, "baz"), - labels.MatchRegexp, "blip"), - labels.MatchNotEqual, "flip"), - labels.MatchNotRegexp, "flap"), + newNestedLineFilterExpr( + newNestedLineFilterExpr( + newNestedLineFilterExpr( + newLineFilterExpr(labels.MatchEqual, "", "baz"), + newLineFilterExpr(labels.MatchRegexp, "", "blip"), + ), + newLineFilterExpr(labels.MatchNotEqual, "", "flip"), + ), + newLineFilterExpr(labels.MatchNotRegexp, "", "flap"), + ), }, ), interval: 5 * time.Minute, @@ -573,13 +686,16 @@ func TestParse(t *testing.T) { left: newPipelineExpr( newMatcherExpr([]*labels.Matcher{mustNewMatcher(labels.MatchEqual, "foo", "bar")}), MultiStageExpr{ - newLineFilterExpr( - newLineFilterExpr( - newLineFilterExpr( - newLineFilterExpr(nil, labels.MatchEqual, "baz"), - labels.MatchRegexp, "blip"), - labels.MatchNotEqual, "flip"), - labels.MatchNotRegexp, "flap"), + newNestedLineFilterExpr( + newNestedLineFilterExpr( + newNestedLineFilterExpr( + newLineFilterExpr(labels.MatchEqual, "", "baz"), + newLineFilterExpr(labels.MatchRegexp, "", "blip"), + ), + newLineFilterExpr(labels.MatchNotEqual, "", "flip"), + ), + newLineFilterExpr(labels.MatchNotRegexp, "", "flap"), + ), }, ), interval: 5 * time.Minute, @@ -598,13 +714,16 @@ func TestParse(t *testing.T) { left: newPipelineExpr( newMatcherExpr([]*labels.Matcher{mustNewMatcher(labels.MatchEqual, "foo", "bar")}), MultiStageExpr{ - newLineFilterExpr( - newLineFilterExpr( - newLineFilterExpr( - newLineFilterExpr(nil, labels.MatchEqual, "baz"), - labels.MatchRegexp, "blip"), - labels.MatchNotEqual, "flip"), - labels.MatchNotRegexp, "flap"), + newNestedLineFilterExpr( + newNestedLineFilterExpr( + newNestedLineFilterExpr( + newLineFilterExpr(labels.MatchEqual, "", "baz"), + newLineFilterExpr(labels.MatchRegexp, "", "blip"), + ), + newLineFilterExpr(labels.MatchNotEqual, "", "flip"), + ), + newLineFilterExpr(labels.MatchNotRegexp, "", "flap"), + ), }, ), interval: 5 * time.Minute, @@ -623,13 +742,16 @@ func TestParse(t *testing.T) { left: newPipelineExpr( newMatcherExpr([]*labels.Matcher{mustNewMatcher(labels.MatchEqual, "foo", "bar")}), MultiStageExpr{ - newLineFilterExpr( - newLineFilterExpr( - newLineFilterExpr( - newLineFilterExpr(nil, labels.MatchEqual, "baz"), - labels.MatchRegexp, "blip"), - labels.MatchNotEqual, "flip"), - labels.MatchNotRegexp, "flap"), + newNestedLineFilterExpr( + newNestedLineFilterExpr( + newNestedLineFilterExpr( + newLineFilterExpr(labels.MatchEqual, "", "baz"), + newLineFilterExpr(labels.MatchRegexp, "", "blip"), + ), + newLineFilterExpr(labels.MatchNotEqual, "", "flip"), + ), + newLineFilterExpr(labels.MatchNotRegexp, "", "flap"), + ), }, ), interval: 5 * time.Minute, @@ -650,13 +772,16 @@ func TestParse(t *testing.T) { left: newPipelineExpr( newMatcherExpr([]*labels.Matcher{mustNewMatcher(labels.MatchEqual, "foo", "bar")}), MultiStageExpr{ - newLineFilterExpr( - newLineFilterExpr( - newLineFilterExpr( - newLineFilterExpr(nil, labels.MatchEqual, "baz"), - labels.MatchRegexp, "blip"), - labels.MatchNotEqual, "flip"), - labels.MatchNotRegexp, "flap"), + newNestedLineFilterExpr( + newNestedLineFilterExpr( + newNestedLineFilterExpr( + newLineFilterExpr(labels.MatchEqual, "", "baz"), + newLineFilterExpr(labels.MatchRegexp, "", "blip"), + ), + newLineFilterExpr(labels.MatchNotEqual, "", "flip"), + ), + newLineFilterExpr(labels.MatchNotRegexp, "", "flap"), + ), }, ), interval: 5 * time.Minute, @@ -678,13 +803,16 @@ func TestParse(t *testing.T) { left: newPipelineExpr( newMatcherExpr([]*labels.Matcher{mustNewMatcher(labels.MatchEqual, "foo", "bar")}), MultiStageExpr{ - newLineFilterExpr( - newLineFilterExpr( - newLineFilterExpr( - newLineFilterExpr(nil, labels.MatchEqual, "baz"), - labels.MatchRegexp, "blip"), - labels.MatchNotEqual, "flip"), - labels.MatchNotRegexp, "flap"), + newNestedLineFilterExpr( + newNestedLineFilterExpr( + newNestedLineFilterExpr( + newLineFilterExpr(labels.MatchEqual, "", "baz"), + newLineFilterExpr(labels.MatchRegexp, "", "blip"), + ), + newLineFilterExpr(labels.MatchNotEqual, "", "flip"), + ), + newLineFilterExpr(labels.MatchNotRegexp, "", "flap"), + ), }, ), interval: 5 * time.Minute, @@ -697,13 +825,16 @@ func TestParse(t *testing.T) { left: newPipelineExpr( newMatcherExpr([]*labels.Matcher{mustNewMatcher(labels.MatchEqual, "foo", "bar")}), MultiStageExpr{ - newLineFilterExpr( - newLineFilterExpr( - newLineFilterExpr( - newLineFilterExpr(nil, labels.MatchEqual, "baz"), - labels.MatchRegexp, "blip"), - labels.MatchNotEqual, "flip"), - labels.MatchNotRegexp, "flap"), + newNestedLineFilterExpr( + newNestedLineFilterExpr( + newNestedLineFilterExpr( + newLineFilterExpr(labels.MatchEqual, "", "baz"), + newLineFilterExpr(labels.MatchRegexp, "", "blip"), + ), + newLineFilterExpr(labels.MatchNotEqual, "", "flip"), + ), + newLineFilterExpr(labels.MatchNotRegexp, "", "flap"), + ), }, ), interval: 5 * time.Minute, @@ -722,13 +853,16 @@ func TestParse(t *testing.T) { left: newPipelineExpr( newMatcherExpr([]*labels.Matcher{mustNewMatcher(labels.MatchEqual, "foo", "bar")}), MultiStageExpr{ - newLineFilterExpr( - newLineFilterExpr( - newLineFilterExpr( - newLineFilterExpr(nil, labels.MatchEqual, "baz"), - labels.MatchRegexp, "blip"), - labels.MatchNotEqual, "flip"), - labels.MatchNotRegexp, "flap"), + newNestedLineFilterExpr( + newNestedLineFilterExpr( + newNestedLineFilterExpr( + newLineFilterExpr(labels.MatchEqual, "", "baz"), + newLineFilterExpr(labels.MatchRegexp, "", "blip"), + ), + newLineFilterExpr(labels.MatchNotEqual, "", "flip"), + ), + newLineFilterExpr(labels.MatchNotRegexp, "", "flap"), + ), }, ), interval: 5 * time.Minute, @@ -749,13 +883,16 @@ func TestParse(t *testing.T) { left: newPipelineExpr( newMatcherExpr([]*labels.Matcher{mustNewMatcher(labels.MatchEqual, "foo", "bar")}), MultiStageExpr{ - newLineFilterExpr( - newLineFilterExpr( - newLineFilterExpr( - newLineFilterExpr(nil, labels.MatchEqual, "baz"), - labels.MatchRegexp, "blip"), - labels.MatchNotEqual, "flip"), - labels.MatchNotRegexp, "flap"), + newNestedLineFilterExpr( + newNestedLineFilterExpr( + newNestedLineFilterExpr( + newLineFilterExpr(labels.MatchEqual, "", "baz"), + newLineFilterExpr(labels.MatchRegexp, "", "blip"), + ), + newLineFilterExpr(labels.MatchNotEqual, "", "flip"), + ), + newLineFilterExpr(labels.MatchNotRegexp, "", "flap"), + ), }, ), interval: 5 * time.Minute, @@ -781,7 +918,7 @@ func TestParse(t *testing.T) { { in: `{foo="bar"} |~`, - err: logqlmodel.NewParseError("syntax error: unexpected $end, expecting STRING", 1, 15), + err: logqlmodel.NewParseError("syntax error: unexpected $end, expecting STRING or ip", 1, 15), }, { @@ -999,7 +1136,7 @@ func TestParse(t *testing.T) { mustNewMatcher(labels.MatchEqual, "namespace", "tns"), }), MultiStageExpr{ - newLineFilterExpr(nil, labels.MatchEqual, "level=error"), + newLineFilterExpr(labels.MatchEqual, "", "level=error"), }), interval: 5 * time.Minute, }, OpRangeTypeCount, nil, nil), @@ -1029,7 +1166,7 @@ func TestParse(t *testing.T) { mustNewMatcher(labels.MatchEqual, "namespace", "tns"), }), MultiStageExpr{ - newLineFilterExpr(nil, labels.MatchEqual, "level=error"), + newLineFilterExpr(labels.MatchEqual, "", "level=error"), }), interval: 5 * time.Minute, }, OpRangeTypeCount, nil, nil), @@ -1096,7 +1233,7 @@ func TestParse(t *testing.T) { exp: &PipelineExpr{ left: newMatcherExpr([]*labels.Matcher{{Type: labels.MatchEqual, Name: "app", Value: "foo"}}), pipeline: MultiStageExpr{ - newLineFilterExpr(nil, labels.MatchEqual, "bar"), + newLineFilterExpr(labels.MatchEqual, "", "bar"), newLabelParserExpr(OpParserTypeJSON, ""), &LabelFilterExpr{ LabelFilterer: log.NewOrLabelFilter( @@ -1115,7 +1252,7 @@ func TestParse(t *testing.T) { exp: &PipelineExpr{ left: newMatcherExpr([]*labels.Matcher{{Type: labels.MatchEqual, Name: "app", Value: "foo"}}), pipeline: MultiStageExpr{ - newLineFilterExpr(nil, labels.MatchEqual, "bar"), + newLineFilterExpr(labels.MatchEqual, "", "bar"), newLabelParserExpr(OpParserTypeUnpack, ""), newLabelParserExpr(OpParserTypeJSON, ""), &LabelFilterExpr{ @@ -1135,7 +1272,7 @@ func TestParse(t *testing.T) { exp: &PipelineExpr{ left: newMatcherExpr([]*labels.Matcher{{Type: labels.MatchEqual, Name: "app", Value: "foo"}}), pipeline: MultiStageExpr{ - newLineFilterExpr(nil, labels.MatchEqual, "bar"), + newLineFilterExpr(labels.MatchEqual, "", "bar"), newLabelParserExpr(OpParserTypeJSON, ""), &LabelFilterExpr{ LabelFilterer: log.NewAndLabelFilter( @@ -1154,7 +1291,7 @@ func TestParse(t *testing.T) { exp: &PipelineExpr{ left: newMatcherExpr([]*labels.Matcher{{Type: labels.MatchEqual, Name: "app", Value: "foo"}}), pipeline: MultiStageExpr{ - newLineFilterExpr(nil, labels.MatchEqual, "bar"), + newLineFilterExpr(labels.MatchEqual, "", "bar"), newLabelParserExpr(OpParserTypePattern, " bar "), &LabelFilterExpr{ LabelFilterer: log.NewAndLabelFilter( @@ -1173,7 +1310,7 @@ func TestParse(t *testing.T) { exp: &PipelineExpr{ left: newMatcherExpr([]*labels.Matcher{{Type: labels.MatchEqual, Name: "app", Value: "foo"}}), pipeline: MultiStageExpr{ - newLineFilterExpr(nil, labels.MatchEqual, "bar"), + newLineFilterExpr(labels.MatchEqual, "", "bar"), newLabelParserExpr(OpParserTypeJSON, ""), &LabelFilterExpr{ LabelFilterer: log.NewOrLabelFilter( @@ -1192,7 +1329,7 @@ func TestParse(t *testing.T) { exp: &PipelineExpr{ left: newMatcherExpr([]*labels.Matcher{{Type: labels.MatchEqual, Name: "app", Value: "foo"}}), pipeline: MultiStageExpr{ - newLineFilterExpr(nil, labels.MatchEqual, "bar"), + newLineFilterExpr(labels.MatchEqual, "", "bar"), newLabelParserExpr(OpParserTypeJSON, ""), &LabelFilterExpr{ LabelFilterer: log.NewAndLabelFilter( @@ -1211,7 +1348,7 @@ func TestParse(t *testing.T) { exp: &PipelineExpr{ left: newMatcherExpr([]*labels.Matcher{{Type: labels.MatchEqual, Name: "app", Value: "foo"}}), pipeline: MultiStageExpr{ - newLineFilterExpr(nil, labels.MatchEqual, "bar"), + newLineFilterExpr(labels.MatchEqual, "", "bar"), newLabelParserExpr(OpParserTypeJSON, ""), &LabelFilterExpr{ LabelFilterer: log.NewOrLabelFilter( @@ -1231,7 +1368,7 @@ func TestParse(t *testing.T) { exp: &PipelineExpr{ left: newMatcherExpr([]*labels.Matcher{{Type: labels.MatchEqual, Name: "app", Value: "foo"}}), pipeline: MultiStageExpr{ - newLineFilterExpr(nil, labels.MatchEqual, "bar"), + newLineFilterExpr(labels.MatchEqual, "", "bar"), newLabelParserExpr(OpParserTypeJSON, ""), &LabelFilterExpr{ LabelFilterer: log.NewOrLabelFilter( @@ -1262,7 +1399,7 @@ func TestParse(t *testing.T) { exp: &PipelineExpr{ left: newMatcherExpr([]*labels.Matcher{{Type: labels.MatchEqual, Name: "app", Value: "foo"}}), pipeline: MultiStageExpr{ - newLineFilterExpr(nil, labels.MatchEqual, "bar"), + newLineFilterExpr(labels.MatchEqual, "", "bar"), newLineFmtExpr("blip{{ .foo }}blop"), }, }, @@ -1273,7 +1410,7 @@ func TestParse(t *testing.T) { exp: &PipelineExpr{ left: newMatcherExpr([]*labels.Matcher{{Type: labels.MatchEqual, Name: "app", Value: "foo"}}), pipeline: MultiStageExpr{ - newLineFilterExpr(nil, labels.MatchEqual, "bar"), + newLineFilterExpr(labels.MatchEqual, "", "bar"), newLabelParserExpr(OpParserTypeJSON, ""), &LabelFilterExpr{ LabelFilterer: log.NewOrLabelFilter( @@ -1294,7 +1431,7 @@ func TestParse(t *testing.T) { exp: &PipelineExpr{ left: newMatcherExpr([]*labels.Matcher{{Type: labels.MatchEqual, Name: "app", Value: "foo"}}), pipeline: MultiStageExpr{ - newLineFilterExpr(nil, labels.MatchEqual, "bar"), + newLineFilterExpr(labels.MatchEqual, "", "bar"), newLabelParserExpr(OpParserTypeJSON, ""), &LabelFilterExpr{ LabelFilterer: log.NewOrLabelFilter( @@ -1320,7 +1457,7 @@ func TestParse(t *testing.T) { newLogRange(&PipelineExpr{ left: newMatcherExpr([]*labels.Matcher{{Type: labels.MatchEqual, Name: "app", Value: "foo"}}), pipeline: MultiStageExpr{ - newLineFilterExpr(nil, labels.MatchEqual, "bar"), + newLineFilterExpr(labels.MatchEqual, "", "bar"), newLabelParserExpr(OpParserTypeJSON, ""), &LabelFilterExpr{ LabelFilterer: log.NewOrLabelFilter( @@ -1366,7 +1503,7 @@ func TestParse(t *testing.T) { exp: &PipelineExpr{ left: newMatcherExpr([]*labels.Matcher{{Type: labels.MatchEqual, Name: "app", Value: "foo"}}), pipeline: MultiStageExpr{ - newLineFilterExpr(nil, labels.MatchEqual, "bar"), + newLineFilterExpr(labels.MatchEqual, "", "bar"), newLabelParserExpr(OpParserTypeJSON, ""), &LabelFilterExpr{ LabelFilterer: log.NewOrLabelFilter( @@ -1387,7 +1524,7 @@ func TestParse(t *testing.T) { newLogRange(&PipelineExpr{ left: newMatcherExpr([]*labels.Matcher{{Type: labels.MatchEqual, Name: "app", Value: "foo"}}), pipeline: MultiStageExpr{ - newLineFilterExpr(nil, labels.MatchEqual, "bar"), + newLineFilterExpr(labels.MatchEqual, "", "bar"), newLabelParserExpr(OpParserTypeJSON, ""), &LabelFilterExpr{ LabelFilterer: log.NewOrLabelFilter( @@ -1418,7 +1555,7 @@ func TestParse(t *testing.T) { newLogRange(&PipelineExpr{ left: newMatcherExpr([]*labels.Matcher{{Type: labels.MatchEqual, Name: "app", Value: "foo"}}), pipeline: MultiStageExpr{ - newLineFilterExpr(nil, labels.MatchEqual, "bar"), + newLineFilterExpr(labels.MatchEqual, "", "bar"), newLabelParserExpr(OpParserTypeJSON, ""), &LabelFilterExpr{ LabelFilterer: log.NewOrLabelFilter( @@ -1448,7 +1585,7 @@ func TestParse(t *testing.T) { newLogRange(&PipelineExpr{ left: newMatcherExpr([]*labels.Matcher{{Type: labels.MatchEqual, Name: "namespace", Value: "tns"}}), pipeline: MultiStageExpr{ - newLineFilterExpr(nil, labels.MatchEqual, "level=error"), + newLineFilterExpr(labels.MatchEqual, "", "level=error"), newLabelParserExpr(OpParserTypeJSON, ""), &LabelFilterExpr{ LabelFilterer: log.NewAndLabelFilter( @@ -1470,7 +1607,7 @@ func TestParse(t *testing.T) { newLogRange(&PipelineExpr{ left: newMatcherExpr([]*labels.Matcher{{Type: labels.MatchEqual, Name: "namespace", Value: "tns"}}), pipeline: MultiStageExpr{ - newLineFilterExpr(nil, labels.MatchEqual, "level=error"), + newLineFilterExpr(labels.MatchEqual, "", "level=error"), newLabelParserExpr(OpParserTypeJSON, ""), &LabelFilterExpr{ LabelFilterer: log.NewAndLabelFilter( @@ -1492,7 +1629,7 @@ func TestParse(t *testing.T) { newLogRange(&PipelineExpr{ left: newMatcherExpr([]*labels.Matcher{{Type: labels.MatchEqual, Name: "namespace", Value: "tns"}}), pipeline: MultiStageExpr{ - newLineFilterExpr(nil, labels.MatchEqual, "level=error"), + newLineFilterExpr(labels.MatchEqual, "", "level=error"), newLabelParserExpr(OpParserTypeJSON, ""), &LabelFilterExpr{ LabelFilterer: log.NewAndLabelFilter( @@ -1514,7 +1651,7 @@ func TestParse(t *testing.T) { newLogRange(&PipelineExpr{ left: newMatcherExpr([]*labels.Matcher{{Type: labels.MatchEqual, Name: "namespace", Value: "tns"}}), pipeline: MultiStageExpr{ - newLineFilterExpr(nil, labels.MatchEqual, "level=error"), + newLineFilterExpr(labels.MatchEqual, "", "level=error"), newLabelParserExpr(OpParserTypeJSON, ""), &LabelFilterExpr{ LabelFilterer: log.NewAndLabelFilter( @@ -1536,7 +1673,7 @@ func TestParse(t *testing.T) { newLogRange(&PipelineExpr{ left: newMatcherExpr([]*labels.Matcher{{Type: labels.MatchEqual, Name: "app", Value: "foo"}}), pipeline: MultiStageExpr{ - newLineFilterExpr(nil, labels.MatchEqual, "bar"), + newLineFilterExpr(labels.MatchEqual, "", "bar"), }, }, 5*time.Minute, @@ -1607,7 +1744,7 @@ func TestParse(t *testing.T) { newLogRange(&PipelineExpr{ left: newMatcherExpr([]*labels.Matcher{{Type: labels.MatchEqual, Name: "app", Value: "foo"}}), pipeline: MultiStageExpr{ - newLineFilterExpr(nil, labels.MatchEqual, "bar"), + newLineFilterExpr(labels.MatchEqual, "", "bar"), newLabelParserExpr(OpParserTypeJSON, ""), &LabelFilterExpr{ LabelFilterer: log.NewOrLabelFilter( @@ -1638,7 +1775,7 @@ func TestParse(t *testing.T) { newLogRange(&PipelineExpr{ left: newMatcherExpr([]*labels.Matcher{{Type: labels.MatchEqual, Name: "app", Value: "foo"}}), pipeline: MultiStageExpr{ - newLineFilterExpr(nil, labels.MatchEqual, "bar"), + newLineFilterExpr(labels.MatchEqual, "", "bar"), newLabelParserExpr(OpParserTypeJSON, ""), &LabelFilterExpr{ LabelFilterer: log.NewOrLabelFilter( @@ -1669,7 +1806,7 @@ func TestParse(t *testing.T) { newLogRange(&PipelineExpr{ left: newMatcherExpr([]*labels.Matcher{{Type: labels.MatchEqual, Name: "app", Value: "foo"}}), pipeline: MultiStageExpr{ - newLineFilterExpr(nil, labels.MatchEqual, "bar"), + newLineFilterExpr(labels.MatchEqual, "", "bar"), newLabelParserExpr(OpParserTypeJSON, ""), &LabelFilterExpr{ LabelFilterer: log.NewOrLabelFilter( @@ -1700,7 +1837,7 @@ func TestParse(t *testing.T) { newLogRange(&PipelineExpr{ left: newMatcherExpr([]*labels.Matcher{{Type: labels.MatchEqual, Name: "app", Value: "foo"}}), pipeline: MultiStageExpr{ - newLineFilterExpr(nil, labels.MatchEqual, "bar"), + newLineFilterExpr(labels.MatchEqual, "", "bar"), newLabelParserExpr(OpParserTypeJSON, ""), &LabelFilterExpr{ LabelFilterer: log.NewOrLabelFilter( @@ -1735,7 +1872,7 @@ func TestParse(t *testing.T) { newLogRange(&PipelineExpr{ left: newMatcherExpr([]*labels.Matcher{{Type: labels.MatchEqual, Name: "app", Value: "foo"}}), pipeline: MultiStageExpr{ - newLineFilterExpr(nil, labels.MatchEqual, "bar"), + newLineFilterExpr(labels.MatchEqual, "", "bar"), newLabelParserExpr(OpParserTypeJSON, ""), &LabelFilterExpr{ LabelFilterer: log.NewOrLabelFilter( @@ -1774,7 +1911,7 @@ func TestParse(t *testing.T) { newLogRange(&PipelineExpr{ left: newMatcherExpr([]*labels.Matcher{{Type: labels.MatchEqual, Name: "app", Value: "foo"}}), pipeline: MultiStageExpr{ - newLineFilterExpr(nil, labels.MatchEqual, "bar"), + newLineFilterExpr(labels.MatchEqual, "", "bar"), newLabelParserExpr(OpParserTypeJSON, ""), &LabelFilterExpr{ LabelFilterer: log.NewOrLabelFilter( @@ -1813,7 +1950,7 @@ func TestParse(t *testing.T) { newLogRange(&PipelineExpr{ left: newMatcherExpr([]*labels.Matcher{{Type: labels.MatchEqual, Name: "app", Value: "foo"}}), pipeline: MultiStageExpr{ - newLineFilterExpr(nil, labels.MatchEqual, "bar"), + newLineFilterExpr(labels.MatchEqual, "", "bar"), newLabelParserExpr(OpParserTypeJSON, ""), &LabelFilterExpr{ LabelFilterer: log.NewOrLabelFilter( @@ -1852,7 +1989,7 @@ func TestParse(t *testing.T) { newLogRange(&PipelineExpr{ left: newMatcherExpr([]*labels.Matcher{{Type: labels.MatchEqual, Name: "app", Value: "foo"}}), pipeline: MultiStageExpr{ - newLineFilterExpr(nil, labels.MatchEqual, "bar"), + newLineFilterExpr(labels.MatchEqual, "", "bar"), newLabelParserExpr(OpParserTypeJSON, ""), &LabelFilterExpr{ LabelFilterer: log.NewOrLabelFilter( @@ -1891,7 +2028,7 @@ func TestParse(t *testing.T) { newLogRange(&PipelineExpr{ left: newMatcherExpr([]*labels.Matcher{{Type: labels.MatchEqual, Name: "app", Value: "foo"}}), pipeline: MultiStageExpr{ - newLineFilterExpr(nil, labels.MatchEqual, "bar"), + newLineFilterExpr(labels.MatchEqual, "", "bar"), newLabelParserExpr(OpParserTypeJSON, ""), &LabelFilterExpr{ LabelFilterer: log.NewOrLabelFilter( @@ -1930,7 +2067,7 @@ func TestParse(t *testing.T) { newLogRange(&PipelineExpr{ left: newMatcherExpr([]*labels.Matcher{{Type: labels.MatchEqual, Name: "app", Value: "foo"}}), pipeline: MultiStageExpr{ - newLineFilterExpr(nil, labels.MatchEqual, "bar"), + newLineFilterExpr(labels.MatchEqual, "", "bar"), newLabelParserExpr(OpParserTypeJSON, ""), &LabelFilterExpr{ LabelFilterer: log.NewOrLabelFilter( @@ -1978,7 +2115,7 @@ func TestParse(t *testing.T) { newLogRange(&PipelineExpr{ left: newMatcherExpr([]*labels.Matcher{{Type: labels.MatchEqual, Name: "app", Value: "foo"}}), pipeline: MultiStageExpr{ - newLineFilterExpr(nil, labels.MatchEqual, "bar"), + newLineFilterExpr(labels.MatchEqual, "", "bar"), newLabelParserExpr(OpParserTypeJSON, ""), &LabelFilterExpr{ LabelFilterer: log.NewOrLabelFilter( @@ -2010,7 +2147,7 @@ func TestParse(t *testing.T) { newLogRange(&PipelineExpr{ left: newMatcherExpr([]*labels.Matcher{{Type: labels.MatchEqual, Name: "app", Value: "foo"}}), pipeline: MultiStageExpr{ - newLineFilterExpr(nil, labels.MatchEqual, "bar"), + newLineFilterExpr(labels.MatchEqual, "", "bar"), newLabelParserExpr(OpParserTypeJSON, ""), &LabelFilterExpr{ LabelFilterer: log.NewOrLabelFilter( @@ -2065,7 +2202,7 @@ func TestParse(t *testing.T) { newLogRange(&PipelineExpr{ left: newMatcherExpr([]*labels.Matcher{{Type: labels.MatchEqual, Name: "app", Value: "foo"}}), pipeline: MultiStageExpr{ - newLineFilterExpr(nil, labels.MatchEqual, "bar"), + newLineFilterExpr(labels.MatchEqual, "", "bar"), newLabelParserExpr(OpParserTypeJSON, ""), &LabelFilterExpr{ LabelFilterer: log.NewOrLabelFilter( @@ -2097,7 +2234,7 @@ func TestParse(t *testing.T) { newLogRange(&PipelineExpr{ left: newMatcherExpr([]*labels.Matcher{{Type: labels.MatchEqual, Name: "app", Value: "foo"}}), pipeline: MultiStageExpr{ - newLineFilterExpr(nil, labels.MatchEqual, "bar"), + newLineFilterExpr(labels.MatchEqual, "", "bar"), newLabelParserExpr(OpParserTypeJSON, ""), &LabelFilterExpr{ LabelFilterer: log.NewOrLabelFilter( @@ -2305,7 +2442,7 @@ func TestParse(t *testing.T) { exp: &PipelineExpr{ left: newMatcherExpr([]*labels.Matcher{{Type: labels.MatchEqual, Name: "app", Value: "foo"}}), pipeline: MultiStageExpr{ - newLineFilterExpr(nil, labels.MatchEqual, "bar"), + newLineFilterExpr(labels.MatchEqual, "", "bar"), newLabelParserExpr(OpParserTypeJSON, ""), }, }, @@ -2336,7 +2473,7 @@ func TestParse(t *testing.T) { exp: &PipelineExpr{ left: newMatcherExpr([]*labels.Matcher{{Type: labels.MatchEqual, Name: "app", Value: "foo"}}), pipeline: MultiStageExpr{ - newLineFilterExpr(nil, labels.MatchEqual, "#"), + newLineFilterExpr(labels.MatchEqual, "", "#"), }, }, }, diff --git a/pkg/logql/shardmapper_test.go b/pkg/logql/shardmapper_test.go index b22d449d0ed24..71ac607f51926 100644 --- a/pkg/logql/shardmapper_test.go +++ b/pkg/logql/shardmapper_test.go @@ -255,7 +255,7 @@ func TestMapping(t *testing.T) { LogSelectorExpr: newPipelineExpr( newMatcherExpr([]*labels.Matcher{mustNewMatcher(labels.MatchEqual, "foo", "bar")}), MultiStageExpr{ - newLineFilterExpr(nil, labels.MatchEqual, "error"), + newLineFilterExpr( labels.MatchEqual,"", "error"), }, ), }, @@ -268,7 +268,7 @@ func TestMapping(t *testing.T) { LogSelectorExpr: newPipelineExpr( newMatcherExpr([]*labels.Matcher{mustNewMatcher(labels.MatchEqual, "foo", "bar")}), MultiStageExpr{ - newLineFilterExpr(nil, labels.MatchEqual, "error"), + newLineFilterExpr( labels.MatchEqual,"", "error"), }, ), }, diff --git a/pkg/querier/http.go b/pkg/querier/http.go index ccb572ea9636a..2ce0c7ce7cf15 100644 --- a/pkg/querier/http.go +++ b/pkg/querier/http.go @@ -337,7 +337,7 @@ func parseRegexQuery(httpRequest *http.Request) (string, error) { if err != nil { return "", err } - newExpr, err := logql.AddFilterExpr(expr, labels.MatchRegexp, regexp) + newExpr, err := logql.AddFilterExpr(expr, labels.MatchRegexp, "", regexp) if err != nil { return "", err } diff --git a/pkg/querier/queryrange/roundtrip.go b/pkg/querier/queryrange/roundtrip.go index 400e156726250..42946b7faebb8 100644 --- a/pkg/querier/queryrange/roundtrip.go +++ b/pkg/querier/queryrange/roundtrip.go @@ -159,7 +159,7 @@ func (r roundTripper) RoundTrip(req *http.Request) (*http.Response, error) { func transformRegexQuery(req *http.Request, expr logql.LogSelectorExpr) (logql.LogSelectorExpr, error) { regexp := req.Form.Get("regexp") if regexp != "" { - filterExpr, err := logql.AddFilterExpr(expr, labels.MatchRegexp, regexp) + filterExpr, err := logql.AddFilterExpr(expr, labels.MatchRegexp, "", regexp) if err != nil { return nil, err } From 52646aeb7da7053a2123b7b770d0db710fd567de Mon Sep 17 00:00:00 2001 From: Kaviraj Date: Wed, 14 Jul 2021 14:22:48 +0200 Subject: [PATCH 11/24] Make linter happy Signed-off-by: Kaviraj --- pkg/logql/shardmapper_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/logql/shardmapper_test.go b/pkg/logql/shardmapper_test.go index 71ac607f51926..2c1d5c3ea6024 100644 --- a/pkg/logql/shardmapper_test.go +++ b/pkg/logql/shardmapper_test.go @@ -255,7 +255,7 @@ func TestMapping(t *testing.T) { LogSelectorExpr: newPipelineExpr( newMatcherExpr([]*labels.Matcher{mustNewMatcher(labels.MatchEqual, "foo", "bar")}), MultiStageExpr{ - newLineFilterExpr( labels.MatchEqual,"", "error"), + newLineFilterExpr(labels.MatchEqual, "", "error"), }, ), }, @@ -268,7 +268,7 @@ func TestMapping(t *testing.T) { LogSelectorExpr: newPipelineExpr( newMatcherExpr([]*labels.Matcher{mustNewMatcher(labels.MatchEqual, "foo", "bar")}), MultiStageExpr{ - newLineFilterExpr( labels.MatchEqual,"", "error"), + newLineFilterExpr(labels.MatchEqual, "", "error"), }, ), }, From 657c2770ba7c4476da97f42d8f996dd0c2d792bc Mon Sep 17 00:00:00 2001 From: Kaviraj Date: Wed, 14 Jul 2021 15:48:05 +0200 Subject: [PATCH 12/24] Bug with filter matcher Its wierd bug. Not handing string() method breaks the parser :( Signed-off-by: Kaviraj --- pkg/logql/ast.go | 2 +- pkg/logql/expr.y | 4 +++- pkg/logql/log/ip.go | 48 +++++++++++++++++++++++++++++----------- pkg/logql/log/ip_test.go | 6 ++--- 4 files changed, 42 insertions(+), 18 deletions(-) diff --git a/pkg/logql/ast.go b/pkg/logql/ast.go index c798b6698dc4f..20bf03e3ef606 100644 --- a/pkg/logql/ast.go +++ b/pkg/logql/ast.go @@ -315,7 +315,7 @@ func (e *lineFilterExpr) Filter() (log.Filterer, error) { return nil, log.ErrIPFilterInvalidOperation } - f = log.NewIPLineFilter(e.match) + f = log.NewIPLineFilter(e.match, e.ty) } else { var err error // to avoid `f` being shadowed. f, err = log.NewFilter(e.match, e.ty) diff --git a/pkg/logql/expr.y b/pkg/logql/expr.y index a210da7f0b21c..0a547bf0c3416 100644 --- a/pkg/logql/expr.y +++ b/pkg/logql/expr.y @@ -249,11 +249,13 @@ filterOp: lineFilter: filter STRING { $$ = newLineFilterExpr($1, "", $2,) } - | filter filterOp OPEN_PARENTHESIS STRING CLOSE_PARENTHESIS { $$ = newLineFilterExpr($1, $2, $4) } + | filter filterOp OPEN_PARENTHESIS STRING CLOSE_PARENTHESIS { $$ = newLineFilterExpr($1, $2, $4)} + ; lineFilters: lineFilter { $$ = $1 } | lineFilters lineFilter { $$ = newNestedLineFilterExpr($1, $2) } + ; labelParser: JSON { $$ = newLabelParserExpr(OpParserTypeJSON, "") } diff --git a/pkg/logql/log/ip.go b/pkg/logql/log/ip.go index 87011c6314172..80a9b99f8b615 100644 --- a/pkg/logql/log/ip.go +++ b/pkg/logql/log/ip.go @@ -5,6 +5,7 @@ import ( "fmt" "unicode" + "github.com/prometheus/prometheus/pkg/labels" "inet.af/netaddr" ) @@ -47,6 +48,9 @@ type IPFilter struct { // filter `operation` used during label filter. labelOp LabelFilterType + + // filter `operation` used during line filter. + lineOp labels.MatchType } func newIPFilter(pattern string) *IPFilter { @@ -60,15 +64,18 @@ func newIPFilter(pattern string) *IPFilter { } // NewIPLineFilter is used to construct ip filter as a `LineFilter` -func NewIPLineFilter(pattern string) *IPFilter { - return newIPFilter(pattern) +func NewIPLineFilter(pattern string, op labels.MatchType) *IPFilter { + filter := newIPFilter(pattern) + filter.matchType = IPMatchLine + filter.lineOp = op + return filter } // NewIPLabelFilter is used to construct ip filter as label filter for the given `label`. func NewIPLabelFilter(label string, op LabelFilterType, pattern string) *IPFilter { filter := newIPFilter(pattern) - filter.labelName = label filter.matchType = IPMatchLabel + filter.labelName = label filter.labelOp = op return filter } @@ -120,9 +127,16 @@ func (ipf *IPFilter) filter(line []byte) bool { return false } -// Filter implement `Filterer` interface. +// Filter implement `Filterer` interface. Used by `LineFilter` func (ipf *IPFilter) Filter(line []byte) bool { - return ipf.filter(line) + ok := ipf.filter(line) + + fmt.Println("linefilter here?", "op", ipf.lineOp, "equal?", ipf.lineOp == labels.MatchNotEqual) + + if ipf.lineOp == labels.MatchNotEqual { + return !ok + } + return ok } // ToStage implements `Filterer` interface. @@ -139,25 +153,29 @@ func (ipf *IPFilter) Process(line []byte, lbs *LabelsBuilder) ([]byte, bool) { return line, false } - if ipf.matchType == IPMatchLabel { + switch ipf.matchType { + case IPMatchLine: + if ipf.lineOp == labels.MatchNotEqual { + return line, !ipf.filter(line) + } + return line, ipf.filter(line) + case IPMatchLabel: v, ok := lbs.Get(ipf.labelName) if !ok { // we have not found the corresponding label. return line, false } - - input := []byte(v) switch ipf.labelOp { case LabelFilterEqual: - return line, ipf.filter(input) + return line, ipf.filter([]byte(v)) case LabelFilterNotEqual: - return line, !ipf.filter(input) + return line, !ipf.filter([]byte(v)) default: lbs.SetErr(ErrIPFilterInvalidOperation.Error()) + return line, false } } - - return line, ipf.filter(line) + return line, false } // `RequiredLabelNames` implements `Stage` interface @@ -167,8 +185,12 @@ func (ipf *IPFilter) RequiredLabelNames() []string { // `String` implements fmt.Stringer inteface, by which also implements `LabelFilterer` inteface. func (ipf *IPFilter) String() string { + eq := "=" // LabelMatchNotEqual emits `==` string. which we don't want. if ipf.matchType == IPMatchLabel { - return fmt.Sprintf("%s=ip(%q)", ipf.labelName, ipf.pattern) + if ipf.labelOp == LabelFilterNotEqual { + eq = "!=" + } + return fmt.Sprintf("%s%sip(%q)", ipf.labelName, eq, ipf.pattern) } return fmt.Sprintf("ip(%q)", ipf.pattern) } diff --git a/pkg/logql/log/ip_test.go b/pkg/logql/log/ip_test.go index c6318df7481fc..fc00f92ed28f8 100644 --- a/pkg/logql/log/ip_test.go +++ b/pkg/logql/log/ip_test.go @@ -152,10 +152,10 @@ func Test_IPLabelFilterOp(t *testing.T) { { name: "not equal operator", pat: "192.168.0.2", - op: LabelFilterEqual, + op: LabelFilterNotEqual, label: "addr", - val: []byte("192.168.0.1"), // no match - expectedMatch: false, + val: []byte("192.168.0.1"), // match because !=ip("192.168.0.2") + expectedMatch: true, }, { name: "great than operator-not-supported", From eb77d3c3b931712943006b496b35ba63bf3a0671 Mon Sep 17 00:00:00 2001 From: Kaviraj Date: Wed, 14 Jul 2021 18:28:27 +0200 Subject: [PATCH 13/24] Split Line and Label filter to different struct. It got out of hands to handle both in same struct. Signed-off-by: Kaviraj --- pkg/logql/ast.go | 11 +- pkg/logql/expr.y | 4 +- pkg/logql/expr.y.go | 4 +- pkg/logql/log/ip.go | 236 +++++++++++++++++++++------------------ pkg/logql/log/ip_test.go | 91 +++++++++++---- pkg/logql/parser_test.go | 20 ++-- 6 files changed, 217 insertions(+), 149 deletions(-) diff --git a/pkg/logql/ast.go b/pkg/logql/ast.go index 20bf03e3ef606..1768558a7ce51 100644 --- a/pkg/logql/ast.go +++ b/pkg/logql/ast.go @@ -308,14 +308,11 @@ func (e *lineFilterExpr) Filter() (log.Filterer, error) { var f log.Filterer if e.op == OpFilterIP { - // ip() supports only |= (matchEqual) and !=(matchNotEqual) matchtypes - switch e.ty { - case labels.MatchEqual, labels.MatchNotEqual: - default: - return nil, log.ErrIPFilterInvalidOperation + var err error + f, err = log.NewIPLineFilter(e.match, e.ty) + if err != nil { + return nil, err } - - f = log.NewIPLineFilter(e.match, e.ty) } else { var err error // to avoid `f` being shadowed. f, err = log.NewFilter(e.match, e.ty) diff --git a/pkg/logql/expr.y b/pkg/logql/expr.y index 0a547bf0c3416..0c4b5ab09b9ba 100644 --- a/pkg/logql/expr.y +++ b/pkg/logql/expr.y @@ -304,8 +304,8 @@ jsonExpressionList: ; ipLabelFilter: - IDENTIFIER EQ IP OPEN_PARENTHESIS STRING CLOSE_PARENTHESIS { $$ = log.NewIPLabelFilter($1, log.LabelFilterEqual, $5) } - | IDENTIFIER NEQ IP OPEN_PARENTHESIS STRING CLOSE_PARENTHESIS { $$ = log.NewIPLabelFilter($1, log.LabelFilterNotEqual, $5) } + IDENTIFIER EQ IP OPEN_PARENTHESIS STRING CLOSE_PARENTHESIS { $$ = log.NewIPLabelFilter($5, $1,log.LabelFilterEqual) } + | IDENTIFIER NEQ IP OPEN_PARENTHESIS STRING CLOSE_PARENTHESIS { $$ = log.NewIPLabelFilter($5, $1, log.LabelFilterNotEqual) } ; unitFilter: diff --git a/pkg/logql/expr.y.go b/pkg/logql/expr.y.go index 9110b927da334..fb9888cd818f6 100644 --- a/pkg/logql/expr.y.go +++ b/pkg/logql/expr.y.go @@ -1324,12 +1324,12 @@ exprdefault: case 107: exprDollar = exprS[exprpt-6 : exprpt+1] { - exprVAL.IPLabelFilter = log.NewIPLabelFilter(exprDollar[1].str, log.LabelFilterEqual, exprDollar[5].str) + exprVAL.IPLabelFilter = log.NewIPLabelFilter(exprDollar[5].str, exprDollar[1].str, log.LabelFilterEqual) } case 108: exprDollar = exprS[exprpt-6 : exprpt+1] { - exprVAL.IPLabelFilter = log.NewIPLabelFilter(exprDollar[1].str, log.LabelFilterNotEqual, exprDollar[5].str) + exprVAL.IPLabelFilter = log.NewIPLabelFilter(exprDollar[5].str, exprDollar[1].str, log.LabelFilterNotEqual) } case 109: exprDollar = exprS[exprpt-1 : exprpt+1] diff --git a/pkg/logql/log/ip.go b/pkg/logql/log/ip.go index 80a9b99f8b615..1e9fc48e9b835 100644 --- a/pkg/logql/log/ip.go +++ b/pkg/logql/log/ip.go @@ -19,70 +19,156 @@ type IPMatchType int const ( IPv4Charset = "0123456789." IPv6Charset = "0123456789abcdefABCDEF:." - - IPMatchLine IPMatchType = iota - IPMatchLabel ) // Should be one of the netaddr.IP, netaddr.IPRange, netadd.IPPrefix. type IPMatcher interface{} -// IPFilter search for IP addresses of given `pattern` in the given `line`. -// It returns true if pattern is matched with at least one IP in the `line` +type IPLineFilter struct { + ip *ipFilter + ty labels.MatchType +} -// pattern - can be of the following form for both IPv4 and IPv6. -// 1. SINGLE-IP - "192.168.0.1" -// 2. IP RANGE - "192.168.0.1-192.168.0.23" -// 3. CIDR - "192.168.0.0/16" -type IPFilter struct { - pattern string - matcher IPMatcher - matchType IPMatchType +// NewIPLineFilter is used to construct ip filter as a `LineFilter` +func NewIPLineFilter(pattern string, ty labels.MatchType) (*IPLineFilter, error) { + // check if `ty` supported in ip matcher. + switch ty { + case labels.MatchEqual, labels.MatchNotEqual: + default: + return nil, ErrIPFilterInvalidOperation + } + + ip, err := newIPFilter(pattern) + if err != nil { + return nil, err + } + return &IPLineFilter{ + ip: ip, + ty: ty, + }, nil +} + +// Filter implement `Filterer` interface. Used by `LineFilter` +func (f *IPLineFilter) Filter(line []byte) bool { + return f.filterTy(line, f.ty) +} + +// ToStage implements `Filterer` interface. +func (f *IPLineFilter) ToStage() Stage { + return f +} + +// `Process` implements `Stage` interface +func (f *IPLineFilter) Process(line []byte, _ *LabelsBuilder) ([]byte, bool) { + return line, f.filterTy(line, f.ty) +} + +// `RequiredLabelNames` implements `Stage` interface +func (f *IPLineFilter) RequiredLabelNames() []string { + return []string{} // empty for line filter +} + +func (f *IPLineFilter) filterTy(line []byte, ty labels.MatchType) bool { + if ty == labels.MatchNotEqual { + return !f.ip.filter(line) + } + return f.ip.filter(line) +} + +// `String` implements fmt.Stringer inteface, by which also implements `LabelFilterer` inteface. +func (f *IPLineFilter) String() string { + // return fmt.Sprintf("%s%sip(%q)", ipf.labelName, eq, ipf.pattern) // label filter + return fmt.Sprintf("ip(%q)", f.ip.pattern) +} + +type IPLabelFilter struct { + ip *ipFilter + ty LabelFilterType // if used as label matcher, this holds the identifier label name. // e.g: (|remote_addr = ip("xxx")). Here labelName is `remote_addr` - labelName string + label string - // patternError represents any invalid pattern provided to match the ip. - patternError error + // patError records if given pattern is invalid. + patError error +} + +// NewIPLabelFilter is used to construct ip filter as label filter for the given `label`. +func NewIPLabelFilter(pattern string, label string, ty LabelFilterType) *IPLabelFilter { + ip, err := newIPFilter(pattern) + return &IPLabelFilter{ + ip: ip, + label: label, + ty: ty, + patError: err, + } +} - // filter `operation` used during label filter. - labelOp LabelFilterType +// `Process` implements `Stage` interface +func (f *IPLabelFilter) Process(line []byte, lbs *LabelsBuilder) ([]byte, bool) { + return line, f.filterTy(line, f.ty, lbs) +} - // filter `operation` used during line filter. - lineOp labels.MatchType +// `RequiredLabelNames` implements `Stage` interface +func (f *IPLabelFilter) RequiredLabelNames() []string { + return []string{f.label} } -func newIPFilter(pattern string) *IPFilter { - filter := &IPFilter{pattern: pattern} +func (f *IPLabelFilter) filterTy(line []byte, ty LabelFilterType, lbs *LabelsBuilder) bool { - matcher, err := getMatcher(pattern) - filter.matcher = matcher - filter.patternError = err + input, ok := lbs.Get(f.label) + if !ok { + lbs.SetErr(fmt.Sprintf("%s: %q %s", errLabelFilter, f.label, "label not found")) + return false + } - return filter + switch ty { + case LabelFilterEqual: + return f.ip.filter([]byte(input)) + case LabelFilterNotEqual: + return !f.ip.filter([]byte(line)) + } + // any other operation not supported + lbs.SetErr(fmt.Sprintf("%q, %s", ty, ErrIPFilterInvalidOperation)) + return false } -// NewIPLineFilter is used to construct ip filter as a `LineFilter` -func NewIPLineFilter(pattern string, op labels.MatchType) *IPFilter { - filter := newIPFilter(pattern) - filter.matchType = IPMatchLine - filter.lineOp = op - return filter +// `String` implements fmt.Stringer inteface, by which also implements `LabelFilterer` inteface. +func (f *IPLabelFilter) String() string { + eq := "=" // LabelFilterEqual -> "==", we don't want in string representation of ip label filter. + if f.ty == LabelFilterNotEqual { + eq = LabelFilterNotEqual.String() + } + return fmt.Sprintf("%s%sip(%q)", f.label, eq, f.ip.pattern) // label filter } -// NewIPLabelFilter is used to construct ip filter as label filter for the given `label`. -func NewIPLabelFilter(label string, op LabelFilterType, pattern string) *IPFilter { - filter := newIPFilter(pattern) - filter.matchType = IPMatchLabel - filter.labelName = label - filter.labelOp = op - return filter +// ipFilter search for IP addresses of given `pattern` in the given `line`. +// It returns true if pattern is matched with at least one IP in the `line` + +// pattern - can be of the following form for both IPv4 and IPv6. +// 1. SINGLE-IP - "192.168.0.1" +// 2. IP RANGE - "192.168.0.1-192.168.0.23" +// 3. CIDR - "192.168.0.0/16" +type ipFilter struct { + pattern string + matcher IPMatcher +} + +func newIPFilter(pattern string) (*ipFilter, error) { + filter := &ipFilter{pattern: pattern} + + matcher, err := getMatcher(pattern) + if err != nil { + return nil, err + } + filter.matcher = matcher + + return filter, nil } // filter does the heavy lifting finding ip `pattern` in the givin `line`. // This is the function if you want to understand how the core logic how ip filter works! -func (ipf *IPFilter) filter(line []byte) bool { +func (f *ipFilter) filter(line []byte) bool { if len(line) == 0 { return false } @@ -96,7 +182,7 @@ func (ipf *IPFilter) filter(line []byte) bool { } ip, err := netaddr.ParseIP(string(line[start : start+iplen])) if err == nil { - if contains(ipf.matcher, ip) { + if contains(f.matcher, ip) { return true, 0 } } @@ -127,74 +213,6 @@ func (ipf *IPFilter) filter(line []byte) bool { return false } -// Filter implement `Filterer` interface. Used by `LineFilter` -func (ipf *IPFilter) Filter(line []byte) bool { - ok := ipf.filter(line) - - fmt.Println("linefilter here?", "op", ipf.lineOp, "equal?", ipf.lineOp == labels.MatchNotEqual) - - if ipf.lineOp == labels.MatchNotEqual { - return !ok - } - return ok -} - -// ToStage implements `Filterer` interface. -func (ipf *IPFilter) ToStage() Stage { - return ipf -} - -// `Process` implements `Stage` interface -func (ipf *IPFilter) Process(line []byte, lbs *LabelsBuilder) ([]byte, bool) { - - // make sure the pattern provided was valid, even before trying to match. - if ipf.patternError != nil { - lbs.SetErr(fmt.Errorf("%s: %s", errLabelFilter, ipf.patternError.Error()).Error()) - return line, false - } - - switch ipf.matchType { - case IPMatchLine: - if ipf.lineOp == labels.MatchNotEqual { - return line, !ipf.filter(line) - } - return line, ipf.filter(line) - case IPMatchLabel: - v, ok := lbs.Get(ipf.labelName) - if !ok { - // we have not found the corresponding label. - return line, false - } - switch ipf.labelOp { - case LabelFilterEqual: - return line, ipf.filter([]byte(v)) - case LabelFilterNotEqual: - return line, !ipf.filter([]byte(v)) - default: - lbs.SetErr(ErrIPFilterInvalidOperation.Error()) - return line, false - } - } - return line, false -} - -// `RequiredLabelNames` implements `Stage` interface -func (ipf *IPFilter) RequiredLabelNames() []string { - return []string{ipf.labelName} -} - -// `String` implements fmt.Stringer inteface, by which also implements `LabelFilterer` inteface. -func (ipf *IPFilter) String() string { - eq := "=" // LabelMatchNotEqual emits `==` string. which we don't want. - if ipf.matchType == IPMatchLabel { - if ipf.labelOp == LabelFilterNotEqual { - eq = "!=" - } - return fmt.Sprintf("%s%sip(%q)", ipf.labelName, eq, ipf.pattern) - } - return fmt.Sprintf("ip(%q)", ipf.pattern) -} - func contains(matcher IPMatcher, ip netaddr.IP) bool { switch m := matcher.(type) { case netaddr.IP: diff --git a/pkg/logql/log/ip_test.go b/pkg/logql/log/ip_test.go index fc00f92ed28f8..ef5cd19d5fb0b 100644 --- a/pkg/logql/log/ip_test.go +++ b/pkg/logql/log/ip_test.go @@ -111,12 +111,12 @@ func Test_IPFilter(t *testing.T) { for _, c := range cases { t.Run(c.name, func(t *testing.T) { - ip := newIPFilter(c.pat) + ip, err := newIPFilter(c.pat) if c.fail { - assert.Error(t, c.err, ip.patternError) + assert.Error(t, c.err, err) return } - assert.NoError(t, ip.patternError) + assert.NoError(t, err) got := make([]int, 0) for i, in := range c.input { @@ -129,11 +129,11 @@ func Test_IPFilter(t *testing.T) { } } -func Test_IPLabelFilterOp(t *testing.T) { +func Test_IPLabelFilterTy(t *testing.T) { cases := []struct { name string pat string - op LabelFilterType + ty LabelFilterType label string val []byte expectedMatch bool @@ -144,7 +144,7 @@ func Test_IPLabelFilterOp(t *testing.T) { { name: "equal operator", pat: "192.168.0.1", - op: LabelFilterEqual, + ty: LabelFilterEqual, label: "addr", val: []byte("192.168.0.1"), expectedMatch: true, @@ -152,7 +152,7 @@ func Test_IPLabelFilterOp(t *testing.T) { { name: "not equal operator", pat: "192.168.0.2", - op: LabelFilterNotEqual, + ty: LabelFilterNotEqual, label: "addr", val: []byte("192.168.0.1"), // match because !=ip("192.168.0.2") expectedMatch: true, @@ -160,7 +160,7 @@ func Test_IPLabelFilterOp(t *testing.T) { { name: "great than operator-not-supported", pat: "192.168.0.2", - op: LabelFilterGreaterThan, // not supported + ty: LabelFilterGreaterThan, // not supported label: "addr", val: []byte("192.168.0.1"), fail: true, @@ -169,7 +169,7 @@ func Test_IPLabelFilterOp(t *testing.T) { { name: "great than or equal to operator-not-supported", pat: "192.168.0.2", - op: LabelFilterGreaterThanOrEqual, // not supported + ty: LabelFilterGreaterThanOrEqual, // not supported label: "addr", val: []byte("192.168.0.1"), fail: true, @@ -178,7 +178,7 @@ func Test_IPLabelFilterOp(t *testing.T) { { name: "less than operator-not-supported", pat: "192.168.0.2", - op: LabelFilterLesserThan, // not supported + ty: LabelFilterLesserThan, // not supported label: "addr", val: []byte("192.168.0.1"), fail: true, @@ -187,7 +187,7 @@ func Test_IPLabelFilterOp(t *testing.T) { { name: "less than or equal to operator-not-supported", pat: "192.168.0.2", - op: LabelFilterLesserThanOrEqual, // not supported + ty: LabelFilterLesserThanOrEqual, // not supported label: "addr", val: []byte("192.168.0.1"), fail: true, @@ -197,8 +197,8 @@ func Test_IPLabelFilterOp(t *testing.T) { for _, c := range cases { t.Run(c.name, func(t *testing.T) { - lf := NewIPLabelFilter(c.label, c.op, c.pat) - require.NoError(t, lf.patternError) + lf := NewIPLabelFilter(c.pat, c.label, c.ty) + require.NoError(t, lf.patError) lbs := labels.Labels{labels.Label{Name: c.label, Value: string(c.val)}} lbb := NewBaseLabelsBuilder().ForLabels(lbs, lbs.Hash()) @@ -214,6 +214,62 @@ func Test_IPLabelFilterOp(t *testing.T) { } } +func Test_IPLineFilterTy(t *testing.T) { + cases := []struct { + name string + pat string + ty labels.MatchType + line []byte + expectedMatch bool + + fail bool + err error + }{ + { + name: "equal operator", + pat: "192.168.0.1", + ty: labels.MatchEqual, + line: []byte("192.168.0.1"), + expectedMatch: true, + }, + { + name: "not equal operator", + pat: "192.168.0.2", + ty: labels.MatchNotEqual, + line: []byte("192.168.0.1"), // match because !=ip("192.168.0.2") + expectedMatch: true, + }, + { + name: "regex not equal", + pat: "192.168.0.2", + ty: labels.MatchNotRegexp, // not supported + line: []byte("192.168.0.1"), + fail: true, + err: ErrIPFilterInvalidOperation, + }, + { + name: "regex equal", + pat: "192.168.0.2", + ty: labels.MatchRegexp, // not supported + line: []byte("192.168.0.1"), + fail: true, + err: ErrIPFilterInvalidOperation, + }, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + lf, err := NewIPLineFilter(c.pat, c.ty) + if c.fail { + require.Error(t, err) + return + } + ok := lf.filterTy(c.line, c.ty) + assert.Equal(t, c.expectedMatch, ok) + }) + } +} + func Benchmark_IPFilter(b *testing.B) { b.ReportAllocs() @@ -222,8 +278,6 @@ func Benchmark_IPFilter(b *testing.B) { []byte(`vpn connected to vm just wanted to make some long line without match`), []byte(`vpn ::1 connected to vm just wanted to make some long line with match at the end 127.0.0.1`), } - lbbb := NewBaseLabelsBuilder() - lbb := lbbb.ForLabels(labels.Labels{labels.Label{Name: "foo", Value: "bar"}}, 0) for _, pattern := range []string{ "127.0.0.1", @@ -231,14 +285,13 @@ func Benchmark_IPFilter(b *testing.B) { "192.168.4.5/16", } { b.Run(pattern, func(b *testing.B) { - stage := newIPFilter(pattern) - require.Nil(b, stage.patternError) + stage, err := newIPFilter(pattern) + require.NoError(b, err) b.ResetTimer() for n := 0; n < b.N; n++ { for _, l := range line { - lbb.Reset() - _, _ = stage.Process(l, lbb) + _ = stage.filter(l) } } }) diff --git a/pkg/logql/parser_test.go b/pkg/logql/parser_test.go index e37e004f56826..57a5412d46f19 100644 --- a/pkg/logql/parser_test.go +++ b/pkg/logql/parser_test.go @@ -456,14 +456,14 @@ func TestParse(t *testing.T) { in: `{ foo = "bar" }|logfmt|addr=ip("1.2.3.4")`, exp: newPipelineExpr( newMatcherExpr([]*labels.Matcher{mustNewMatcher(labels.MatchEqual, "foo", "bar")}), - MultiStageExpr{newLabelParserExpr(OpParserTypeLogfmt, ""), newLabelFilterExpr(log.NewIPLabelFilter("addr", log.LabelFilterEqual, "1.2.3.4"))}, + MultiStageExpr{newLabelParserExpr(OpParserTypeLogfmt, ""), newLabelFilterExpr(log.NewIPLabelFilter("1.2.3.4", "addr", log.LabelFilterEqual))}, ), }, { in: `{ foo = "bar" }|logfmt|addr!=ip("1.2.3.4")`, exp: newPipelineExpr( newMatcherExpr([]*labels.Matcher{mustNewMatcher(labels.MatchEqual, "foo", "bar")}), - MultiStageExpr{newLabelParserExpr(OpParserTypeLogfmt, ""), newLabelFilterExpr(log.NewIPLabelFilter("addr", log.LabelFilterNotEqual, "1.2.3.4"))}, + MultiStageExpr{newLabelParserExpr(OpParserTypeLogfmt, ""), newLabelFilterExpr(log.NewIPLabelFilter("1.2.3.4", "addr", log.LabelFilterNotEqual))}, ), }, { @@ -473,7 +473,7 @@ func TestParse(t *testing.T) { MultiStageExpr{ newLabelParserExpr(OpParserTypeLogfmt, ""), newLabelFilterExpr(log.NewStringLabelFilter(mustNewMatcher(labels.MatchEqual, "level", "error"))), - newLabelFilterExpr(log.NewIPLabelFilter("addr", log.LabelFilterEqual, "1.2.3.4")), + newLabelFilterExpr(log.NewIPLabelFilter("1.2.3.4", "addr", log.LabelFilterEqual)), }, ), }, @@ -484,7 +484,7 @@ func TestParse(t *testing.T) { MultiStageExpr{ newLabelParserExpr(OpParserTypeLogfmt, ""), newLabelFilterExpr(log.NewStringLabelFilter(mustNewMatcher(labels.MatchEqual, "level", "error"))), - newLabelFilterExpr(log.NewIPLabelFilter("addr", log.LabelFilterNotEqual, "1.2.3.4")), + newLabelFilterExpr(log.NewIPLabelFilter("1.2.3.4", "addr", log.LabelFilterNotEqual)), }, ), }, @@ -494,9 +494,9 @@ func TestParse(t *testing.T) { newMatcherExpr([]*labels.Matcher{mustNewMatcher(labels.MatchEqual, "foo", "bar")}), MultiStageExpr{ newLabelParserExpr(OpParserTypeLogfmt, ""), - newLabelFilterExpr(log.NewIPLabelFilter("remote_addr", log.LabelFilterEqual, "2.3.4.5")), + newLabelFilterExpr(log.NewIPLabelFilter("2.3.4.5", "remote_addr", log.LabelFilterEqual)), newLabelFilterExpr(log.NewStringLabelFilter(mustNewMatcher(labels.MatchEqual, "level", "error"))), - newLabelFilterExpr(log.NewIPLabelFilter("addr", log.LabelFilterEqual, "1.2.3.4")), + newLabelFilterExpr(log.NewIPLabelFilter("1.2.3.4", "addr", log.LabelFilterEqual)), }, ), }, @@ -506,9 +506,9 @@ func TestParse(t *testing.T) { newMatcherExpr([]*labels.Matcher{mustNewMatcher(labels.MatchEqual, "foo", "bar")}), MultiStageExpr{ newLabelParserExpr(OpParserTypeLogfmt, ""), - newLabelFilterExpr(log.NewIPLabelFilter("remote_addr", log.LabelFilterNotEqual, "2.3.4.5")), + newLabelFilterExpr(log.NewIPLabelFilter("2.3.4.5", "remote_addr", log.LabelFilterNotEqual)), newLabelFilterExpr(log.NewStringLabelFilter(mustNewMatcher(labels.MatchEqual, "level", "error"))), - newLabelFilterExpr(log.NewIPLabelFilter("addr", log.LabelFilterNotEqual, "1.2.3.4")), + newLabelFilterExpr(log.NewIPLabelFilter("1.2.3.4", "addr", log.LabelFilterNotEqual)), }, ), }, @@ -518,9 +518,9 @@ func TestParse(t *testing.T) { newMatcherExpr([]*labels.Matcher{mustNewMatcher(labels.MatchEqual, "foo", "bar")}), MultiStageExpr{ newLabelParserExpr(OpParserTypeLogfmt, ""), - newLabelFilterExpr(log.NewIPLabelFilter("remote_addr", log.LabelFilterEqual, "2.3.4.5")), + newLabelFilterExpr(log.NewIPLabelFilter("2.3.4.5", "remote_addr", log.LabelFilterEqual)), newLabelFilterExpr(log.NewStringLabelFilter(mustNewMatcher(labels.MatchEqual, "level", "error"))), - newLabelFilterExpr(log.NewIPLabelFilter("addr", log.LabelFilterNotEqual, "1.2.3.4")), + newLabelFilterExpr(log.NewIPLabelFilter("1.2.3.4", "addr", log.LabelFilterNotEqual)), }, ), }, From 4302e8045952a55fb74a5d7361e5ac0ac8d0be1b Mon Sep 17 00:00:00 2001 From: Kaviraj Date: Thu, 15 Jul 2021 02:25:15 +0200 Subject: [PATCH 14/24] Add basic doc for IP matcher Signed-off-by: Kaviraj --- docs/sources/logql/_index.md | 4 ++ docs/sources/logql/ip.md | 74 ++++++++++++++++++++++++++++++++++++ 2 files changed, 78 insertions(+) create mode 100644 docs/sources/logql/ip.md diff --git a/docs/sources/logql/_index.md b/docs/sources/logql/_index.md index fd29dd9ad11e1..f49fe9c1a560d 100644 --- a/docs/sources/logql/_index.md +++ b/docs/sources/logql/_index.md @@ -135,6 +135,8 @@ While line filter expressions could be placed anywhere in a pipeline, it is almo For example, while the result will be the same, the following query `{job="mysql"} |= "error" | json | line_format "{{.err}}"` will always run faster than `{job="mysql"} | json | line_format "{{.message}}" |= "error"`. Line filter expressions are the fastest way to filter logs after log stream selectors. +Line filter also have built-in IP matcher support. It takes the form ` ip("")`. Only two filter operators are supported `|=`(equality) and `!=`(inequality). And `pattern` can be single IP, IP range or CIDR pattern. [Separate doc](./ip.md) explains how to use IP matcher in detail. + #### Parser Expression Parser expression can parse and extract labels from the log content. Those extracted labels can then be used for filtering using [label filter expressions](#label-filter-expression) or for [metric aggregations](#metric-queries). @@ -433,6 +435,8 @@ It will evaluate first `duration >= 20ms or method="GET"`. To evaluate first `me > Label filter expressions are the only expression allowed after the [unwrap expression](#unwrapped-range-aggregations). This is mainly to allow filtering errors from the metric extraction (see [errors](#pipeline-errors)). +Label filter also have built-in IP matcher support. It takes the form ` ip("")`. Only two filter operators are supported `=`(equality) and `!=`(inequality). And `pattern` can be single IP, IP range or CIDR pattern. [Separate doc](./ip.md) explains how to use IP matcher in detail. + #### Line Format Expression The line format expression can rewrite the log line content by using the [text/template](https://golang.org/pkg/text/template/) format. diff --git a/docs/sources/logql/ip.md b/docs/sources/logql/ip.md new file mode 100644 index 0000000000000..3313284d6d0f4 --- /dev/null +++ b/docs/sources/logql/ip.md @@ -0,0 +1,74 @@ +---- +title: IP matcher +---- + +Consider the following logs, + +``` +3.180.71.3 - - [17/May/2015:08:05:32 +0000] "GET /downloads/product_1 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.21)" +80.91.33.133 - - [17/May/2015:08:05:14 +0000] "GET /downloads/product_1 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.16)" +46.4.66.76 - - [17/May/2015:08:05:45 +0000] "GET /downloads/product_1 HTTP/1.1" 404 318 "-" "Debian APT-HTTP/1.3 (1.0.1ubuntu2)" +93.180.71.3 - - [17/May/2015:08:05:26 +0000] "GET /downloads/product_1 HTTP/1.1" 404 324 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.21)" +``` + +How would you use LogQL to search for log lines with IP addresses?. Say single IP to start with? That's easy, we can use LogQL line filter (kinda like distributed grep) + +```logql +{foo="bar"} |= "3.180.71.3" +``` + +will output following log lines. + +``` +3.180.71.3 - - [17/May/2015:08:05:32 +0000] "GET /downloads/product_1 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.21)" +93.180.71.3 - - [17/May/2015:08:05:26 +0000] "GET /downloads/product_1 HTTP/1.1" 404 324 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.21)" +``` + +but wait! what `93.180.71.3` is doing here?. Oh yea. it actually matches with what we queried for `3.180.71.3`. It can also match with other log lines which we don't actually want. The right option would be to use regexp to match the IP address(something like `|~"^3.180.71.3"`). + +Now forget about single IP, what about range of IP addresses? What about IP subnet? Or even more interesting IPv6 addresses? (not easy to come up with regexp for all the use cases). + +Luckily, LogQL comes with built-in support for IP matcher. + +It supports both IPv4 and IPv6 addresses. + +It takes following syntax + +``` +ip("") +``` + +Where pattern can be one of the following. + +1. Single IP - e.g: `ip("192.168.0.1")`, `ip("::1")` +2. IP Range - e.g: `ip("192.168.0.1-192.189.10.12")`, `ip("2001:db8::1-2001:db8::8")` +3. CIDR pattern - e.g: `ip("192.168.4.5/16")`, `ip("2001:db8::/32")` + +IP matcher can be used in both Line Filter and Label Filter expressions. + +Examples: +- Line Filter + +```logql +`{ foo = "bar" }|=ip("192.168.4.5/16") +``` + +Following query return log lines that *doesn't* match with given IPv4 range + +```logql +`{ foo = "bar" }!=ip("192.168.4.5-192.168.4.20") +``` + +- Label Filter + +```logql +{ foo = "bar" }|logfmt|remote_addr=ip("2001:db8::1-2001:db8::8")|level="error" +``` + +Can also be chained multiple times. For example, following query match log lines with all IPv4 subnet `192.168.4.5/16` except the IP `192.168.4.2` + +```logql +{ foo = "bar" }|logfmt|addr=ip("192.168.4.5/16")| addr!=ip("192.168.4.2") +``` + +When using as Line Filter, only `|=` and `!=` operations are allowed. When using as Label Filter, operations `=` and `!=` are allowed. From 80382a42d9171aa770170dcd4ed940ffcddffdcd Mon Sep 17 00:00:00 2001 From: Kaviraj Date: Mon, 19 Jul 2021 10:15:31 +0200 Subject: [PATCH 15/24] Add docs feedback from Karen --- docs/sources/logql/_index.md | 4 +- docs/sources/logql/ip.md | 90 ++++++++++++++++-------------------- 2 files changed, 43 insertions(+), 51 deletions(-) diff --git a/docs/sources/logql/_index.md b/docs/sources/logql/_index.md index f49fe9c1a560d..2d768b93bc4cb 100644 --- a/docs/sources/logql/_index.md +++ b/docs/sources/logql/_index.md @@ -135,7 +135,7 @@ While line filter expressions could be placed anywhere in a pipeline, it is almo For example, while the result will be the same, the following query `{job="mysql"} |= "error" | json | line_format "{{.err}}"` will always run faster than `{job="mysql"} | json | line_format "{{.message}}" |= "error"`. Line filter expressions are the fastest way to filter logs after log stream selectors. -Line filter also have built-in IP matcher support. It takes the form ` ip("")`. Only two filter operators are supported `|=`(equality) and `!=`(inequality). And `pattern` can be single IP, IP range or CIDR pattern. [Separate doc](./ip.md) explains how to use IP matcher in detail. +Line filter expressions have support matching IP addresses. See [Matching IP addresses](ip/) for details. #### Parser Expression @@ -435,7 +435,7 @@ It will evaluate first `duration >= 20ms or method="GET"`. To evaluate first `me > Label filter expressions are the only expression allowed after the [unwrap expression](#unwrapped-range-aggregations). This is mainly to allow filtering errors from the metric extraction (see [errors](#pipeline-errors)). -Label filter also have built-in IP matcher support. It takes the form ` ip("")`. Only two filter operators are supported `=`(equality) and `!=`(inequality). And `pattern` can be single IP, IP range or CIDR pattern. [Separate doc](./ip.md) explains how to use IP matcher in detail. +Label filter expressions have support matching IP addresses. See [Matching IP addresses](ip/) for details. #### Line Format Expression diff --git a/docs/sources/logql/ip.md b/docs/sources/logql/ip.md index 3313284d6d0f4..87ab5c527e3e2 100644 --- a/docs/sources/logql/ip.md +++ b/docs/sources/logql/ip.md @@ -1,8 +1,15 @@ ----- -title: IP matcher ----- +--- +title: Matching IP addresses +weight: 30 +--- -Consider the following logs, +# Matching IP addresses + +Matching IP addresses is an experimental feature. + +LogQL supports matching IP addresses. + +With logs such as ``` 3.180.71.3 - - [17/May/2015:08:05:32 +0000] "GET /downloads/product_1 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.21)" @@ -11,64 +18,49 @@ Consider the following logs, 93.180.71.3 - - [17/May/2015:08:05:26 +0000] "GET /downloads/product_1 HTTP/1.1" 404 324 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.21)" ``` -How would you use LogQL to search for log lines with IP addresses?. Say single IP to start with? That's easy, we can use LogQL line filter (kinda like distributed grep) +the LogQL line filter is not sufficient. +A line filter such as ```logql -{foo="bar"} |= "3.180.71.3" -``` - -will output following log lines. - +{job_name="myapp"} |= "3.180.71.3" ``` -3.180.71.3 - - [17/May/2015:08:05:32 +0000] "GET /downloads/product_1 HTTP/1.1" 304 0 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.21)" -93.180.71.3 - - [17/May/2015:08:05:26 +0000] "GET /downloads/product_1 HTTP/1.1" 404 324 "-" "Debian APT-HTTP/1.3 (0.8.16~exp12ubuntu10.21)" -``` - -but wait! what `93.180.71.3` is doing here?. Oh yea. it actually matches with what we queried for `3.180.71.3`. It can also match with other log lines which we don't actually want. The right option would be to use regexp to match the IP address(something like `|~"^3.180.71.3"`). - -Now forget about single IP, what about range of IP addresses? What about IP subnet? Or even more interesting IPv6 addresses? (not easy to come up with regexp for all the use cases). - -Luckily, LogQL comes with built-in support for IP matcher. -It supports both IPv4 and IPv6 addresses. +also matches example IP addresses such as 93.180.71.3. A better choice uses a regexp: `|~"^3.180.71.3"`. This regexp does not handle IPv6 addresses, and it does not match a range of IP addresses. -It takes following syntax +The LogQL support for matching IP addresses handles both IPv4 and IPv6 single addresses, as well as ranges within IP addresses +and CIDR patterns. -``` -ip("") -``` - -Where pattern can be one of the following. +Match IP addresses wtih the syntax: `ip("")`. +The `` can be: -1. Single IP - e.g: `ip("192.168.0.1")`, `ip("::1")` -2. IP Range - e.g: `ip("192.168.0.1-192.189.10.12")`, `ip("2001:db8::1-2001:db8::8")` -3. CIDR pattern - e.g: `ip("192.168.4.5/16")`, `ip("2001:db8::/32")` +- A single IP address. Examples: `ip("192.0.2.0")`, `ip("::1")` +- A range within the IP address. Examples: `ip("192.168.0.1-192.189.10.12")`, `ip("2001:db8::1-2001:db8::8")` +- A CIDR specification. Examples: `ip("192.51.100.0/24")`, `ip("2001:db8::/32")` -IP matcher can be used in both Line Filter and Label Filter expressions. +The IP matching can be used in both line filter and label filter expressions. +When specifying line filter expressions, only the `|=` and `!=` operations are allowed. +When specifying label filter expressions, only the `=` and `!=` operations are allowed. -Examples: -- Line Filter - -```logql -`{ foo = "bar" }|=ip("192.168.4.5/16") -``` +- Line filter examples -Following query return log lines that *doesn't* match with given IPv4 range + ```logql + {job_name="myapp"} |= ip("192.168.4.5/16") + ``` -```logql -`{ foo = "bar" }!=ip("192.168.4.5-192.168.4.20") -``` + Return log lines that do not match with an IPv4 range: -- Label Filter + ```logql + {job_name="myapp"} != ip("192.168.4.5-192.168.4.20") + ``` -```logql -{ foo = "bar" }|logfmt|remote_addr=ip("2001:db8::1-2001:db8::8")|level="error" -``` +- Label filter examples -Can also be chained multiple times. For example, following query match log lines with all IPv4 subnet `192.168.4.5/16` except the IP `192.168.4.2` + ```logql + {job_name="myapp"} | logfmt | remote_addr = ip("2001:db8::1-2001:db8::8") | level = "error" + ``` -```logql -{ foo = "bar" }|logfmt|addr=ip("192.168.4.5/16")| addr!=ip("192.168.4.2") -``` + Filters can be chained. This example matches log lines with all IPv4 subnet values `192.168.4.5/16` except IP address `192.168.4.2`: -When using as Line Filter, only `|=` and `!=` operations are allowed. When using as Label Filter, operations `=` and `!=` are allowed. + ```logql + {job_name="myapp"} | logfmt | addr = ip("192.168.4.5/16") | addr != ip("192.168.4.2") + ``` From 0a2663a7985580a422675bb43c2546021139c749 Mon Sep 17 00:00:00 2001 From: Kaviraj Date: Mon, 19 Jul 2021 10:39:00 +0200 Subject: [PATCH 16/24] Label filter add patter error report --- pkg/logql/ast.go | 5 +++++ pkg/logql/log/ip.go | 12 +++++++++++- 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/pkg/logql/ast.go b/pkg/logql/ast.go index 1768558a7ce51..f9b09b9546624 100644 --- a/pkg/logql/ast.go +++ b/pkg/logql/ast.go @@ -399,6 +399,11 @@ func newLabelFilterExpr(filterer log.LabelFilterer) *labelFilterExpr { func (e *labelFilterExpr) Shardable() bool { return true } func (e *labelFilterExpr) Stage() (log.Stage, error) { + switch ip := e.LabelFilterer.(type) { + case *log.IPLabelFilter: + return ip, ip.PatternError() + } + return e.LabelFilterer, nil } diff --git a/pkg/logql/log/ip.go b/pkg/logql/log/ip.go index 1e9fc48e9b835..123d9061dd7da 100644 --- a/pkg/logql/log/ip.go +++ b/pkg/logql/log/ip.go @@ -114,7 +114,17 @@ func (f *IPLabelFilter) RequiredLabelNames() []string { return []string{f.label} } +// PatternError will be used `labelFilter.Stage()` method so that, if the given pattern is wrong +// it retuns proper 400 error to the client of LogQL. +func (f *IPLabelFilter) PatternError() error { + return f.patError +} + func (f *IPLabelFilter) filterTy(line []byte, ty LabelFilterType, lbs *LabelsBuilder) bool { + if f.patError != nil { + lbs.SetErr(f.patError.Error()) + return false + } input, ok := lbs.Get(f.label) if !ok { @@ -126,7 +136,7 @@ func (f *IPLabelFilter) filterTy(line []byte, ty LabelFilterType, lbs *LabelsBui case LabelFilterEqual: return f.ip.filter([]byte(input)) case LabelFilterNotEqual: - return !f.ip.filter([]byte(line)) + return !f.ip.filter([]byte(input)) } // any other operation not supported lbs.SetErr(fmt.Sprintf("%q, %s", ty, ErrIPFilterInvalidOperation)) From 2c3b649ded307271cf3875f2931100993f9b2a19 Mon Sep 17 00:00:00 2001 From: Kaviraj Date: Mon, 19 Jul 2021 11:23:54 +0200 Subject: [PATCH 17/24] PR remarks 1. Indendation fix 2. Remove string() interface for ip label filte --- pkg/logql/expr.y | 4 ++-- pkg/logql/log/ip.go | 6 ------ 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/pkg/logql/expr.y b/pkg/logql/expr.y index 0c4b5ab09b9ba..38ce93ffca95c 100644 --- a/pkg/logql/expr.y +++ b/pkg/logql/expr.y @@ -248,8 +248,8 @@ filterOp: ; lineFilter: - filter STRING { $$ = newLineFilterExpr($1, "", $2,) } - | filter filterOp OPEN_PARENTHESIS STRING CLOSE_PARENTHESIS { $$ = newLineFilterExpr($1, $2, $4)} + filter STRING { $$ = newLineFilterExpr($1, "", $2) } + | filter filterOp OPEN_PARENTHESIS STRING CLOSE_PARENTHESIS { $$ = newLineFilterExpr($1, $2, $4) } ; lineFilters: diff --git a/pkg/logql/log/ip.go b/pkg/logql/log/ip.go index 123d9061dd7da..a29ce1616c8dd 100644 --- a/pkg/logql/log/ip.go +++ b/pkg/logql/log/ip.go @@ -75,12 +75,6 @@ func (f *IPLineFilter) filterTy(line []byte, ty labels.MatchType) bool { return f.ip.filter(line) } -// `String` implements fmt.Stringer inteface, by which also implements `LabelFilterer` inteface. -func (f *IPLineFilter) String() string { - // return fmt.Sprintf("%s%sip(%q)", ipf.labelName, eq, ipf.pattern) // label filter - return fmt.Sprintf("ip(%q)", f.ip.pattern) -} - type IPLabelFilter struct { ip *ipFilter ty LabelFilterType From b3de98cf287b60d8429825256b6078b816100c54 Mon Sep 17 00:00:00 2001 From: Kaviraj Date: Mon, 19 Jul 2021 11:39:10 +0200 Subject: [PATCH 18/24] Linter happy --- docs/sources/logql/ip.md | 10 ++++++++-- pkg/logql/log/ip.go | 2 +- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/docs/sources/logql/ip.md b/docs/sources/logql/ip.md index 87ab5c527e3e2..f3a972fce6aa2 100644 --- a/docs/sources/logql/ip.md +++ b/docs/sources/logql/ip.md @@ -56,11 +56,17 @@ When specifying label filter expressions, only the `=` and `!=` operations are - Label filter examples ```logql - {job_name="myapp"} | logfmt | remote_addr = ip("2001:db8::1-2001:db8::8") | level = "error" + {job_name="myapp"} + | logfmt + | remote_addr = ip("2001:db8::1-2001:db8::8") + | level = "error" ``` Filters can be chained. This example matches log lines with all IPv4 subnet values `192.168.4.5/16` except IP address `192.168.4.2`: ```logql - {job_name="myapp"} | logfmt | addr = ip("192.168.4.5/16") | addr != ip("192.168.4.2") + {job_name="myapp"} + | logfmt + | addr =f ip("192.168.4.5/16") + | addr != ip("192.168.4.2") ``` diff --git a/pkg/logql/log/ip.go b/pkg/logql/log/ip.go index a29ce1616c8dd..d78277f9e46e0 100644 --- a/pkg/logql/log/ip.go +++ b/pkg/logql/log/ip.go @@ -114,7 +114,7 @@ func (f *IPLabelFilter) PatternError() error { return f.patError } -func (f *IPLabelFilter) filterTy(line []byte, ty LabelFilterType, lbs *LabelsBuilder) bool { +func (f *IPLabelFilter) filterTy(_ []byte, ty LabelFilterType, lbs *LabelsBuilder) bool { if f.patError != nil { lbs.SetErr(f.patError.Error()) return false From f3855c89df3d40102c20f3b1f8a1efc569411403 Mon Sep 17 00:00:00 2001 From: Kaviraj Date: Mon, 19 Jul 2021 12:08:54 +0200 Subject: [PATCH 19/24] PR remaks --- pkg/logql/ast.go | 6 ++++-- pkg/logql/log/ip.go | 13 +++++-------- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/pkg/logql/ast.go b/pkg/logql/ast.go index f9b09b9546624..c15c6f9c89a41 100644 --- a/pkg/logql/ast.go +++ b/pkg/logql/ast.go @@ -307,19 +307,21 @@ func (e *lineFilterExpr) String() string { func (e *lineFilterExpr) Filter() (log.Filterer, error) { var f log.Filterer - if e.op == OpFilterIP { + switch e.op { + case OpFilterIP: var err error f, err = log.NewIPLineFilter(e.match, e.ty) if err != nil { return nil, err } - } else { + default: var err error // to avoid `f` being shadowed. f, err = log.NewFilter(e.match, e.ty) if err != nil { return nil, err } } + if e.left != nil { nextFilter, err := e.left.Filter() if err != nil { diff --git a/pkg/logql/log/ip.go b/pkg/logql/log/ip.go index d78277f9e46e0..c7ab1715a53d1 100644 --- a/pkg/logql/log/ip.go +++ b/pkg/logql/log/ip.go @@ -109,20 +109,19 @@ func (f *IPLabelFilter) RequiredLabelNames() []string { } // PatternError will be used `labelFilter.Stage()` method so that, if the given pattern is wrong -// it retuns proper 400 error to the client of LogQL. +// it returns proper 400 error to the client of LogQL. func (f *IPLabelFilter) PatternError() error { return f.patError } func (f *IPLabelFilter) filterTy(_ []byte, ty LabelFilterType, lbs *LabelsBuilder) bool { - if f.patError != nil { - lbs.SetErr(f.patError.Error()) - return false + if lbs.HasErr() { + // why `true`?. if there's an error only the string matchers can filter out. + return true } - input, ok := lbs.Get(f.label) if !ok { - lbs.SetErr(fmt.Sprintf("%s: %q %s", errLabelFilter, f.label, "label not found")) + // we have not found the label. return false } @@ -132,8 +131,6 @@ func (f *IPLabelFilter) filterTy(_ []byte, ty LabelFilterType, lbs *LabelsBuilde case LabelFilterNotEqual: return !f.ip.filter([]byte(input)) } - // any other operation not supported - lbs.SetErr(fmt.Sprintf("%q, %s", ty, ErrIPFilterInvalidOperation)) return false } From 7bfd12a5e93967c16d34880b1068005211dddd7e Mon Sep 17 00:00:00 2001 From: Kaviraj Date: Mon, 19 Jul 2021 12:40:37 +0200 Subject: [PATCH 20/24] Fix ip label matcher string() method if pattern is invalid --- pkg/logql/log/ip.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/pkg/logql/log/ip.go b/pkg/logql/log/ip.go index c7ab1715a53d1..ab36270dd6136 100644 --- a/pkg/logql/log/ip.go +++ b/pkg/logql/log/ip.go @@ -140,7 +140,14 @@ func (f *IPLabelFilter) String() string { if f.ty == LabelFilterNotEqual { eq = LabelFilterNotEqual.String() } - return fmt.Sprintf("%s%sip(%q)", f.label, eq, f.ip.pattern) // label filter + + pattern := "invalid" + if f.ip != nil { + // may be pattern error + pattern = f.ip.pattern + } + + return fmt.Sprintf("%s%sip(%q)", f.label, eq, pattern) // label filter } // ipFilter search for IP addresses of given `pattern` in the given `line`. From ff3e2ee166b7d739149909fe47b6359a3b3d0ecf Mon Sep 17 00:00:00 2001 From: Kaviraj Date: Mon, 19 Jul 2021 12:55:10 +0200 Subject: [PATCH 21/24] Fix label filter tests --- pkg/logql/log/ip.go | 4 ++++ pkg/logql/log/ip_test.go | 39 +++++---------------------------------- 2 files changed, 9 insertions(+), 34 deletions(-) diff --git a/pkg/logql/log/ip.go b/pkg/logql/log/ip.go index ab36270dd6136..3e7f54d70c1ae 100644 --- a/pkg/logql/log/ip.go +++ b/pkg/logql/log/ip.go @@ -125,6 +125,10 @@ func (f *IPLabelFilter) filterTy(_ []byte, ty LabelFilterType, lbs *LabelsBuilde return false } + if f.ip == nil { + return false + } + switch ty { case LabelFilterEqual: return f.ip.filter([]byte(input)) diff --git a/pkg/logql/log/ip_test.go b/pkg/logql/log/ip_test.go index ef5cd19d5fb0b..bf68d14980dc7 100644 --- a/pkg/logql/log/ip_test.go +++ b/pkg/logql/log/ip_test.go @@ -158,54 +158,25 @@ func Test_IPLabelFilterTy(t *testing.T) { expectedMatch: true, }, { - name: "great than operator-not-supported", - pat: "192.168.0.2", - ty: LabelFilterGreaterThan, // not supported + name: "pattern-invalid", + pat: "192.168.0.2-1.0.0.0", + ty: LabelFilterEqual, // not supported label: "addr", val: []byte("192.168.0.1"), fail: true, - err: ErrIPFilterInvalidOperation.Error(), - }, - { - name: "great than or equal to operator-not-supported", - pat: "192.168.0.2", - ty: LabelFilterGreaterThanOrEqual, // not supported - label: "addr", - val: []byte("192.168.0.1"), - fail: true, - err: ErrIPFilterInvalidOperation.Error(), - }, - { - name: "less than operator-not-supported", - pat: "192.168.0.2", - ty: LabelFilterLesserThan, // not supported - label: "addr", - val: []byte("192.168.0.1"), - fail: true, - err: ErrIPFilterInvalidOperation.Error(), - }, - { - name: "less than or equal to operator-not-supported", - pat: "192.168.0.2", - ty: LabelFilterLesserThanOrEqual, // not supported - label: "addr", - val: []byte("192.168.0.1"), - fail: true, - err: ErrIPFilterInvalidOperation.Error(), + err: ErrIPFilterInvalidPattern.Error(), }, } for _, c := range cases { t.Run(c.name, func(t *testing.T) { lf := NewIPLabelFilter(c.pat, c.label, c.ty) - require.NoError(t, lf.patError) lbs := labels.Labels{labels.Label{Name: c.label, Value: string(c.val)}} lbb := NewBaseLabelsBuilder().ForLabels(lbs, lbs.Hash()) _, ok := lf.Process([]byte("x"), lbb) if c.fail { - assert.NotEmpty(t, lbb.GetErr()) - assert.False(t, ok) + assert.Error(t, lf.patError) return } require.Empty(t, lbb.GetErr()) From e231da3c1eebdd4df14d8c8b648a56e04b44ac5d Mon Sep 17 00:00:00 2001 From: Kaviraj Date: Mon, 19 Jul 2021 12:58:21 +0200 Subject: [PATCH 22/24] Have local copy of `pattern` in label filter to use in string() --- pkg/logql/log/ip.go | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/pkg/logql/log/ip.go b/pkg/logql/log/ip.go index 3e7f54d70c1ae..f7895fd2d8aec 100644 --- a/pkg/logql/log/ip.go +++ b/pkg/logql/log/ip.go @@ -85,6 +85,9 @@ type IPLabelFilter struct { // patError records if given pattern is invalid. patError error + + // local copy of pattern to display it in errors, even though pattern matcher fails because of invalid pattern. + pattern string } // NewIPLabelFilter is used to construct ip filter as label filter for the given `label`. @@ -95,6 +98,7 @@ func NewIPLabelFilter(pattern string, label string, ty LabelFilterType) *IPLabel label: label, ty: ty, patError: err, + pattern: pattern, } } @@ -145,13 +149,7 @@ func (f *IPLabelFilter) String() string { eq = LabelFilterNotEqual.String() } - pattern := "invalid" - if f.ip != nil { - // may be pattern error - pattern = f.ip.pattern - } - - return fmt.Sprintf("%s%sip(%q)", f.label, eq, pattern) // label filter + return fmt.Sprintf("%s%sip(%q)", f.label, eq, f.pattern) // label filter } // ipFilter search for IP addresses of given `pattern` in the given `line`. From b5c5e809ed4228a9bae5a5e7d4f4d474435f4972 Mon Sep 17 00:00:00 2001 From: Kaviraj Date: Thu, 22 Jul 2021 10:45:11 +0200 Subject: [PATCH 23/24] Rebase conflicts --- pkg/logql/expr.y | 20 ++++++++++---------- pkg/logql/expr.y.go | 20 ++++++++++---------- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/pkg/logql/expr.y b/pkg/logql/expr.y index 38ce93ffca95c..ced1d96b2155b 100644 --- a/pkg/logql/expr.y +++ b/pkg/logql/expr.y @@ -15,7 +15,7 @@ import ( Grouping *grouping Labels []string LogExpr LogSelectorExpr - LogRangeExpr *logRange + LogRangeExpr *LogRange Matcher *labels.Matcher Matchers []*labels.Matcher RangeAggregationExpr SampleExpr @@ -32,11 +32,11 @@ import ( bytes uint64 str string duration time.Duration - LiteralExpr *literalExpr + LiteralExpr *LiteralExpr BinOpModifier BinOpOptions - LabelParser *labelParserExpr - LineFilters *lineFilterExpr - LineFilter *lineFilterExpr + LabelParser *LabelParserExpr + LineFilters *LineFilterExpr + LineFilter *LineFilterExpr PipelineExpr MultiStageExpr PipelineStage StageExpr BytesFilter log.LabelFilterer @@ -45,15 +45,15 @@ import ( LabelFilter log.LabelFilterer UnitFilter log.LabelFilterer IPLabelFilter log.LabelFilterer - LineFormatExpr *lineFmtExpr - LabelFormatExpr *labelFmtExpr + LineFormatExpr *LineFmtExpr + LabelFormatExpr *LabelFmtExpr LabelFormat log.LabelFmt LabelsFormat []log.LabelFmt JSONExpressionParser *jsonExpressionParser JSONExpression log.JSONExpression JSONExpressionList []log.JSONExpression - UnwrapExpr *unwrapExpr - OffsetExpr *offsetExpr + UnwrapExpr *UnwrapExpr + OffsetExpr *OffsetExpr } %start root @@ -238,7 +238,7 @@ pipelineStage: lineFilters { $$ = $1 } | PIPE labelParser { $$ = $2 } | PIPE jsonExpressionParser { $$ = $2 } - | PIPE labelFilter { $$ = &labelFilterExpr{LabelFilterer: $2 }} + | PIPE labelFilter { $$ = &LabelFilterExpr{LabelFilterer: $2 }} | PIPE lineFormatExpr { $$ = $2 } | PIPE labelFormatExpr { $$ = $2 } ; diff --git a/pkg/logql/expr.y.go b/pkg/logql/expr.y.go index fb9888cd818f6..757f49e802a94 100644 --- a/pkg/logql/expr.y.go +++ b/pkg/logql/expr.y.go @@ -18,7 +18,7 @@ type exprSymType struct { Grouping *grouping Labels []string LogExpr LogSelectorExpr - LogRangeExpr *logRange + LogRangeExpr *LogRange Matcher *labels.Matcher Matchers []*labels.Matcher RangeAggregationExpr SampleExpr @@ -35,11 +35,11 @@ type exprSymType struct { bytes uint64 str string duration time.Duration - LiteralExpr *literalExpr + LiteralExpr *LiteralExpr BinOpModifier BinOpOptions - LabelParser *labelParserExpr - LineFilters *lineFilterExpr - LineFilter *lineFilterExpr + LabelParser *LabelParserExpr + LineFilters *LineFilterExpr + LineFilter *LineFilterExpr PipelineExpr MultiStageExpr PipelineStage StageExpr BytesFilter log.LabelFilterer @@ -48,15 +48,15 @@ type exprSymType struct { LabelFilter log.LabelFilterer UnitFilter log.LabelFilterer IPLabelFilter log.LabelFilterer - LineFormatExpr *lineFmtExpr - LabelFormatExpr *labelFmtExpr + LineFormatExpr *LineFmtExpr + LabelFormatExpr *LabelFmtExpr LabelFormat log.LabelFmt LabelsFormat []log.LabelFmt JSONExpressionParser *jsonExpressionParser JSONExpression log.JSONExpression JSONExpressionList []log.JSONExpression - UnwrapExpr *unwrapExpr - OffsetExpr *offsetExpr + UnwrapExpr *UnwrapExpr + OffsetExpr *OffsetExpr } const BYTES = 57346 @@ -1164,7 +1164,7 @@ exprdefault: case 74: exprDollar = exprS[exprpt-2 : exprpt+1] { - exprVAL.PipelineStage = &labelFilterExpr{LabelFilterer: exprDollar[2].LabelFilter} + exprVAL.PipelineStage = &LabelFilterExpr{LabelFilterer: exprDollar[2].LabelFilter} } case 75: exprDollar = exprS[exprpt-2 : exprpt+1] From a2349ab517d830b0c99bbb70067d12d763bc5a3f Mon Sep 17 00:00:00 2001 From: Kaviraj Date: Thu, 22 Jul 2021 11:21:04 +0200 Subject: [PATCH 24/24] ast.go fixes --- pkg/logql/ast.go | 316 +++++++++++++++++++++++++++----------------- pkg/logql/expr.y | 2 +- pkg/logql/expr.y.go | 2 +- 3 files changed, 198 insertions(+), 122 deletions(-) diff --git a/pkg/logql/ast.go b/pkg/logql/ast.go index c15c6f9c89a41..0a54651b6e1ff 100644 --- a/pkg/logql/ast.go +++ b/pkg/logql/ast.go @@ -22,6 +22,7 @@ import ( type Expr interface { logQLExpr() // ensure it's not implemented accidentally Shardable() bool // A recursive check on the AST to see if it's shardable. + Walkable fmt.Stringer } @@ -86,7 +87,7 @@ type Querier interface { // LogSelectorExpr is a LogQL expression filtering and returning logs. type LogSelectorExpr interface { Matchers() []*labels.Matcher - PipelineExpr + LogPipelineExpr HasFilter() bool Expr } @@ -97,8 +98,8 @@ type ( SampleExtractor = log.SampleExtractor ) -// PipelineExpr is an expression defining a log pipeline. -type PipelineExpr interface { +// LogPipelineExpr is an expression defining a log pipeline. +type LogPipelineExpr interface { Pipeline() (Pipeline, error) Expr } @@ -148,22 +149,28 @@ func (m MultiStageExpr) String() string { func (MultiStageExpr) logQLExpr() {} // nolint:unused -type matchersExpr struct { +type MatchersExpr struct { matchers []*labels.Matcher implicit } -func newMatcherExpr(matchers []*labels.Matcher) *matchersExpr { - return &matchersExpr{matchers: matchers} +func newMatcherExpr(matchers []*labels.Matcher) *MatchersExpr { + return &MatchersExpr{matchers: matchers} } -func (e *matchersExpr) Matchers() []*labels.Matcher { +func (e *MatchersExpr) Matchers() []*labels.Matcher { return e.matchers } -func (e *matchersExpr) Shardable() bool { return true } +func (e *MatchersExpr) AppendMatchers(m []*labels.Matcher) { + e.matchers = append(e.matchers, m...) +} + +func (e *MatchersExpr) Shardable() bool { return true } + +func (e *MatchersExpr) Walk(f WalkFn) { f(e) } -func (e *matchersExpr) String() string { +func (e *MatchersExpr) String() string { var sb strings.Builder sb.WriteString("{") for i, m := range e.matchers { @@ -176,28 +183,28 @@ func (e *matchersExpr) String() string { return sb.String() } -func (e *matchersExpr) Pipeline() (log.Pipeline, error) { +func (e *MatchersExpr) Pipeline() (log.Pipeline, error) { return log.NewNoopPipeline(), nil } -func (e *matchersExpr) HasFilter() bool { +func (e *MatchersExpr) HasFilter() bool { return false } -type pipelineExpr struct { +type PipelineExpr struct { pipeline MultiStageExpr - left *matchersExpr + left *MatchersExpr implicit } -func newPipelineExpr(left *matchersExpr, pipeline MultiStageExpr) LogSelectorExpr { - return &pipelineExpr{ +func newPipelineExpr(left *MatchersExpr, pipeline MultiStageExpr) LogSelectorExpr { + return &PipelineExpr{ left: left, pipeline: pipeline, } } -func (e *pipelineExpr) Shardable() bool { +func (e *PipelineExpr) Shardable() bool { for _, p := range e.pipeline { if !p.Shardable() { return false @@ -206,11 +213,26 @@ func (e *pipelineExpr) Shardable() bool { return true } -func (e *pipelineExpr) Matchers() []*labels.Matcher { +func (e *PipelineExpr) Walk(f WalkFn) { + f(e) + + if e.left == nil { + return + } + + xs := make([]Walkable, 0, len(e.pipeline)+1) + xs = append(xs, e.left) + for _, p := range e.pipeline { + xs = append(xs, p) + } + walkAll(f, xs...) +} + +func (e *PipelineExpr) Matchers() []*labels.Matcher { return e.left.Matchers() } -func (e *pipelineExpr) String() string { +func (e *PipelineExpr) String() string { var sb strings.Builder sb.WriteString(e.left.String()) sb.WriteString(" ") @@ -218,15 +240,15 @@ func (e *pipelineExpr) String() string { return sb.String() } -func (e *pipelineExpr) Pipeline() (log.Pipeline, error) { +func (e *PipelineExpr) Pipeline() (log.Pipeline, error) { return e.pipeline.Pipeline() } // HasFilter returns true if the pipeline contains stage that can filter out lines. -func (e *pipelineExpr) HasFilter() bool { +func (e *PipelineExpr) HasFilter() bool { for _, p := range e.pipeline { switch p.(type) { - case *lineFilterExpr, *labelFilterExpr: + case *LineFilterExpr, *LabelFilterExpr: return true default: continue @@ -235,24 +257,24 @@ func (e *pipelineExpr) HasFilter() bool { return false } -type lineFilterExpr struct { - left *lineFilterExpr +type LineFilterExpr struct { + left *LineFilterExpr ty labels.MatchType match string op string implicit } -func newLineFilterExpr(ty labels.MatchType, op, match string) *lineFilterExpr { - return &lineFilterExpr{ +func newLineFilterExpr(ty labels.MatchType, op, match string) *LineFilterExpr { + return &LineFilterExpr{ ty: ty, match: match, op: op, } } -func newNestedLineFilterExpr(left *lineFilterExpr, right *lineFilterExpr) *lineFilterExpr { - return &lineFilterExpr{ +func newNestedLineFilterExpr(left *LineFilterExpr, right *LineFilterExpr) *LineFilterExpr { + return &LineFilterExpr{ left: left, ty: right.ty, match: right.match, @@ -260,13 +282,21 @@ func newNestedLineFilterExpr(left *lineFilterExpr, right *lineFilterExpr) *lineF } } +func (e *LineFilterExpr) Walk(f WalkFn) { + f(e) + if e.left == nil { + return + } + e.left.Walk(f) +} + // AddFilterExpr adds a filter expression to a logselector expression. func AddFilterExpr(expr LogSelectorExpr, ty labels.MatchType, op, match string) (LogSelectorExpr, error) { filter := newLineFilterExpr(ty, op, match) switch e := expr.(type) { - case *matchersExpr: + case *MatchersExpr: return newPipelineExpr(e, MultiStageExpr{filter}), nil - case *pipelineExpr: + case *PipelineExpr: e.pipeline = append(e.pipeline, filter) return e, nil default: @@ -274,9 +304,9 @@ func AddFilterExpr(expr LogSelectorExpr, ty labels.MatchType, op, match string) } } -func (e *lineFilterExpr) Shardable() bool { return true } +func (e *LineFilterExpr) Shardable() bool { return true } -func (e *lineFilterExpr) String() string { +func (e *LineFilterExpr) String() string { var sb strings.Builder if e.left != nil { sb.WriteString(e.left.String()) @@ -304,7 +334,7 @@ func (e *lineFilterExpr) String() string { return sb.String() } -func (e *lineFilterExpr) Filter() (log.Filterer, error) { +func (e *LineFilterExpr) Filter() (log.Filterer, error) { var f log.Filterer switch e.op { @@ -335,7 +365,7 @@ func (e *lineFilterExpr) Filter() (log.Filterer, error) { return f, nil } -func (e *lineFilterExpr) Stage() (log.Stage, error) { +func (e *LineFilterExpr) Stage() (log.Stage, error) { f, err := e.Filter() if err != nil { return nil, err @@ -343,22 +373,24 @@ func (e *lineFilterExpr) Stage() (log.Stage, error) { return f.ToStage(), nil } -type labelParserExpr struct { +type LabelParserExpr struct { op string param string implicit } -func newLabelParserExpr(op, param string) *labelParserExpr { - return &labelParserExpr{ +func newLabelParserExpr(op, param string) *LabelParserExpr { + return &LabelParserExpr{ op: op, param: param, } } -func (e *labelParserExpr) Shardable() bool { return true } +func (e *LabelParserExpr) Shardable() bool { return true } -func (e *labelParserExpr) Stage() (log.Stage, error) { +func (e *LabelParserExpr) Walk(f WalkFn) { f(e) } + +func (e *LabelParserExpr) Stage() (log.Stage, error) { switch e.op { case OpParserTypeJSON: return log.NewJSONParser(), nil @@ -375,7 +407,7 @@ func (e *labelParserExpr) Stage() (log.Stage, error) { } } -func (e *labelParserExpr) String() string { +func (e *LabelParserExpr) String() string { var sb strings.Builder sb.WriteString(OpPipe) sb.WriteString(" ") @@ -387,72 +419,77 @@ func (e *labelParserExpr) String() string { return sb.String() } -type labelFilterExpr struct { +type LabelFilterExpr struct { log.LabelFilterer implicit } -func newLabelFilterExpr(filterer log.LabelFilterer) *labelFilterExpr { - return &labelFilterExpr{ +func newLabelFilterExpr(filterer log.LabelFilterer) *LabelFilterExpr { + return &LabelFilterExpr{ LabelFilterer: filterer, } } -func (e *labelFilterExpr) Shardable() bool { return true } +func (e *LabelFilterExpr) Shardable() bool { return true } + +func (e *LabelFilterExpr) Walk(f WalkFn) { f(e) } -func (e *labelFilterExpr) Stage() (log.Stage, error) { +func (e *LabelFilterExpr) Stage() (log.Stage, error) { switch ip := e.LabelFilterer.(type) { case *log.IPLabelFilter: return ip, ip.PatternError() } - return e.LabelFilterer, nil } -func (e *labelFilterExpr) String() string { +func (e *LabelFilterExpr) String() string { return fmt.Sprintf("%s %s", OpPipe, e.LabelFilterer.String()) } -type lineFmtExpr struct { +type LineFmtExpr struct { value string implicit } -func newLineFmtExpr(value string) *lineFmtExpr { - return &lineFmtExpr{ +func newLineFmtExpr(value string) *LineFmtExpr { + return &LineFmtExpr{ value: value, } } -func (e *lineFmtExpr) Shardable() bool { return true } +func (e *LineFmtExpr) Shardable() bool { return true } + +func (e *LineFmtExpr) Walk(f WalkFn) { f(e) } -func (e *lineFmtExpr) Stage() (log.Stage, error) { +func (e *LineFmtExpr) Stage() (log.Stage, error) { return log.NewFormatter(e.value) } -func (e *lineFmtExpr) String() string { +func (e *LineFmtExpr) String() string { return fmt.Sprintf("%s %s %s", OpPipe, OpFmtLine, strconv.Quote(e.value)) } -type labelFmtExpr struct { +type LabelFmtExpr struct { formats []log.LabelFmt implicit } -func newLabelFmtExpr(fmts []log.LabelFmt) *labelFmtExpr { - return &labelFmtExpr{ +func newLabelFmtExpr(fmts []log.LabelFmt) *LabelFmtExpr { + return &LabelFmtExpr{ formats: fmts, } } -func (e *labelFmtExpr) Shardable() bool { return false } +func (e *LabelFmtExpr) Shardable() bool { return false } -func (e *labelFmtExpr) Stage() (log.Stage, error) { +func (e *LabelFmtExpr) Walk(f WalkFn) { f(e) } + +func (e *LabelFmtExpr) Stage() (log.Stage, error) { return log.NewLabelsFormatter(e.formats) } -func (e *labelFmtExpr) String() string { +func (e *LabelFmtExpr) String() string { var sb strings.Builder sb.WriteString(fmt.Sprintf("%s %s ", OpPipe, OpFmtLabel)) for i, f := range e.formats { @@ -470,25 +507,27 @@ func (e *labelFmtExpr) String() string { return sb.String() } -type jsonExpressionParser struct { +type JSONExpressionParser struct { expressions []log.JSONExpression implicit } -func newJSONExpressionParser(expressions []log.JSONExpression) *jsonExpressionParser { - return &jsonExpressionParser{ +func newJSONExpressionParser(expressions []log.JSONExpression) *JSONExpressionParser { + return &JSONExpressionParser{ expressions: expressions, } } -func (j *jsonExpressionParser) Shardable() bool { return true } +func (j *JSONExpressionParser) Shardable() bool { return true } + +func (j *JSONExpressionParser) Walk(f WalkFn) { f(j) } -func (j *jsonExpressionParser) Stage() (log.Stage, error) { +func (j *JSONExpressionParser) Stage() (log.Stage, error) { return log.NewJSONExpressionParser(j.expressions) } -func (j *jsonExpressionParser) String() string { +func (j *JSONExpressionParser) String() string { var sb strings.Builder sb.WriteString(fmt.Sprintf("%s %s ", OpPipe, OpParserTypeJSON)) for i, exp := range j.expressions { @@ -519,14 +558,14 @@ func mustNewFloat(s string) float64 { return n } -type unwrapExpr struct { +type UnwrapExpr struct { identifier string operation string postFilters []log.LabelFilterer } -func (u unwrapExpr) String() string { +func (u UnwrapExpr) String() string { var sb strings.Builder if u.operation != "" { sb.WriteString(fmt.Sprintf(" %s %s %s(%s)", OpPipe, OpUnwrap, u.operation, u.identifier)) @@ -539,25 +578,25 @@ func (u unwrapExpr) String() string { return sb.String() } -func (u *unwrapExpr) addPostFilter(f log.LabelFilterer) *unwrapExpr { +func (u *UnwrapExpr) addPostFilter(f log.LabelFilterer) *UnwrapExpr { u.postFilters = append(u.postFilters, f) return u } -func newUnwrapExpr(id string, operation string) *unwrapExpr { - return &unwrapExpr{identifier: id, operation: operation} +func newUnwrapExpr(id string, operation string) *UnwrapExpr { + return &UnwrapExpr{identifier: id, operation: operation} } -type logRange struct { +type LogRange struct { left LogSelectorExpr interval time.Duration offset time.Duration - unwrap *unwrapExpr + unwrap *UnwrapExpr } // impls Stringer -func (r logRange) String() string { +func (r LogRange) String() string { var sb strings.Builder sb.WriteString(r.left.String()) if r.unwrap != nil { @@ -565,20 +604,28 @@ func (r logRange) String() string { } sb.WriteString(fmt.Sprintf("[%v]", model.Duration(r.interval))) if r.offset != 0 { - offsetExpr := offsetExpr{offset: r.offset} + offsetExpr := OffsetExpr{offset: r.offset} sb.WriteString(offsetExpr.String()) } return sb.String() } -func (r *logRange) Shardable() bool { return r.left.Shardable() } +func (r *LogRange) Shardable() bool { return r.left.Shardable() } + +func (r *LogRange) Walk(f WalkFn) { + f(r) + if r.left == nil { + return + } + r.left.Walk(f) +} -func newLogRange(left LogSelectorExpr, interval time.Duration, u *unwrapExpr, o *offsetExpr) *logRange { +func newLogRange(left LogSelectorExpr, interval time.Duration, u *UnwrapExpr, o *OffsetExpr) *LogRange { var offset time.Duration if o != nil { offset = o.offset } - return &logRange{ + return &LogRange{ left: left, interval: interval, unwrap: u, @@ -586,18 +633,18 @@ func newLogRange(left LogSelectorExpr, interval time.Duration, u *unwrapExpr, o } } -type offsetExpr struct { +type OffsetExpr struct { offset time.Duration } -func (o *offsetExpr) String() string { +func (o *OffsetExpr) String() string { var sb strings.Builder sb.WriteString(fmt.Sprintf(" %s %s", OpOffset, o.offset.String())) return sb.String() } -func newOffsetExpr(offset time.Duration) *offsetExpr { - return &offsetExpr{ +func newOffsetExpr(offset time.Duration) *OffsetExpr { + return &OffsetExpr{ offset: offset, } } @@ -703,8 +750,8 @@ type SampleExpr interface { Expr } -type rangeAggregationExpr struct { - left *logRange +type RangeAggregationExpr struct { + left *LogRange operation string params *float64 @@ -712,7 +759,7 @@ type rangeAggregationExpr struct { implicit } -func newRangeAggregationExpr(left *logRange, operation string, gr *grouping, stringParams *string) SampleExpr { +func newRangeAggregationExpr(left *LogRange, operation string, gr *grouping, stringParams *string) SampleExpr { var params *float64 if stringParams != nil { if operation != OpRangeTypeQuantile { @@ -730,7 +777,7 @@ func newRangeAggregationExpr(left *logRange, operation string, gr *grouping, str panic(logqlmodel.NewParseError(fmt.Sprintf("parameter required for operation %s", operation), 0, 0)) } } - e := &rangeAggregationExpr{ + e := &RangeAggregationExpr{ left: left, operation: operation, grouping: gr, @@ -742,11 +789,11 @@ func newRangeAggregationExpr(left *logRange, operation string, gr *grouping, str return e } -func (e *rangeAggregationExpr) Selector() LogSelectorExpr { +func (e *RangeAggregationExpr) Selector() LogSelectorExpr { return e.left.left } -func (e rangeAggregationExpr) validate() error { +func (e RangeAggregationExpr) validate() error { if e.grouping != nil { switch e.operation { case OpRangeTypeAvg, OpRangeTypeStddev, OpRangeTypeStdvar, OpRangeTypeQuantile, OpRangeTypeMax, OpRangeTypeMin, OpRangeTypeFirst, OpRangeTypeLast: @@ -771,7 +818,7 @@ func (e rangeAggregationExpr) validate() error { } // impls Stringer -func (e *rangeAggregationExpr) String() string { +func (e *RangeAggregationExpr) String() string { var sb strings.Builder sb.WriteString(e.operation) sb.WriteString("(") @@ -788,10 +835,18 @@ func (e *rangeAggregationExpr) String() string { } // impl SampleExpr -func (e *rangeAggregationExpr) Shardable() bool { +func (e *RangeAggregationExpr) Shardable() bool { return shardableOps[e.operation] && e.left.Shardable() } +func (e *RangeAggregationExpr) Walk(f WalkFn) { + f(e) + if e.left == nil { + return + } + e.left.Walk(f) +} + type grouping struct { groups []string without bool @@ -815,7 +870,7 @@ func (g grouping) String() string { return sb.String() } -type vectorAggregationExpr struct { +type VectorAggregationExpr struct { left SampleExpr grouping *grouping @@ -844,7 +899,7 @@ func mustNewVectorAggregationExpr(left SampleExpr, operation string, gr *groupin if gr == nil { gr = &grouping{} } - return &vectorAggregationExpr{ + return &VectorAggregationExpr{ left: left, operation: operation, grouping: gr, @@ -852,14 +907,14 @@ func mustNewVectorAggregationExpr(left SampleExpr, operation string, gr *groupin } } -func (e *vectorAggregationExpr) Selector() LogSelectorExpr { +func (e *VectorAggregationExpr) Selector() LogSelectorExpr { return e.left.Selector() } -func (e *vectorAggregationExpr) Extractor() (log.SampleExtractor, error) { +func (e *VectorAggregationExpr) Extractor() (log.SampleExtractor, error) { // inject in the range vector extractor the outer groups to improve performance. // This is only possible if the operation is a sum. Anything else needs all labels. - if r, ok := e.left.(*rangeAggregationExpr); ok && canInjectVectorGrouping(e.operation, r.operation) { + if r, ok := e.left.(*RangeAggregationExpr); ok && canInjectVectorGrouping(e.operation, r.operation) { // if the range vec operation has no grouping we can push down the vec one. if r.grouping == nil { return r.extractor(e.grouping) @@ -881,7 +936,7 @@ func canInjectVectorGrouping(vecOp, rangeOp string) bool { } } -func (e *vectorAggregationExpr) String() string { +func (e *VectorAggregationExpr) String() string { var params []string if e.params != 0 { params = []string{fmt.Sprintf("%d", e.params), e.left.String()} @@ -892,22 +947,30 @@ func (e *vectorAggregationExpr) String() string { } // impl SampleExpr -func (e *vectorAggregationExpr) Shardable() bool { +func (e *VectorAggregationExpr) Shardable() bool { return shardableOps[e.operation] && e.left.Shardable() } +func (e *VectorAggregationExpr) Walk(f WalkFn) { + f(e) + if e.left == nil { + return + } + e.left.Walk(f) +} + type BinOpOptions struct { ReturnBool bool } -type binOpExpr struct { +type BinOpExpr struct { SampleExpr RHS SampleExpr op string opts BinOpOptions } -func (e *binOpExpr) String() string { +func (e *BinOpExpr) String() string { if e.opts.ReturnBool { return fmt.Sprintf("(%s %s bool %s)", e.SampleExpr.String(), e.op, e.RHS.String()) } @@ -915,10 +978,14 @@ func (e *binOpExpr) String() string { } // impl SampleExpr -func (e *binOpExpr) Shardable() bool { +func (e *BinOpExpr) Shardable() bool { return shardableOps[e.op] && e.SampleExpr.Shardable() && e.RHS.Shardable() } +func (e *BinOpExpr) Walk(f WalkFn) { + walkAll(f, e.SampleExpr, e.RHS) +} + func mustNewBinOpExpr(op string, opts BinOpOptions, lhs, rhs Expr) SampleExpr { left, ok := lhs.(SampleExpr) if !ok { @@ -938,8 +1005,8 @@ func mustNewBinOpExpr(op string, opts BinOpOptions, lhs, rhs Expr) SampleExpr { ), 0, 0)) } - leftLit, lOk := left.(*literalExpr) - rightLit, rOk := right.(*literalExpr) + leftLit, lOk := left.(*LiteralExpr) + rightLit, rOk := right.(*LiteralExpr) if IsLogicalBinOp(op) { if lOk { @@ -964,7 +1031,7 @@ func mustNewBinOpExpr(op string, opts BinOpOptions, lhs, rhs Expr) SampleExpr { return reduceBinOp(op, leftLit, rightLit) } - return &binOpExpr{ + return &BinOpExpr{ SampleExpr: left, RHS: right, op: op, @@ -975,7 +1042,7 @@ func mustNewBinOpExpr(op string, opts BinOpOptions, lhs, rhs Expr) SampleExpr { // Reduces a binary operation expression. A binop is reducible if both of its legs are literal expressions. // This is because literals need match all labels, which is currently difficult to encode into StepEvaluators. // Therefore, we ensure a binop can be reduced/simplified, maintaining the invariant that it does not have two literal legs. -func reduceBinOp(op string, left, right *literalExpr) *literalExpr { +func reduceBinOp(op string, left, right *LiteralExpr) *LiteralExpr { merged := mergeBinOp( op, &promql.Sample{Point: promql.Point{V: left.value}}, @@ -983,15 +1050,15 @@ func reduceBinOp(op string, left, right *literalExpr) *literalExpr { false, false, ) - return &literalExpr{value: merged.V} + return &LiteralExpr{value: merged.V} } -type literalExpr struct { +type LiteralExpr struct { value float64 implicit } -func mustNewLiteralExpr(s string, invert bool) *literalExpr { +func mustNewLiteralExpr(s string, invert bool) *LiteralExpr { n, err := strconv.ParseFloat(s, 64) if err != nil { panic(logqlmodel.NewParseError(fmt.Sprintf("unable to parse literal as a float: %s", err.Error()), 0, 0)) @@ -1001,24 +1068,25 @@ func mustNewLiteralExpr(s string, invert bool) *literalExpr { n = -n } - return &literalExpr{ + return &LiteralExpr{ value: n, } } -func (e *literalExpr) String() string { +func (e *LiteralExpr) String() string { return fmt.Sprint(e.value) } // literlExpr impls SampleExpr & LogSelectorExpr mainly to reduce the need for more complicated typings // to facilitate sum types. We'll be type switching when evaluating them anyways // and they will only be present in binary operation legs. -func (e *literalExpr) Selector() LogSelectorExpr { return e } -func (e *literalExpr) HasFilter() bool { return false } -func (e *literalExpr) Shardable() bool { return true } -func (e *literalExpr) Pipeline() (log.Pipeline, error) { return log.NewNoopPipeline(), nil } -func (e *literalExpr) Matchers() []*labels.Matcher { return nil } -func (e *literalExpr) Extractor() (log.SampleExtractor, error) { return nil, nil } +func (e *LiteralExpr) Selector() LogSelectorExpr { return e } +func (e *LiteralExpr) HasFilter() bool { return false } +func (e *LiteralExpr) Shardable() bool { return true } +func (e *LiteralExpr) Walk(f WalkFn) { f(e) } +func (e *LiteralExpr) Pipeline() (log.Pipeline, error) { return log.NewNoopPipeline(), nil } +func (e *LiteralExpr) Matchers() []*labels.Matcher { return nil } +func (e *LiteralExpr) Extractor() (log.SampleExtractor, error) { return nil, nil } // helper used to impl Stringer for vector and range aggregations // nolint:interfacer @@ -1041,7 +1109,7 @@ func formatOperation(op string, grouping *grouping, params ...string) string { return sb.String() } -type labelReplaceExpr struct { +type LabelReplaceExpr struct { left SampleExpr dst string replacement string @@ -1052,12 +1120,12 @@ type labelReplaceExpr struct { implicit } -func mustNewLabelReplaceExpr(left SampleExpr, dst, replacement, src, regex string) *labelReplaceExpr { +func mustNewLabelReplaceExpr(left SampleExpr, dst, replacement, src, regex string) *LabelReplaceExpr { re, err := regexp.Compile("^(?:" + regex + ")$") if err != nil { panic(logqlmodel.NewParseError(fmt.Sprintf("invalid regex in label_replace: %s", err.Error()), 0, 0)) } - return &labelReplaceExpr{ + return &LabelReplaceExpr{ left: left, dst: dst, replacement: replacement, @@ -1067,19 +1135,27 @@ func mustNewLabelReplaceExpr(left SampleExpr, dst, replacement, src, regex strin } } -func (e *labelReplaceExpr) Selector() LogSelectorExpr { +func (e *LabelReplaceExpr) Selector() LogSelectorExpr { return e.left.Selector() } -func (e *labelReplaceExpr) Extractor() (SampleExtractor, error) { +func (e *LabelReplaceExpr) Extractor() (SampleExtractor, error) { return e.left.Extractor() } -func (e *labelReplaceExpr) Shardable() bool { +func (e *LabelReplaceExpr) Shardable() bool { return false } -func (e *labelReplaceExpr) String() string { +func (e *LabelReplaceExpr) Walk(f WalkFn) { + f(e) + if e.left == nil { + return + } + e.left.Walk(f) +} + +func (e *LabelReplaceExpr) String() string { var sb strings.Builder sb.WriteString(OpLabelReplace) sb.WriteString("(") diff --git a/pkg/logql/expr.y b/pkg/logql/expr.y index ced1d96b2155b..b9d1c65e2dacc 100644 --- a/pkg/logql/expr.y +++ b/pkg/logql/expr.y @@ -49,7 +49,7 @@ import ( LabelFormatExpr *LabelFmtExpr LabelFormat log.LabelFmt LabelsFormat []log.LabelFmt - JSONExpressionParser *jsonExpressionParser + JSONExpressionParser *JSONExpressionParser JSONExpression log.JSONExpression JSONExpressionList []log.JSONExpression UnwrapExpr *UnwrapExpr diff --git a/pkg/logql/expr.y.go b/pkg/logql/expr.y.go index 757f49e802a94..11053f201bf7b 100644 --- a/pkg/logql/expr.y.go +++ b/pkg/logql/expr.y.go @@ -52,7 +52,7 @@ type exprSymType struct { LabelFormatExpr *LabelFmtExpr LabelFormat log.LabelFmt LabelsFormat []log.LabelFmt - JSONExpressionParser *jsonExpressionParser + JSONExpressionParser *JSONExpressionParser JSONExpression log.JSONExpression JSONExpressionList []log.JSONExpression UnwrapExpr *UnwrapExpr