diff --git a/Makefile b/Makefile index 9c03307db..685d5f9c8 100644 --- a/Makefile +++ b/Makefile @@ -39,16 +39,12 @@ define ALL_HELP_INFO # debugging tools like delve. endef .PHONY: all -all: test hypersphere ks-apiserver ks-apigateway controller-manager +all: test hypersphere ks-apiserver controller-manager # Build ks-apiserver binary ks-apiserver: fmt vet hack/gobuild.sh cmd/ks-apiserver -# Build ks-apigateway binary -ks-apigateway: fmt vet - hack/gobuild.sh cmd/ks-apigateway - # Build controller-manager binary controller-manager: fmt vet hack/gobuild.sh cmd/controller-manager diff --git a/build/ks-apigateway/Dockerfile b/build/ks-apigateway/Dockerfile deleted file mode 100644 index 61700244d..000000000 --- a/build/ks-apigateway/Dockerfile +++ /dev/null @@ -1,20 +0,0 @@ -# Copyright 2018 The KubeSphere Authors. All rights reserved. -# Use of this source code is governed by a Apache license -# that can be found in the LICENSE file. - -# Copyright 2018 The KubeSphere Authors. All rights reserved. -# Use of this source code is governed by a Apache license -# that can be found in the LICENSE file. - -FROM golang:1.12 as ks-apigateway-builder - -COPY / /go/src/kubesphere.io/kubesphere -WORKDIR /go/src/kubesphere.io/kubesphere -RUN CGO_ENABLED=0 GO111MODULE=on GOOS=linux GOARCH=amd64 GOFLAGS=-mod=vendor go build -i -ldflags '-w -s' -o ks-apigateway cmd/ks-apigateway/apiserver.go && \ - go run tools/cmd/doc-gen/main.go --output=install/swagger-ui/api.json - -FROM alpine:3.9 -RUN apk add --update ca-certificates && update-ca-certificates -COPY --from=ks-apigateway-builder /go/src/kubesphere.io/kubesphere/ks-apigateway /usr/local/bin/ -COPY --from=ks-apigateway-builder /go/src/kubesphere.io/kubesphere/install/swagger-ui /var/static/swagger-ui -CMD ["sh"] diff --git a/build/ks-iam/Dockerfile b/build/ks-iam/Dockerfile deleted file mode 100644 index 3e65a47e8..000000000 --- a/build/ks-iam/Dockerfile +++ /dev/null @@ -1,18 +0,0 @@ -# Copyright 2018 The KubeSphere Authors. All rights reserved. -# Use of this source code is governed by a Apache license -# that can be found in the LICENSE file. - -# Copyright 2018 The KubeSphere Authors. All rights reserved. -# Use of this source code is governed by a Apache license -# that can be found in the LICENSE file. -FROM golang:1.12 as ks-iam-builder - -COPY / /go/src/kubesphere.io/kubesphere - -WORKDIR /go/src/kubesphere.io/kubesphere -RUN CGO_ENABLED=0 GO111MODULE=on GOOS=linux GOARCH=amd64 GOFLAGS=-mod=vendor go build -i -ldflags '-w -s' -o ks-iam cmd/ks-iam/apiserver.go - -FROM alpine:3.9 -RUN apk add --update ca-certificates && update-ca-certificates -COPY --from=ks-iam-builder /go/src/kubesphere.io/kubesphere/ks-iam /usr/local/bin/ -CMD ["sh"] diff --git a/cmd/hypersphere/hypersphere.go b/cmd/hypersphere/hypersphere.go index cb781b330..45db4ca8a 100644 --- a/cmd/hypersphere/hypersphere.go +++ b/cmd/hypersphere/hypersphere.go @@ -8,7 +8,6 @@ import ( "github.com/spf13/cobra" "github.com/spf13/pflag" controllermanager "kubesphere.io/kubesphere/cmd/controller-manager/app" - ksapigateway "kubesphere.io/kubesphere/cmd/ks-apigateway/app" ksapiserver "kubesphere.io/kubesphere/cmd/ks-apiserver/app" "os" ) @@ -45,12 +44,10 @@ func commandFor(basename string, defaultCommand *cobra.Command, commands []func( func NewHyperSphereCommand() (*cobra.Command, []func() *cobra.Command) { apiserver := func() *cobra.Command { return ksapiserver.NewAPIServerCommand() } controllermanager := func() *cobra.Command { return controllermanager.NewControllerManagerCommand() } - apigateway := func() *cobra.Command { return ksapigateway.NewAPIGatewayCommand() } commandFns := []func() *cobra.Command{ apiserver, controllermanager, - apigateway, } cmd := &cobra.Command{ diff --git a/cmd/ks-apigateway/apiserver.go b/cmd/ks-apigateway/apiserver.go deleted file mode 100644 index 05446247e..000000000 --- a/cmd/ks-apigateway/apiserver.go +++ /dev/null @@ -1,32 +0,0 @@ -/* - - Copyright 2019 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - -*/ -package main - -import ( - "kubesphere.io/kubesphere/cmd/ks-apigateway/app" - "os" -) - -func main() { - - cmd := app.NewAPIGatewayCommand() - - if err := cmd.Execute(); err != nil { - os.Exit(1) - } -} diff --git a/cmd/ks-apigateway/app/server.go b/cmd/ks-apigateway/app/server.go deleted file mode 100644 index ec750f73e..000000000 --- a/cmd/ks-apigateway/app/server.go +++ /dev/null @@ -1,40 +0,0 @@ -package app - -import ( - "flag" - "github.com/mholt/caddy/caddy/caddymain" - "github.com/mholt/caddy/caddyhttp/httpserver" - "github.com/spf13/cobra" - "kubesphere.io/kubesphere/pkg/utils/signals" - - "kubesphere.io/kubesphere/pkg/apigateway" -) - -func NewAPIGatewayCommand() *cobra.Command { - - cmd := &cobra.Command{ - Use: "ks-apigateway", - Long: `The KubeSphere API Gateway, which is responsible -for proxy request to the right backend. API Gateway also proxy -Kubernetes API Server for KubeSphere authorization purpose. -`, - RunE: func(cmd *cobra.Command, args []string) error { - apigateway.RegisterPlugins() - - return Run(signals.SetupSignalHandler()) - }, - } - - cmd.Flags().AddGoFlagSet(flag.CommandLine) - - return cmd -} - -func Run(stopCh <-chan struct{}) error { - httpserver.RegisterDevDirective("authenticate", "jwt") - httpserver.RegisterDevDirective("authentication", "jwt") - httpserver.RegisterDevDirective("swagger", "jwt") - caddymain.Run() - - return nil -} diff --git a/cmd/ks-apiserver/app/options/options.go b/cmd/ks-apiserver/app/options/options.go index 1e5a349ed..5c41826b3 100644 --- a/cmd/ks-apiserver/app/options/options.go +++ b/cmd/ks-apiserver/app/options/options.go @@ -6,7 +6,7 @@ import ( "fmt" cliflag "k8s.io/component-base/cli/flag" "k8s.io/klog" - "kubesphere.io/kubesphere/pkg/api/iam" + "kubesphere.io/kubesphere/pkg/api/auth" "kubesphere.io/kubesphere/pkg/apiserver" "kubesphere.io/kubesphere/pkg/informers" genericoptions "kubesphere.io/kubesphere/pkg/server/options" @@ -40,7 +40,7 @@ type ServerRunOptions struct { LoggingOptions *esclient.Options LdapOptions *ldap.Options CacheOptions *cache.Options - AuthenticateOptions *iam.AuthenticationOptions + AuthenticateOptions *auth.AuthenticationOptions // DebugMode bool @@ -61,7 +61,7 @@ func NewServerRunOptions() *ServerRunOptions { LoggingOptions: esclient.NewElasticSearchOptions(), LdapOptions: ldap.NewOptions(), CacheOptions: cache.NewRedisOptions(), - AuthenticateOptions: iam.NewAuthenticateOptions(), + AuthenticateOptions: auth.NewAuthenticateOptions(), } return &s diff --git a/cmd/ks-apiserver/app/server.go b/cmd/ks-apiserver/app/server.go index 3375ced9f..78caa399e 100644 --- a/cmd/ks-apiserver/app/server.go +++ b/cmd/ks-apiserver/app/server.go @@ -46,6 +46,8 @@ func NewAPIServerCommand() *cobra.Command { S3Options: conf.S3Options, OpenPitrixOptions: conf.OpenPitrixOptions, LoggingOptions: conf.LoggingOptions, + LdapOptions: conf.LdapOptions, + CacheOptions: conf.RedisOptions, AuthenticateOptions: conf.AuthenticateOptions, } } diff --git a/go.mod b/go.mod index ad71eef41..8a17b7d5d 100644 --- a/go.mod +++ b/go.mod @@ -55,7 +55,6 @@ require ( github.com/kelseyhightower/envconfig v1.4.0 // indirect github.com/kiali/kiali v0.15.1-0.20191210080139-edbbad1ef779 github.com/klauspost/cpuid v1.2.1 // indirect - github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect github.com/kubernetes-sigs/application v0.0.0-20191210100950-18cc93526ab4 github.com/kubesphere/sonargo v0.0.2 github.com/leodido/go-urn v1.1.0 // indirect @@ -69,6 +68,7 @@ require ( github.com/morikuni/aec v0.0.0-20170113033406-39771216ff4c // indirect github.com/onsi/ginkgo v1.8.0 github.com/onsi/gomega v1.5.0 + github.com/open-policy-agent/opa v0.18.0 github.com/opencontainers/go-digest v1.0.0-rc1 github.com/opencontainers/image-spec v1.0.1 // indirect github.com/openshift/api v3.9.0+incompatible // indirect @@ -84,7 +84,6 @@ require ( github.com/xanzy/ssh-agent v0.2.1 // indirect golang.org/x/net v0.0.0-20191004110552-13f9640d40b9 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 - golang.org/x/tools v0.0.0-20190920225731-5eefd052ad72 // indirect google.golang.org/grpc v1.23.1 gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d // indirect gopkg.in/go-playground/validator.v9 v9.29.1 // indirect @@ -208,6 +207,7 @@ replace ( github.com/go-sql-driver/mysql => github.com/go-sql-driver/mysql v1.4.1 github.com/go-stack/stack => github.com/go-stack/stack v1.8.0 github.com/gobuffalo/flect => github.com/gobuffalo/flect v0.1.5 + github.com/gobwas/glob => github.com/gobwas/glob v0.2.3 github.com/gocraft/dbr => github.com/gocraft/dbr v0.0.0-20180507214907-a0fd650918f6 github.com/gofrs/uuid => github.com/gofrs/uuid v3.2.0+incompatible github.com/gogo/protobuf => github.com/gogo/protobuf v1.3.0 @@ -277,6 +277,7 @@ replace ( github.com/marten-seemann/qtls => github.com/marten-seemann/qtls v0.2.3 github.com/mattn/go-colorable => github.com/mattn/go-colorable v0.1.2 github.com/mattn/go-isatty => github.com/mattn/go-isatty v0.0.8 + github.com/mattn/go-runewidth => github.com/mattn/go-runewidth v0.0.0-20181025052659-b20a3daf6a39 github.com/mattn/go-sqlite3 => github.com/mattn/go-sqlite3 v1.11.0 github.com/matttproud/golang_protobuf_extensions => github.com/matttproud/golang_protobuf_extensions v1.0.1 github.com/mholt/caddy => github.com/mholt/caddy v1.0.0 @@ -284,6 +285,7 @@ replace ( github.com/miekg/dns => github.com/miekg/dns v1.1.9 github.com/mitchellh/go-homedir => github.com/mitchellh/go-homedir v1.1.0 github.com/mitchellh/mapstructure => github.com/mitchellh/mapstructure v1.1.2 + github.com/mna/pigeon => github.com/mna/pigeon v0.0.0-20180808201053-bb0192cfc2ae github.com/modern-go/concurrent => github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd github.com/modern-go/reflect2 => github.com/modern-go/reflect2 v1.0.1 github.com/morikuni/aec => github.com/morikuni/aec v0.0.0-20170113033406-39771216ff4c @@ -293,8 +295,10 @@ replace ( github.com/naoina/go-stringutil => github.com/naoina/go-stringutil v0.1.0 github.com/naoina/toml => github.com/naoina/toml v0.1.1 github.com/oklog/ulid => github.com/oklog/ulid v1.3.1 + github.com/olekukonko/tablewriter => github.com/olekukonko/tablewriter v0.0.1 github.com/onsi/ginkgo => github.com/onsi/ginkgo v1.8.0 github.com/onsi/gomega => github.com/onsi/gomega v1.5.0 + github.com/open-policy-agent/opa => github.com/open-policy-agent/opa v0.18.0 github.com/opencontainers/go-digest => github.com/opencontainers/go-digest v1.0.0-rc1 github.com/opencontainers/image-spec => github.com/opencontainers/image-spec v1.0.1 github.com/openshift/api => github.com/openshift/api v3.9.0+incompatible @@ -302,6 +306,7 @@ replace ( github.com/pelletier/go-buffruneio => github.com/pelletier/go-buffruneio v0.2.0 github.com/pelletier/go-toml => github.com/pelletier/go-toml v1.2.0 github.com/peterbourgon/diskv => github.com/peterbourgon/diskv v2.0.1+incompatible + github.com/peterh/liner => github.com/peterh/liner v0.0.0-20170211195444-bf27d3ba8e1d github.com/philhofer/fwd => github.com/philhofer/fwd v1.0.0 github.com/pkg/errors => github.com/pkg/errors v0.8.1 github.com/pmezard/go-difflib => github.com/pmezard/go-difflib v1.0.0 @@ -316,6 +321,7 @@ replace ( github.com/prometheus/common => github.com/prometheus/common v0.4.0 github.com/prometheus/procfs => github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084 github.com/prometheus/tsdb => github.com/prometheus/tsdb v0.7.1 + github.com/rcrowley/go-metrics => github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a github.com/remyoudompheng/bigfft => github.com/remyoudompheng/bigfft v0.0.0-20170806203942-52369c62f446 github.com/rogpeppe/fastuuid => github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af github.com/rogpeppe/go-charset => github.com/rogpeppe/go-charset v0.0.0-20180617210344-2471d30d28b4 @@ -346,6 +352,7 @@ replace ( github.com/xiang90/probing => github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 github.com/xlab/treeprint => github.com/xlab/treeprint v0.0.0-20180616005107-d6fb6747feb6 github.com/xordataexchange/crypt => github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77 + github.com/yashtewari/glob-intersection => github.com/yashtewari/glob-intersection v0.0.0-20180916065949-5c77d914dd0b go.etcd.io/bbolt => go.etcd.io/bbolt v1.3.3 go.opencensus.io => go.opencensus.io v0.21.0 go.uber.org/atomic => go.uber.org/atomic v1.4.0 diff --git a/go.sum b/go.sum index 5be03dec8..e9ac5831d 100644 --- a/go.sum +++ b/go.sum @@ -24,6 +24,7 @@ github.com/Microsoft/go-winio v0.4.12 h1:xAfWHN1IrQ0NJ9TBC0KBZoqLjzDTr1ML+4MywiU github.com/Microsoft/go-winio v0.4.12/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA= github.com/NYTimes/gziphandler v1.1.1 h1:ZUDjpQae29j0ryrS0u/B8HZfJBtBQHjqw2rQ2cqUQ3I= github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= +github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/PuerkitoBio/goquery v1.5.0 h1:uGvmFXOA73IKluu/F84Xd1tt/z07GYm8X49XKHP7EJk= github.com/PuerkitoBio/goquery v1.5.0/go.mod h1:qD2PgZ9lccMbQlc7eEOjaeRlFQON7xY8kdmcsrnKqMg= @@ -59,6 +60,7 @@ github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4Yn github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4= github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= +github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cheekybits/genny v1.0.0 h1:uGGa4nei+j20rOSeDeP5Of12XVm7TGUd4dJA9RDitfE= github.com/cheekybits/genny v1.0.0/go.mod h1:+tQajlRqAUrPI7DOSpB0XAqZYtQakVtB7wXkRAgjxjQ= @@ -176,6 +178,8 @@ github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/gobuffalo/flect v0.1.5 h1:xpKq9ap8MbYfhuPCF0dBH854Gp9CxZjr/IocxELFflo= github.com/gobuffalo/flect v0.1.5/go.mod h1:W3K3X9ksuZfir8f/LrfVtWmCDQFfayuylOJ7sz/Fj80= +github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= +github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= github.com/gocraft/dbr v0.0.0-20180507214907-a0fd650918f6 h1:kumyNm8Vr8cbVm/aLQYTbDE3SKCbbn5HEVoDp/Dyyfc= github.com/gocraft/dbr v0.0.0-20180507214907-a0fd650918f6/go.mod h1:K/9g3pPouf13kP5K7pdriQEJAy272R9yXuWuDIEWJTM= github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= @@ -296,6 +300,7 @@ github.com/mattn/go-colorable v0.1.2 h1:/bC9yWikZXAL9uJdulbSfyVNIR3n3trXl+v8+1sx github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-isatty v0.0.8 h1:HLtExJ+uU2HOZ+wI0Tt5DtUDrx8yhUqDcp7fYERX4CE= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-runewidth v0.0.0-20181025052659-b20a3daf6a39/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-sqlite3 v1.11.0 h1:LDdKkqtYlom37fkvqs8rMPFKAMe8+SgjbwZ6ex1/A/Q= github.com/mattn/go-sqlite3 v1.11.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= @@ -310,6 +315,7 @@ github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mna/pigeon v0.0.0-20180808201053-bb0192cfc2ae/go.mod h1:Iym28+kJVnC1hfQvv5MUtI6AiFFzvQjHcvI4RFTG/04= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= @@ -326,10 +332,13 @@ github.com/naoina/go-stringutil v0.1.0/go.mod h1:XJ2SJL9jCtBh+P9q5btrd/Ylo8XwT/h github.com/naoina/toml v0.1.1 h1:PT/lllxVVN0gzzSqSlHEmP8MJB4MY2U7STGxiouV4X8= github.com/naoina/toml v0.1.1/go.mod h1:NBIhNtsFMo3G2szEBne+bO4gS192HuIYRqfvOWb4i1E= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= +github.com/olekukonko/tablewriter v0.0.1/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= github.com/onsi/ginkgo v1.8.0 h1:VkHVNpR4iVnU8XQR6DBm8BqYjN7CRzw+xKUbVVbbW9w= github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/gomega v1.5.0 h1:izbySO9zDPmjJ8rDjLvkA2zJHIo+HkYXHnf7eN7SSyo= github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/open-policy-agent/opa v0.18.0 h1:EC81mO3/517Kq5brJHydqKE5MLzJ+4cdJvUQKxLzHy8= +github.com/open-policy-agent/opa v0.18.0/go.mod h1:6pC1cMYDI92i9EY/GoA2m+HcZlcCrh3jbfny5F7JVTA= github.com/opencontainers/go-digest v1.0.0-rc1 h1:WzifXhOVOEOuFYOJAW6aQqW0TooG2iki3E3Ii+WN7gQ= github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= github.com/opencontainers/image-spec v1.0.1 h1:JMemWkRwHx4Zj+fVxWoMCFm/8sYGGrUVojFA6h/TRcI= @@ -344,6 +353,7 @@ github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= +github.com/peterh/liner v0.0.0-20170211195444-bf27d3ba8e1d/go.mod h1:xIteQHvHuaLYG9IFj6mSxM0fCKrs34IrEQUhOYuGPHc= github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -369,6 +379,8 @@ github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084 h1:sofwID9zm4tzr github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/tsdb v0.7.1 h1:YZcsG11NqnK4czYLrWd9mpEuAJIHVQLwdrleYfszMAA= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= +github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a h1:9ZKAASQSHhDYGoxY8uLVpewe1GDZ2vu2Tr/vTdVAkFQ= +github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/remyoudompheng/bigfft v0.0.0-20170806203942-52369c62f446/go.mod h1:uYEyJGbgTkfkS4+E/PavXkNJcbFIpEtjt2B0KDQ5+9M= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/go-charset v0.0.0-20180617210344-2471d30d28b4/go.mod h1:qgYeAmZ5ZIpBWTGllZSQnw97Dj+woV0toclVaRGI8pc= @@ -386,6 +398,7 @@ github.com/soheilhy/cmux v0.1.4 h1:0HKaf1o97UwFjHH9o5XsHUOF+tqmdA7KEzXLpiyaw0E= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/sony/sonyflake v0.0.0-20181109022403-6d5bd6181009 h1:3wBL/e/qjpSYaXacpbIV+Bsj/nwQ4UO1llG/av54zzw= github.com/sony/sonyflake v0.0.0-20181109022403-6d5bd6181009/go.mod h1:dVvZuWJd174umvm5g8CmZD6S2GWwHKtpK/0ZPHswuNo= +github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 h1:qLC7fQah7D6K1B0ujays3HV9gkFtllcxhzImRR7ArPQ= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/speps/go-hashids v2.0.0+incompatible h1:kSfxGfESueJKTx0mpER9Y/1XHl+FVQjtCqRyYcviFbw= github.com/speps/go-hashids v2.0.0+incompatible/go.mod h1:P7hqPzMdnZOfyIk+xrlG1QaSMw+gCBdHKsBDnhpaZvc= @@ -418,6 +431,8 @@ github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xlab/treeprint v0.0.0-20180616005107-d6fb6747feb6/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +github.com/yashtewari/glob-intersection v0.0.0-20180916065949-5c77d914dd0b h1:vVRagRXf67ESqAb72hG2C/ZwI8NtJF2u2V76EsuOHGY= +github.com/yashtewari/glob-intersection v0.0.0-20180916065949-5c77d914dd0b/go.mod h1:HptNXiXVDcJjXe9SqMd0v2FsL9f8dz4GnXgltU6q/co= go.etcd.io/bbolt v1.3.3 h1:MUGmc65QhB3pIlaQ5bB4LwqSj6GIonVJXpZiaKNyaKk= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= diff --git a/pkg/api/iam/authenticate.go b/pkg/api/auth/authenticate.go similarity index 78% rename from pkg/api/iam/authenticate.go rename to pkg/api/auth/authenticate.go index 98c14b520..522e31635 100644 --- a/pkg/api/iam/authenticate.go +++ b/pkg/api/auth/authenticate.go @@ -1,4 +1,22 @@ -package iam +/* + * + * Copyright 2020 The KubeSphere Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * / + */ + +package auth import ( "fmt" diff --git a/pkg/api/auth/types.go b/pkg/api/auth/types.go new file mode 100644 index 000000000..63a80e747 --- /dev/null +++ b/pkg/api/auth/types.go @@ -0,0 +1,48 @@ +/* + * + * Copyright 2020 The KubeSphere Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * / + */ + +package auth + +import "fmt" + +const ( + KindTokenReview = "TokenReview" +) + +type Spec struct { + Token string `json:"token" description:"access token"` +} + +type Status struct { + Authenticated bool `json:"authenticated" description:"is authenticated"` + User map[string]interface{} `json:"user,omitempty" description:"user info"` +} + +type TokenReview struct { + APIVersion string `json:"apiVersion" description:"Kubernetes API version"` + Kind string `json:"kind" description:"kind of the API object"` + Spec *Spec `json:"spec,omitempty"` + Status *Status `json:"status,omitempty" description:"token review status"` +} + +func (request *TokenReview) Validate() error { + if request.Spec == nil || request.Spec.Token == "" { + return fmt.Errorf("token must not be null") + } + return nil +} diff --git a/pkg/api/iam/token/issuer.go b/pkg/api/iam/token/issuer.go deleted file mode 100644 index 199ce26e1..000000000 --- a/pkg/api/iam/token/issuer.go +++ /dev/null @@ -1,10 +0,0 @@ -package token - -// Issuer issues token to user, tokens are required to perform mutating requests to resources -type Issuer interface { - // IssueTo issues a token a User, return error if issuing process failed - IssueTo(User) (string, error) - - // Verify verifies a token, and return a User if it's a valid token, otherwise return error - Verify(string) (User, error) -} diff --git a/pkg/api/iam/token/jwt.go b/pkg/api/iam/token/jwt.go deleted file mode 100644 index b85fe38d5..000000000 --- a/pkg/api/iam/token/jwt.go +++ /dev/null @@ -1,75 +0,0 @@ -package token - -import ( - "fmt" - "github.com/dgrijalva/jwt-go" - "kubesphere.io/kubesphere/pkg/api/iam" - "kubesphere.io/kubesphere/pkg/server/errors" - "time" -) - -const DefaultIssuerName = "kubesphere" - -var errInvalidToken = errors.New("invalid token") - -type claims struct { - Username string `json:"username"` - UID string `json:"uid"` - // Currently, we are not using any field in jwt.StandardClaims - jwt.StandardClaims -} - -type jwtTokenIssuer struct { - name string - secret []byte - keyFunc jwt.Keyfunc -} - -func (s *jwtTokenIssuer) Verify(tokenString string) (User, error) { - if len(tokenString) == 0 { - return nil, errInvalidToken - } - - clm := &claims{} - - _, err := jwt.ParseWithClaims(tokenString, clm, s.keyFunc) - if err != nil { - return nil, err - } - - return &iam.User{Username: clm.Username, Email: clm.UID}, nil -} - -func (s *jwtTokenIssuer) IssueTo(user User) (string, error) { - clm := &claims{ - Username: user.Name(), - UID: user.UID(), - StandardClaims: jwt.StandardClaims{ - IssuedAt: time.Now().Unix(), - Issuer: s.name, - NotBefore: time.Now().Unix(), - }, - } - - token := jwt.NewWithClaims(jwt.SigningMethodHS256, clm) - tokenString, err := token.SignedString(s.secret) - if err != nil { - return "", err - } - - return tokenString, nil -} - -func NewJwtTokenIssuer(issuerName string, secret []byte) Issuer { - return &jwtTokenIssuer{ - name: issuerName, - secret: secret, - keyFunc: func(token *jwt.Token) (i interface{}, err error) { - if _, ok := token.Method.(*jwt.SigningMethodHMAC); ok { - return secret, nil - } else { - return nil, fmt.Errorf("expect token signed with HMAC but got %v", token.Header["alg"]) - } - }, - } -} diff --git a/pkg/api/iam/token/jwt_test.go b/pkg/api/iam/token/jwt_test.go deleted file mode 100644 index ae5343b68..000000000 --- a/pkg/api/iam/token/jwt_test.go +++ /dev/null @@ -1,49 +0,0 @@ -package token - -import ( - "github.com/google/go-cmp/cmp" - "kubesphere.io/kubesphere/pkg/api/iam" - "testing" -) - -func TestJwtTokenIssuer(t *testing.T) { - issuer := NewJwtTokenIssuer(DefaultIssuerName, []byte("kubesphere")) - - testCases := []struct { - description string - name string - email string - }{ - { - name: "admin", - email: "admin@kubesphere.io", - }, - { - name: "bar", - email: "bar@kubesphere.io", - }, - } - - for _, testCase := range testCases { - user := &iam.User{ - Username: testCase.name, - Email: testCase.email, - } - - t.Run(testCase.description, func(t *testing.T) { - token, err := issuer.IssueTo(user) - if err != nil { - t.Fatal(err) - } - - got, err := issuer.Verify(token) - if err != nil { - t.Fatal(err) - } - - if diff := cmp.Diff(user, got); len(diff) != 0 { - t.Errorf("%T differ (-got, +expected), %s", user, diff) - } - }) - } -} diff --git a/pkg/api/iam/token/user.go b/pkg/api/iam/token/user.go deleted file mode 100644 index d29768dd8..000000000 --- a/pkg/api/iam/token/user.go +++ /dev/null @@ -1,8 +0,0 @@ -package token - -type User interface { - // Name - Name() string - - UID() string -} diff --git a/pkg/api/iam/user.go b/pkg/api/iam/user.go index 759ad9cfd..526cf346f 100644 --- a/pkg/api/iam/user.go +++ b/pkg/api/iam/user.go @@ -6,37 +6,30 @@ import ( ) type User struct { - Username string `json:"username"` + Name string `json:"username"` + UID string `json:"uid"` Email string `json:"email"` Lang string `json:"lang,omitempty"` Description string `json:"description"` - CreateTime time.Time `json:"create_time"` + CreateTime time.Time `json:"createTime"` Groups []string `json:"groups,omitempty"` Password string `json:"password,omitempty"` } -func NewUser() *User { - return &User{ - Username: "", - Email: "", - Lang: "", - Description: "", - CreateTime: time.Time{}, - Groups: nil, - Password: "", - } +func (u *User) GetName() string { + return u.Name } -func (u *User) Name() string { - return u.Username +func (u *User) GetUID() string { + return u.UID } -func (u *User) UID() string { +func (u *User) GetEmail() string { return u.Email } func (u *User) Validate() error { - if u.Username == "" { + if u.Name == "" { return errors.New("username can not be empty") } diff --git a/pkg/api/iam/v1alpha2/types.go b/pkg/api/iam/v1alpha2/types.go deleted file mode 100644 index 27ea3fde7..000000000 --- a/pkg/api/iam/v1alpha2/types.go +++ /dev/null @@ -1,110 +0,0 @@ -/* - * - * Copyright 2020 The KubeSphere Authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * / - */ - -package v1alpha2 - -import ( - "fmt" - "kubesphere.io/kubesphere/pkg/api/iam" - "net/mail" -) - -const minPasswordLength = 6 - -type Spec struct { - Token string `json:"token" description:"access token"` -} - -type Status struct { - Authenticated bool `json:"authenticated" description:"is authenticated"` - User map[string]interface{} `json:"user,omitempty" description:"user info"` -} - -type TokenReview struct { - APIVersion string `json:"apiVersion" description:"Kubernetes API version"` - Kind string `json:"kind" description:"kind of the API object"` - Spec *Spec `json:"spec,omitempty"` - Status *Status `json:"status,omitempty" description:"token review status"` -} - -type LoginRequest struct { - Username string `json:"username" description:"username"` - Password string `json:"password" description:"password"` -} - -type UserDetail struct { - *iam.User - ClusterRole string `json:"cluster_role"` -} - -type CreateUserRequest struct { - *UserDetail -} - -func (request *CreateUserRequest) Validate() error { - if request.Username == "" { - return fmt.Errorf("username must not be empty") - } - - // Parses a single RFC 5322 address, e.g. "Barry Gibbs " - if _, err := mail.ParseAddress(request.Email); err != nil { - return fmt.Errorf("invalid email: %s", request.Email) - } - - if len(request.Password) < minPasswordLength { - return fmt.Errorf("password must be at least %d characters long", minPasswordLength) - } - - return nil -} - -type ModifyUserRequest struct { - *UserDetail - CurrentPassword string `json:"current_password,omitempty" description:"this is necessary if you need to change your password"` -} - -func (request *TokenReview) Validate() error { - if request.Spec == nil || request.Spec.Token == "" { - return fmt.Errorf("token must not be null") - } - return nil -} - -func (request ModifyUserRequest) Validate() error { - - // Parses a single RFC 5322 address, e.g. "Barry Gibbs " - if _, err := mail.ParseAddress(request.Email); err != nil { - return fmt.Errorf("invalid email: %s", request.Email) - } - - if request.Password != "" { - if len(request.Password) < minPasswordLength { - return fmt.Errorf("password must be at least %d characters long", minPasswordLength) - } - if len(request.CurrentPassword) < minPasswordLength { - return fmt.Errorf("password must be at least %d characters long", minPasswordLength) - } - - } - return nil -} - -type ListUserResponse struct { - Items []*UserDetail `json:"items"` - TotalCount int `json:"total_count"` -} diff --git a/pkg/apigateway/caddy-plugin/authenticate/authenticate.go b/pkg/apigateway/caddy-plugin/authenticate/authenticate.go deleted file mode 100644 index e37efb9b9..000000000 --- a/pkg/apigateway/caddy-plugin/authenticate/authenticate.go +++ /dev/null @@ -1,248 +0,0 @@ -/* - - Copyright 2019 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - -*/ -package authenticate - -import ( - "errors" - "fmt" - "k8s.io/apimachinery/pkg/util/sets" - "k8s.io/apiserver/pkg/authentication/user" - "k8s.io/apiserver/pkg/endpoints/request" - "k8s.io/klog" - "kubesphere.io/kubesphere/pkg/apigateway/caddy-plugin/internal" - "kubesphere.io/kubesphere/pkg/simple/client/cache" - "log" - "net/http" - "strconv" - "strings" - "time" - - "github.com/dgrijalva/jwt-go" - "github.com/mholt/caddy/caddyhttp/httpserver" -) - -type Auth struct { - Rule *Rule - Next httpserver.Handler -} - -type Rule struct { - Secret []byte - Path string - RedisOptions *cache.Options - TokenIdleTimeout time.Duration - RedisClient cache.Interface - ExclusionRules []internal.ExclusionRule -} - -type User struct { - Username string `json:"username"` - UID string `json:"uid"` - Groups *[]string `json:"groups,omitempty"` - Extra *map[string]interface{} `json:"extra,omitempty"` -} - -var requestInfoFactory = request.RequestInfoFactory{ - APIPrefixes: sets.NewString("api", "apis", "kapis", "kapi"), - GrouplessAPIPrefixes: sets.NewString("api")} - -func (h Auth) ServeHTTP(resp http.ResponseWriter, req *http.Request) (int, error) { - for _, rule := range h.Rule.ExclusionRules { - if httpserver.Path(req.URL.Path).Matches(rule.Path) && (rule.Method == internal.AllMethod || req.Method == rule.Method) { - return h.Next.ServeHTTP(resp, req) - } - } - - if httpserver.Path(req.URL.Path).Matches(h.Rule.Path) { - - uToken, err := h.ExtractToken(req) - - if err != nil { - return h.HandleUnauthorized(resp, err), nil - } - - token, err := h.Validate(uToken) - - if err != nil { - return h.HandleUnauthorized(resp, err), nil - } - - req, err = h.InjectContext(req, token) - - if err != nil { - return h.HandleUnauthorized(resp, err), nil - } - } - - return h.Next.ServeHTTP(resp, req) -} - -func (h Auth) InjectContext(req *http.Request, token *jwt.Token) (*http.Request, error) { - - payload, ok := token.Claims.(jwt.MapClaims) - - if !ok { - return nil, errors.New("invalid payload") - } - - for header := range req.Header { - if strings.HasPrefix(header, "X-Token-") { - req.Header.Del(header) - } - } - - usr := &user.DefaultInfo{} - - username, ok := payload["username"].(string) - - if ok && username != "" { - req.Header.Set("X-Token-Username", username) - usr.Name = username - } - - uid := payload["uid"] - - if uid != nil { - switch uid.(type) { - case int: - req.Header.Set("X-Token-UID", strconv.Itoa(uid.(int))) - usr.UID = strconv.Itoa(uid.(int)) - break - case string: - req.Header.Set("X-Token-UID", uid.(string)) - usr.UID = uid.(string) - break - } - } - - groups, ok := payload["groups"].([]string) - if ok && len(groups) > 0 { - req.Header.Set("X-Token-Groups", strings.Join(groups, ",")) - usr.Groups = groups - } - - // hard code, support jenkins auth plugin - if httpserver.Path(req.URL.Path).Matches("/kapis/jenkins.kubesphere.io") || - httpserver.Path(req.URL.Path).Matches("job") || - httpserver.Path(req.URL.Path).Matches("/kapis/devops.kubesphere.io/v1alpha2") { - req.SetBasicAuth(username, token.Raw) - } - - context := request.WithUser(req.Context(), usr) - - requestInfo, err := requestInfoFactory.NewRequestInfo(req) - - if err == nil { - context = request.WithRequestInfo(context, requestInfo) - } else { - return nil, err - } - - req = req.WithContext(context) - - return req, nil -} - -func (h Auth) Validate(uToken string) (*jwt.Token, error) { - - if len(uToken) == 0 { - return nil, fmt.Errorf("token length is zero") - } - - token, err := jwt.Parse(uToken, h.ProvideKey) - - if err != nil { - klog.Errorln(err) - return nil, err - } - - payload, ok := token.Claims.(jwt.MapClaims) - - if !ok { - err := fmt.Errorf("invalid payload") - klog.Errorln(err) - return nil, err - } - - username, ok := payload["username"].(string) - - if !ok { - err := fmt.Errorf("invalid payload") - klog.Errorln(err) - return nil, err - } - - if _, ok = payload["exp"]; ok { - // allow static token has expiration time - return token, nil - } - - tokenKey := fmt.Sprintf("kubesphere:users:%s:token:%s", username, uToken) - - exist, err := h.Rule.RedisClient.Exists(tokenKey) - if err != nil { - klog.Error(err) - return nil, err - } - - if exist { - // reset expiration time if token exist - h.Rule.RedisClient.Expire(tokenKey, h.Rule.TokenIdleTimeout) - return token, nil - } else { - return nil, errors.New("illegal token") - } -} - -func (h Auth) HandleUnauthorized(w http.ResponseWriter, err error) int { - message := fmt.Sprintf("Unauthorized,%v", err) - w.Header().Add("WWW-Authenticate", message) - log.Println(message) - return http.StatusUnauthorized -} - -func (h Auth) ExtractToken(r *http.Request) (string, error) { - - jwtHeader := strings.Split(r.Header.Get("Authorization"), " ") - - if jwtHeader[0] == "Bearer" && len(jwtHeader) == 2 { - return jwtHeader[1], nil - } - - jwtCookie, err := r.Cookie("token") - - if err == nil { - return jwtCookie.Value, nil - } - - jwtQuery := r.URL.Query().Get("token") - - if jwtQuery != "" { - return jwtQuery, nil - } - - return "", fmt.Errorf("no token found") -} - -func (h Auth) ProvideKey(token *jwt.Token) (interface{}, error) { - if _, ok := token.Method.(*jwt.SigningMethodHMAC); ok { - return h.Rule.Secret, nil - } else { - return nil, fmt.Errorf("expect token signed with HMAC but got %v", token.Header["alg"]) - } -} diff --git a/pkg/apigateway/caddy-plugin/authenticate/auto_load.go b/pkg/apigateway/caddy-plugin/authenticate/auto_load.go deleted file mode 100644 index 65c23d4c3..000000000 --- a/pkg/apigateway/caddy-plugin/authenticate/auto_load.go +++ /dev/null @@ -1,155 +0,0 @@ -/* - - Copyright 2019 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - -*/ -package authenticate - -import ( - "fmt" - "kubesphere.io/kubesphere/pkg/apigateway/caddy-plugin/internal" - "kubesphere.io/kubesphere/pkg/simple/client/cache" - "kubesphere.io/kubesphere/pkg/utils/sliceutil" - "time" - - "github.com/mholt/caddy" - "github.com/mholt/caddy/caddyhttp/httpserver" -) - -func Setup(c *caddy.Controller) error { - - rule, err := parse(c) - - if err != nil { - return err - } - - stopCh := make(chan struct{}) - - c.OnStartup(func() error { - rule.RedisClient, err = cache.NewRedisClient(rule.RedisOptions, stopCh) - // ensure redis is connected when startup - if err != nil { - return err - } - fmt.Println("Authenticate middleware is initiated") - return nil - }) - - c.OnShutdown(func() error { - close(stopCh) - return nil - }) - - httpserver.GetConfig(c).AddMiddleware(func(next httpserver.Handler) httpserver.Handler { - return &Auth{Next: next, Rule: rule} - }) - - return nil -} - -func parse(c *caddy.Controller) (*Rule, error) { - - rule := &Rule{} - rule.ExclusionRules = make([]internal.ExclusionRule, 0) - if c.Next() { - args := c.RemainingArgs() - switch len(args) { - case 0: - for c.NextBlock() { - switch c.Val() { - case "path": - if !c.NextArg() { - return nil, c.ArgErr() - } - - rule.Path = c.Val() - - if c.NextArg() { - return nil, c.ArgErr() - } - case "token-idle-timeout": - if !c.NextArg() { - return nil, c.ArgErr() - } - - if timeout, err := time.ParseDuration(c.Val()); err != nil { - return nil, c.ArgErr() - } else { - rule.TokenIdleTimeout = timeout - } - - if c.NextArg() { - return nil, c.ArgErr() - } - case "redis-url": - if !c.NextArg() { - return nil, c.ArgErr() - } - - options := &cache.Options{Host: c.Val()} - - if err := options.Validate(); len(err) > 0 { - return nil, c.ArgErr() - } else { - rule.RedisOptions = options - } - - if c.NextArg() { - return nil, c.ArgErr() - } - case "secret": - if !c.NextArg() { - return nil, c.ArgErr() - } - - rule.Secret = []byte(c.Val()) - - if c.NextArg() { - return nil, c.ArgErr() - } - case "except": - - if !c.NextArg() { - return nil, c.ArgErr() - } - - method := c.Val() - - if !sliceutil.HasString(internal.HttpMethods, method) { - return nil, c.ArgErr() - } - - for c.NextArg() { - path := c.Val() - rule.ExclusionRules = append(rule.ExclusionRules, internal.ExclusionRule{Method: method, Path: path}) - } - } - } - default: - return nil, c.ArgErr() - } - } - - if c.Next() { - return nil, c.ArgErr() - } - - if rule.RedisOptions == nil { - return nil, c.Err("redis-url must be specified") - } - - return rule, nil -} diff --git a/pkg/apigateway/caddy-plugin/authentication/authentication.go b/pkg/apigateway/caddy-plugin/authentication/authentication.go deleted file mode 100644 index c4b817a29..000000000 --- a/pkg/apigateway/caddy-plugin/authentication/authentication.go +++ /dev/null @@ -1,309 +0,0 @@ -/* - - Copyright 2019 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - -*/ -package authentication - -import ( - "context" - "errors" - "fmt" - "k8s.io/apiserver/pkg/authorization/authorizer" - "k8s.io/apiserver/pkg/endpoints/request" - "k8s.io/client-go/informers" - "kubesphere.io/kubesphere/pkg/apigateway/caddy-plugin/internal" - "kubesphere.io/kubesphere/pkg/models/iam" - "kubesphere.io/kubesphere/pkg/simple/client/k8s" - "log" - "net/http" - "strings" - - "github.com/mholt/caddy/caddyhttp/httpserver" - "k8s.io/api/rbac/v1" - k8serr "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/labels" - "k8s.io/apimachinery/pkg/runtime/schema" - "kubesphere.io/kubesphere/pkg/utils/sliceutil" -) - -type Authentication struct { - Rule *Rule - Next httpserver.Handler - informerFactory informers.SharedInformerFactory -} - -type Rule struct { - Path string - KubernetesOptions *k8s.KubernetesOptions - ExclusionRules []internal.ExclusionRule -} - -func (c *Authentication) ServeHTTP(w http.ResponseWriter, r *http.Request) (int, error) { - - if httpserver.Path(r.URL.Path).Matches(c.Rule.Path) { - - for _, rule := range c.Rule.ExclusionRules { - if httpserver.Path(r.URL.Path).Matches(rule.Path) && (rule.Method == internal.AllMethod || r.Method == rule.Method) { - return c.Next.ServeHTTP(w, r) - } - } - - attrs, err := getAuthorizerAttributes(r.Context()) - - // without authenticate, no requestInfo found in the context - if err != nil { - return c.Next.ServeHTTP(w, r) - } - - permitted, err := c.permissionValidate(attrs) - - if err != nil { - return http.StatusInternalServerError, err - } - - if !permitted { - err = k8serr.NewForbidden(schema.GroupResource{Group: attrs.GetAPIGroup(), Resource: attrs.GetResource()}, attrs.GetName(), fmt.Errorf("permission undefined")) - return handleForbidden(w, err), nil - } - } - - return c.Next.ServeHTTP(w, r) - -} - -func handleForbidden(w http.ResponseWriter, err error) int { - message := fmt.Sprintf("Forbidden,%s", err.Error()) - w.Header().Add("WWW-Authenticate", message) - log.Println(message) - return http.StatusForbidden -} - -func (c *Authentication) permissionValidate(attrs authorizer.Attributes) (bool, error) { - - if attrs.GetResource() == "users" && attrs.GetUser().GetName() == attrs.GetName() { - return true, nil - } - - permitted, err := c.clusterRoleValidate(attrs) - - if err != nil { - log.Println("lister error", err) - return false, err - } - - if permitted { - return true, nil - } - - if attrs.GetNamespace() != "" { - permitted, err = c.roleValidate(attrs) - - if err != nil { - log.Println("lister error", err) - return false, err - } - - if permitted { - return true, nil - } - } - - return false, nil -} - -func (c *Authentication) roleValidate(attrs authorizer.Attributes) (bool, error) { - roleBindingLister := c.informerFactory.Rbac().V1().RoleBindings().Lister() - roleLister := c.informerFactory.Rbac().V1().Roles().Lister() - roleBindings, err := roleBindingLister.RoleBindings(attrs.GetNamespace()).List(labels.Everything()) - - if err != nil { - return false, err - } - - fullSource := attrs.GetResource() - - if attrs.GetSubresource() != "" { - fullSource = fullSource + "/" + attrs.GetSubresource() - } - - for _, roleBinding := range roleBindings { - if iam.ContainsUser(roleBinding.Subjects, attrs.GetUser().GetName()) { - role, err := roleLister.Roles(attrs.GetNamespace()).Get(roleBinding.RoleRef.Name) - - if err != nil { - if k8serr.IsNotFound(err) { - continue - } - return false, err - } - - for _, rule := range role.Rules { - if ruleMatchesRequest(rule, attrs.GetAPIGroup(), "", attrs.GetResource(), attrs.GetSubresource(), attrs.GetName(), attrs.GetVerb()) { - return true, nil - } - } - } - } - - return false, nil -} - -func (c *Authentication) clusterRoleValidate(attrs authorizer.Attributes) (bool, error) { - clusterRoleBindingLister := c.informerFactory.Rbac().V1().ClusterRoleBindings().Lister() - clusterRoleBindings, err := clusterRoleBindingLister.List(labels.Everything()) - clusterRoleLister := c.informerFactory.Rbac().V1().ClusterRoles().Lister() - if err != nil { - return false, err - } - - for _, clusterRoleBinding := range clusterRoleBindings { - - if iam.ContainsUser(clusterRoleBinding.Subjects, attrs.GetUser().GetName()) { - clusterRole, err := clusterRoleLister.Get(clusterRoleBinding.RoleRef.Name) - - if err != nil { - if k8serr.IsNotFound(err) { - continue - } - return false, err - } - - for _, rule := range clusterRole.Rules { - if attrs.IsResourceRequest() { - if ruleMatchesRequest(rule, attrs.GetAPIGroup(), "", attrs.GetResource(), attrs.GetSubresource(), attrs.GetName(), attrs.GetVerb()) { - return true, nil - } - } else { - if ruleMatchesRequest(rule, "", attrs.GetPath(), "", "", "", attrs.GetVerb()) { - return true, nil - } - } - } - - } - } - - return false, nil -} - -func ruleMatchesResources(rule v1.PolicyRule, apiGroup string, resource string, subresource string, resourceName string) bool { - - if resource == "" { - return false - } - - if !sliceutil.HasString(rule.APIGroups, apiGroup) && !sliceutil.HasString(rule.APIGroups, v1.ResourceAll) { - return false - } - - if len(rule.ResourceNames) > 0 && !sliceutil.HasString(rule.ResourceNames, resourceName) { - return false - } - - combinedResource := resource - - if subresource != "" { - combinedResource = combinedResource + "/" + subresource - } - - for _, res := range rule.Resources { - - // match "*" - if res == v1.ResourceAll || res == combinedResource { - return true - } - - // match "*/subresource" - if len(subresource) > 0 && strings.HasPrefix(res, "*/") && subresource == strings.TrimLeft(res, "*/") { - return true - } - // match "resource/*" - if strings.HasSuffix(res, "/*") && resource == strings.TrimRight(res, "/*") { - return true - } - } - - return false -} - -func ruleMatchesRequest(rule v1.PolicyRule, apiGroup string, nonResourceURL string, resource string, subresource string, resourceName string, verb string) bool { - - if !sliceutil.HasString(rule.Verbs, verb) && !sliceutil.HasString(rule.Verbs, v1.VerbAll) { - return false - } - - if nonResourceURL == "" { - return ruleMatchesResources(rule, apiGroup, resource, subresource, resourceName) - } else { - return ruleMatchesNonResource(rule, nonResourceURL) - } -} - -func ruleMatchesNonResource(rule v1.PolicyRule, nonResourceURL string) bool { - - if nonResourceURL == "" { - return false - } - - for _, spec := range rule.NonResourceURLs { - if pathMatches(nonResourceURL, spec) { - return true - } - } - - return false -} - -func pathMatches(path, spec string) bool { - if spec == "*" { - return true - } - if spec == path { - return true - } - if strings.HasSuffix(spec, "*") && strings.HasPrefix(path, strings.TrimRight(spec, "*")) { - return true - } - return false -} - -func getAuthorizerAttributes(ctx context.Context) (authorizer.Attributes, error) { - attribs := authorizer.AttributesRecord{} - - user, ok := request.UserFrom(ctx) - if ok { - attribs.User = user - } - - requestInfo, found := request.RequestInfoFrom(ctx) - if !found { - return nil, errors.New("no RequestInfo found in the context") - } - - // Start with common attributes that apply to resource and non-resource requests - attribs.ResourceRequest = requestInfo.IsResourceRequest - attribs.Path = requestInfo.Path - attribs.Verb = requestInfo.Verb - - attribs.APIGroup = requestInfo.APIGroup - attribs.APIVersion = requestInfo.APIVersion - attribs.Resource = requestInfo.Resource - attribs.Subresource = requestInfo.Subresource - attribs.Namespace = requestInfo.Namespace - attribs.Name = requestInfo.Name - - return &attribs, nil -} diff --git a/pkg/apigateway/caddy-plugin/authentication/auto_load.go b/pkg/apigateway/caddy-plugin/authentication/auto_load.go deleted file mode 100644 index 24f0f17c9..000000000 --- a/pkg/apigateway/caddy-plugin/authentication/auto_load.go +++ /dev/null @@ -1,128 +0,0 @@ -/* - - Copyright 2019 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - -*/ -package authentication - -import ( - "fmt" - "github.com/mholt/caddy" - "github.com/mholt/caddy/caddyhttp/httpserver" - "k8s.io/klog" - "kubesphere.io/kubesphere/pkg/apigateway/caddy-plugin/internal" - "kubesphere.io/kubesphere/pkg/simple/client/k8s" - "kubesphere.io/kubesphere/pkg/utils/sliceutil" - - "kubesphere.io/kubesphere/pkg/informers" -) - -// Setup is called by Caddy to parse the config block -func Setup(c *caddy.Controller) error { - - rule, err := parse(c) - if err != nil { - return err - } - - if rule.KubernetesOptions == nil && rule.KubernetesOptions.KubeConfig == "" { - klog.Warning("no kubeconfig provided, will use in cluster config, this may not work") - } - - kubeClient, err := k8s.NewKubernetesClient(rule.KubernetesOptions) - if err != nil { - return err - } - informerFactory := informers.NewInformerFactories(kubeClient.Kubernetes(), nil, nil, nil) - - stopChan := make(chan struct{}, 0) - c.OnStartup(func() error { - informerFactory.KubernetesSharedInformerFactory().Rbac().V1().Roles().Lister() - informerFactory.KubernetesSharedInformerFactory().Rbac().V1().RoleBindings().Lister() - informerFactory.KubernetesSharedInformerFactory().Rbac().V1().ClusterRoles().Lister() - informerFactory.KubernetesSharedInformerFactory().Rbac().V1().ClusterRoleBindings().Lister() - informerFactory.KubernetesSharedInformerFactory().Start(stopChan) - informerFactory.KubernetesSharedInformerFactory().WaitForCacheSync(stopChan) - fmt.Println("Authentication middleware is initiated") - return nil - }) - - c.OnShutdown(func() error { - close(stopChan) - return nil - }) - - httpserver.GetConfig(c).AddMiddleware(func(next httpserver.Handler) httpserver.Handler { - return &Authentication{Next: next, Rule: rule, informerFactory: informerFactory.KubernetesSharedInformerFactory()} - }) - return nil -} - -func parse(c *caddy.Controller) (*Rule, error) { - - rule := &Rule{} - rule.ExclusionRules = make([]internal.ExclusionRule, 0) - if c.Next() { - args := c.RemainingArgs() - switch len(args) { - case 0: - for c.NextBlock() { - switch c.Val() { - case "path": - if !c.NextArg() { - return rule, c.ArgErr() - } - - rule.Path = c.Val() - - if c.NextArg() { - return rule, c.ArgErr() - } - - break - case "except": - if !c.NextArg() { - return nil, c.ArgErr() - } - - method := c.Val() - - if !sliceutil.HasString(internal.HttpMethods, method) { - return nil, c.ArgErr() - } - - for c.NextArg() { - path := c.Val() - rule.ExclusionRules = append(rule.ExclusionRules, internal.ExclusionRule{Method: method, Path: path}) - } - break - } - } - case 1: - rule.Path = args[0] - if c.NextBlock() { - return rule, c.ArgErr() - } - default: - return rule, c.ArgErr() - } - } - - if c.Next() { - return rule, c.ArgErr() - } - - return rule, nil -} diff --git a/pkg/apigateway/caddy-plugin/swagger/auto_load.go b/pkg/apigateway/caddy-plugin/swagger/auto_load.go deleted file mode 100644 index c71095281..000000000 --- a/pkg/apigateway/caddy-plugin/swagger/auto_load.go +++ /dev/null @@ -1,93 +0,0 @@ -/* - - Copyright 2019 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - -*/ -package authenticate - -import ( - "fmt" - "net/http" - - "github.com/mholt/caddy" - "github.com/mholt/caddy/caddyhttp/httpserver" -) - -func Setup(c *caddy.Controller) error { - - handler, err := parse(c) - - if err != nil { - return err - } - - c.OnStartup(func() error { - fmt.Println("Swagger middleware is initiated") - return nil - }) - - httpserver.GetConfig(c).AddMiddleware(func(next httpserver.Handler) httpserver.Handler { - return &Swagger{Next: next, Handler: handler} - }) - - return nil -} -func parse(c *caddy.Controller) (Handler, error) { - - handler := Handler{URL: "/swagger-ui", FilePath: "/var/static/swagger-ui"} - - if c.Next() { - args := c.RemainingArgs() - switch len(args) { - case 0: - for c.NextBlock() { - switch c.Val() { - case "url": - if !c.NextArg() { - return handler, c.ArgErr() - } - - handler.URL = c.Val() - - if c.NextArg() { - return handler, c.ArgErr() - } - case "filePath": - if !c.NextArg() { - return handler, c.ArgErr() - } - - handler.FilePath = c.Val() - - if c.NextArg() { - return handler, c.ArgErr() - } - default: - return handler, c.ArgErr() - } - } - default: - return handler, c.ArgErr() - } - } - - if c.Next() { - return handler, c.ArgErr() - } - - handler.Handler = http.StripPrefix(handler.URL, http.FileServer(http.Dir(handler.FilePath))) - - return handler, nil -} diff --git a/pkg/apigateway/caddy-plugin/swagger/swagger.go b/pkg/apigateway/caddy-plugin/swagger/swagger.go deleted file mode 100644 index 6c847a447..000000000 --- a/pkg/apigateway/caddy-plugin/swagger/swagger.go +++ /dev/null @@ -1,45 +0,0 @@ -/* - - Copyright 2019 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - -*/ -package authenticate - -import ( - "net/http" - - "github.com/mholt/caddy/caddyhttp/httpserver" -) - -type Swagger struct { - Handler Handler - Next httpserver.Handler -} - -type Handler struct { - URL string - FilePath string - Handler http.Handler -} - -func (h Swagger) ServeHTTP(resp http.ResponseWriter, req *http.Request) (int, error) { - - if httpserver.Path(req.URL.Path).Matches(h.Handler.URL) { - h.Handler.Handler.ServeHTTP(resp, req) - return http.StatusOK, nil - } - - return h.Next.ServeHTTP(resp, req) -} diff --git a/pkg/apigateway/plugins.go b/pkg/apigateway/plugins.go deleted file mode 100644 index a120e45d3..000000000 --- a/pkg/apigateway/plugins.go +++ /dev/null @@ -1,26 +0,0 @@ -package apigateway - -import ( - "github.com/mholt/caddy" - - "kubesphere.io/kubesphere/pkg/apigateway/caddy-plugin/authenticate" - "kubesphere.io/kubesphere/pkg/apigateway/caddy-plugin/authentication" - swagger "kubesphere.io/kubesphere/pkg/apigateway/caddy-plugin/swagger" -) - -func RegisterPlugins() { - caddy.RegisterPlugin("swagger", caddy.Plugin{ - ServerType: "http", - Action: swagger.Setup, - }) - - caddy.RegisterPlugin("authenticate", caddy.Plugin{ - ServerType: "http", - Action: authenticate.Setup, - }) - - caddy.RegisterPlugin("authentication", caddy.Plugin{ - ServerType: "http", - Action: authentication.Setup, - }) -} diff --git a/pkg/apiserver/apiserver.go b/pkg/apiserver/apiserver.go index 23209166c..7f83560d5 100644 --- a/pkg/apiserver/apiserver.go +++ b/pkg/apiserver/apiserver.go @@ -9,13 +9,19 @@ import ( urlruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apiserver/pkg/authentication/request/bearertoken" - "k8s.io/apiserver/pkg/authentication/request/union" - "k8s.io/apiserver/pkg/authorization/authorizerfactory" + unionauth "k8s.io/apiserver/pkg/authentication/request/union" "k8s.io/apiserver/pkg/endpoints/handlers/responsewriters" "k8s.io/klog" - "kubesphere.io/kubesphere/pkg/api/iam" + "kubesphere.io/kubesphere/pkg/api/auth" + "kubesphere.io/kubesphere/pkg/apiserver/authentication/authenticators/basic" "kubesphere.io/kubesphere/pkg/apiserver/authentication/authenticators/jwttoken" - authenticationrequest "kubesphere.io/kubesphere/pkg/apiserver/authentication/request" + oauth2 "kubesphere.io/kubesphere/pkg/apiserver/authentication/oauth" + "kubesphere.io/kubesphere/pkg/apiserver/authentication/request/anonymous" + "kubesphere.io/kubesphere/pkg/apiserver/authentication/request/basictoken" + "kubesphere.io/kubesphere/pkg/apiserver/authentication/token" + "kubesphere.io/kubesphere/pkg/apiserver/authorization/authorizerfactory" + "kubesphere.io/kubesphere/pkg/apiserver/authorization/path" + unionauthorizer "kubesphere.io/kubesphere/pkg/apiserver/authorization/union" "kubesphere.io/kubesphere/pkg/apiserver/dispatch" "kubesphere.io/kubesphere/pkg/apiserver/filters" "kubesphere.io/kubesphere/pkg/apiserver/request" @@ -24,6 +30,7 @@ import ( iamv1alpha2 "kubesphere.io/kubesphere/pkg/kapis/iam/v1alpha2" loggingv1alpha2 "kubesphere.io/kubesphere/pkg/kapis/logging/v1alpha2" monitoringv1alpha2 "kubesphere.io/kubesphere/pkg/kapis/monitoring/v1alpha2" + "kubesphere.io/kubesphere/pkg/kapis/oauth" openpitrixv1 "kubesphere.io/kubesphere/pkg/kapis/openpitrix/v1" operationsv1alpha2 "kubesphere.io/kubesphere/pkg/kapis/operations/v1alpha2" resourcesv1alpha2 "kubesphere.io/kubesphere/pkg/kapis/resources/v1alpha2" @@ -31,6 +38,8 @@ import ( servicemeshv1alpha2 "kubesphere.io/kubesphere/pkg/kapis/servicemesh/metrics/v1alpha2" tenantv1alpha2 "kubesphere.io/kubesphere/pkg/kapis/tenant/v1alpha2" terminalv1alpha2 "kubesphere.io/kubesphere/pkg/kapis/terminal/v1alpha2" + "kubesphere.io/kubesphere/pkg/models/iam/am" + "kubesphere.io/kubesphere/pkg/models/iam/im" "kubesphere.io/kubesphere/pkg/simple/client/cache" "kubesphere.io/kubesphere/pkg/simple/client/devops" "kubesphere.io/kubesphere/pkg/simple/client/k8s" @@ -66,7 +75,7 @@ type APIServer struct { // Server *http.Server - AuthenticateOptions *iam.AuthenticationOptions + AuthenticateOptions *auth.AuthenticationOptions // webservice container, where all webservice defines container *restful.Container @@ -101,8 +110,6 @@ type APIServer struct { // LdapClient ldap.Interface - - // } func (s *APIServer) PrepareRun() error { @@ -140,6 +147,7 @@ func (s *APIServer) installKubeSphereAPIs() { urlruntime.Must(tenantv1alpha2.AddToContainer(s.container, s.KubernetesClient, s.InformerFactory, s.DBClient.Database())) urlruntime.Must(terminalv1alpha2.AddToContainer(s.container, s.KubernetesClient.Kubernetes(), s.KubernetesClient.Config())) urlruntime.Must(iamv1alpha2.AddToContainer(s.container, s.KubernetesClient, s.InformerFactory, s.LdapClient, s.CacheClient, s.AuthenticateOptions)) + urlruntime.Must(oauth.AddToContainer(s.container, token.NewJwtTokenIssuer(token.DefaultIssuerName, s.AuthenticateOptions, s.CacheClient), &oauth2.SimpleConfigManager{})) urlruntime.Must(servicemeshv1alpha2.AddToContainer(s.container)) } @@ -174,19 +182,23 @@ func (s *APIServer) buildHandlerChain() { GrouplessAPIPrefixes: sets.NewString("api", "kapi"), } - failed := http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { - w.WriteHeader(http.StatusUnauthorized) - }) - handler := s.Server.Handler + handler = filters.WithKubeAPIServer(handler, s.KubernetesClient.Config(), &errorResponder{}) handler = filters.WithMultipleClusterDispatcher(handler, dispatch.DefaultClusterDispatch) - handler = filters.WithAuthorization(handler, authorizerfactory.NewAlwaysAllowAuthorizer()) - authn := union.New(&authenticationrequest.AnonymousAuthenticator{}, bearertoken.New(jwttoken.NewTokenAuthenticator(s.CacheClient, s.AuthenticateOptions.JwtSecret))) - handler = filters.WithAuthentication(handler, authn, failed) + excludedPaths := []string{"/oauth/authorize", "/oauth/token"} + pathAuthorizer, _ := path.NewAuthorizer(excludedPaths) + authorizer := unionauthorizer.New(pathAuthorizer, + authorizerfactory.NewOPAAuthorizer(am.NewFakeAMOperator())) + handler = filters.WithAuthorization(handler, authorizer) + + authn := unionauth.New(anonymous.NewAuthenticator(), + basictoken.New(basic.NewBasicAuthenticator(im.NewFakeOperator())), + bearertoken.New(jwttoken.NewTokenAuthenticator( + token.NewJwtTokenIssuer(token.DefaultIssuerName, s.AuthenticateOptions, s.CacheClient)))) + handler = filters.WithAuthentication(handler, authn) handler = filters.WithRequestInfo(handler, requestInfoResolver) - s.Server.Handler = handler } @@ -194,7 +206,7 @@ func (s *APIServer) waitForResourceSync(stopCh <-chan struct{}) error { klog.V(0).Info("Start cache objects") discoveryClient := s.KubernetesClient.Kubernetes().Discovery() - apiResourcesList, err := discoveryClient.ServerResources() + _, apiResourcesList, err := discoveryClient.ServerGroupsAndResources() if err != nil { return err } diff --git a/pkg/apiserver/authentication/authenticators/basic/basic.go b/pkg/apiserver/authentication/authenticators/basic/basic.go new file mode 100644 index 000000000..b97073a7e --- /dev/null +++ b/pkg/apiserver/authentication/authenticators/basic/basic.go @@ -0,0 +1,58 @@ +/* + * + * Copyright 2020 The KubeSphere Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * / + */ + +package basic + +import ( + "context" + "k8s.io/apiserver/pkg/authentication/authenticator" + "k8s.io/apiserver/pkg/authentication/user" + "kubesphere.io/kubesphere/pkg/models/iam/im" +) + +// TokenAuthenticator implements kubernetes token authenticate interface with our custom logic. +// TokenAuthenticator will retrieve user info from cache by given token. If empty or invalid token +// was given, authenticator will still give passed response at the condition user will be user.Anonymous +// and group from user.AllUnauthenticated. This helps requests be passed along the handler chain, +// because some resources are public accessible. +type basicAuthenticator struct { + im im.IdentityManagementInterface +} + +func NewBasicAuthenticator(im im.IdentityManagementInterface) authenticator.Password { + return &basicAuthenticator{ + im: im, + } +} + +func (t *basicAuthenticator) AuthenticatePassword(ctx context.Context, username, password string) (*authenticator.Response, bool, error) { + + providedUser, err := t.im.Authenticate(username, password) + + if err != nil { + return nil, false, err + } + + return &authenticator.Response{ + User: &user.DefaultInfo{ + Name: providedUser.GetName(), + UID: providedUser.GetUID(), + Groups: []string{user.AllAuthenticated}, + }, + }, true, nil +} diff --git a/pkg/apiserver/authentication/authenticators/jwttoken/jwt_token.go b/pkg/apiserver/authentication/authenticators/jwttoken/jwt_token.go index f860d979c..bdbe8bd49 100644 --- a/pkg/apiserver/authentication/authenticators/jwttoken/jwt_token.go +++ b/pkg/apiserver/authentication/authenticators/jwttoken/jwt_token.go @@ -2,56 +2,37 @@ package jwttoken import ( "context" - "fmt" "k8s.io/apiserver/pkg/authentication/authenticator" "k8s.io/apiserver/pkg/authentication/user" - "kubesphere.io/kubesphere/pkg/api/iam/token" - "kubesphere.io/kubesphere/pkg/server/errors" - "kubesphere.io/kubesphere/pkg/simple/client/cache" + token2 "kubesphere.io/kubesphere/pkg/apiserver/authentication/token" ) -var errTokenExpired = errors.New("expired token") - // TokenAuthenticator implements kubernetes token authenticate interface with our custom logic. // TokenAuthenticator will retrieve user info from cache by given token. If empty or invalid token // was given, authenticator will still give passed response at the condition user will be user.Anonymous // and group from user.AllUnauthenticated. This helps requests be passed along the handler chain, // because some resources are public accessible. type tokenAuthenticator struct { - cacheClient cache.Interface - jwtTokenIssuer token.Issuer + jwtTokenIssuer token2.Issuer } -func NewTokenAuthenticator(cacheClient cache.Interface, jwtSecret string) authenticator.Token { +func NewTokenAuthenticator(issuer token2.Issuer) authenticator.Token { return &tokenAuthenticator{ - cacheClient: cacheClient, - jwtTokenIssuer: token.NewJwtTokenIssuer(token.DefaultIssuerName, []byte(jwtSecret)), + jwtTokenIssuer: issuer, } } func (t *tokenAuthenticator) AuthenticateToken(ctx context.Context, token string) (*authenticator.Response, bool, error) { - providedUser, err := t.jwtTokenIssuer.Verify(token) + providedUser, _, err := t.jwtTokenIssuer.Verify(token) if err != nil { return nil, false, err } - _, err = t.cacheClient.Get(tokenKeyForUsername(providedUser.Name(), token)) - if err != nil { - return nil, false, errTokenExpired - } - - // Should we need to refresh token? - return &authenticator.Response{ User: &user.DefaultInfo{ - Name: providedUser.Name(), - UID: providedUser.UID(), + Name: providedUser.GetName(), + UID: providedUser.GetUID(), Groups: []string{user.AllAuthenticated}, }, }, true, nil - -} - -func tokenKeyForUsername(username, token string) string { - return fmt.Sprintf("kubesphere:users:%s:token:%s", username, token) } diff --git a/pkg/apigateway/caddy-plugin/internal/exclusion_rule.go b/pkg/apiserver/authentication/oauth/oauth.go similarity index 62% rename from pkg/apigateway/caddy-plugin/internal/exclusion_rule.go rename to pkg/apiserver/authentication/oauth/oauth.go index 3b6054b3d..12a9a3973 100644 --- a/pkg/apigateway/caddy-plugin/internal/exclusion_rule.go +++ b/pkg/apiserver/authentication/oauth/oauth.go @@ -1,6 +1,6 @@ /* * - * Copyright 2019 The KubeSphere Authors. + * Copyright 2020 The KubeSphere Authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,17 +16,15 @@ * / */ -package internal +package oauth -import "net/http" +import ( + "errors" + "golang.org/x/oauth2" +) -const AllMethod = "*" +var ConfigNotFound = errors.New("config not found") -var HttpMethods = []string{AllMethod, http.MethodPost, http.MethodDelete, - http.MethodPatch, http.MethodPut, http.MethodGet, http.MethodOptions, http.MethodConnect} - -// Path exclusion rule -type ExclusionRule struct { - Method string - Path string +type Configuration interface { + Load(clientId string) (*oauth2.Config, error) } diff --git a/pkg/apiserver/authentication/oauth/simple_config.go b/pkg/apiserver/authentication/oauth/simple_config.go new file mode 100644 index 000000000..7e0597561 --- /dev/null +++ b/pkg/apiserver/authentication/oauth/simple_config.go @@ -0,0 +1,37 @@ +/* + * + * Copyright 2020 The KubeSphere Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * / + */ + +package oauth + +import "golang.org/x/oauth2" + +type SimpleConfigManager struct { +} + +func (s *SimpleConfigManager) Load(clientId string) (*oauth2.Config, error) { + if clientId == "kubesphere-console-client" { + return &oauth2.Config{ + ClientID: "8b21fef43889a28f2bd6", + ClientSecret: "xb21fef43889a28f2bd6", + Endpoint: oauth2.Endpoint{AuthURL: "http://ks-apiserver.kubesphere-system.svc/oauth/authorize", TokenURL: "http://ks-apiserver.kubesphere.io/oauth/token"}, + RedirectURL: "http://ks-console.kubesphere-system.svc/oauth/token/implicit", + Scopes: nil, + }, nil + } + return nil, ConfigNotFound +} diff --git a/pkg/apiserver/authentication/request/anonymous.go b/pkg/apiserver/authentication/request/anonymous.go deleted file mode 100644 index f309e4564..000000000 --- a/pkg/apiserver/authentication/request/anonymous.go +++ /dev/null @@ -1,24 +0,0 @@ -package request - -import ( - "k8s.io/apiserver/pkg/authentication/authenticator" - "k8s.io/apiserver/pkg/authentication/user" - "net/http" - "strings" -) - -type AnonymousAuthenticator struct{} - -func (a *AnonymousAuthenticator) AuthenticateRequest(req *http.Request) (*authenticator.Response, bool, error) { - auth := strings.TrimSpace(req.Header.Get("Authorization")) - if auth == "" { - return &authenticator.Response{ - User: &user.DefaultInfo{ - Name: user.Anonymous, - UID: "", - Groups: []string{user.AllUnauthenticated}, - }, - }, true, nil - } - return nil, false, nil -} diff --git a/pkg/apiserver/authentication/request/anonymous/anonymous.go b/pkg/apiserver/authentication/request/anonymous/anonymous.go new file mode 100644 index 000000000..9ff1115a2 --- /dev/null +++ b/pkg/apiserver/authentication/request/anonymous/anonymous.go @@ -0,0 +1,46 @@ +/* + * + * Copyright 2020 The KubeSphere Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * / + */ + +package anonymous + +import ( + "k8s.io/apiserver/pkg/authentication/authenticator" + "k8s.io/apiserver/pkg/authentication/user" + "net/http" + "strings" +) + +type Authenticator struct{} + +func NewAuthenticator() authenticator.Request { + return &Authenticator{} +} + +func (a *Authenticator) AuthenticateRequest(req *http.Request) (*authenticator.Response, bool, error) { + auth := strings.TrimSpace(req.Header.Get("Authorization")) + if auth == "" { + return &authenticator.Response{ + User: &user.DefaultInfo{ + Name: user.Anonymous, + UID: "", + Groups: []string{user.AllUnauthenticated}, + }, + }, true, nil + } + return nil, false, nil +} diff --git a/pkg/apiserver/authentication/request/basictoken/basic_token.go b/pkg/apiserver/authentication/request/basictoken/basic_token.go new file mode 100644 index 000000000..ebb75f6a5 --- /dev/null +++ b/pkg/apiserver/authentication/request/basictoken/basic_token.go @@ -0,0 +1,56 @@ +/* +Copyright 2014 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package basictoken + +import ( + "errors" + "k8s.io/apiserver/pkg/authentication/authenticator" + "net/http" +) + +type Authenticator struct { + auth authenticator.Password +} + +func New(auth authenticator.Password) *Authenticator { + return &Authenticator{auth} +} + +var invalidToken = errors.New("invalid basic token") + +func (a *Authenticator) AuthenticateRequest(req *http.Request) (*authenticator.Response, bool, error) { + + username, password, ok := req.BasicAuth() + + if !ok { + return nil, false, nil + } + + resp, ok, err := a.auth.AuthenticatePassword(req.Context(), username, password) + // if we authenticated successfully, go ahead and remove the bearer token so that no one + // is ever tempted to use it inside of the API server + if ok { + req.Header.Del("Authorization") + } + + // If the token authenticator didn't error, provide a default error + if !ok && err == nil { + err = invalidToken + } + + return resp, ok, err +} diff --git a/pkg/apiserver/authentication/token/issuer.go b/pkg/apiserver/authentication/token/issuer.go new file mode 100644 index 000000000..5c1fdfa18 --- /dev/null +++ b/pkg/apiserver/authentication/token/issuer.go @@ -0,0 +1,31 @@ +/* + * + * Copyright 2020 The KubeSphere Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * / + */ + +package token + +// Issuer issues token to user, tokens are required to perform mutating requests to resources +type Issuer interface { + // IssueTo issues a token a User, return error if issuing process failed + IssueTo(User) (string, *Claims, error) + + // Verify verifies a token, and return a User if it's a valid token, otherwise return error + Verify(string) (User, *Claims, error) + + // Revoke a token, + Revoke(token string) error +} diff --git a/pkg/apiserver/authentication/token/jwt.go b/pkg/apiserver/authentication/token/jwt.go new file mode 100644 index 000000000..10e8db30e --- /dev/null +++ b/pkg/apiserver/authentication/token/jwt.go @@ -0,0 +1,124 @@ +/* + * + * Copyright 2020 The KubeSphere Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * / + */ + +package token + +import ( + "fmt" + "github.com/dgrijalva/jwt-go" + "kubesphere.io/kubesphere/pkg/api/auth" + "kubesphere.io/kubesphere/pkg/api/iam" + "kubesphere.io/kubesphere/pkg/server/errors" + "kubesphere.io/kubesphere/pkg/simple/client/cache" + "time" +) + +const DefaultIssuerName = "kubesphere" + +var ( + errInvalidToken = errors.New("invalid token") + errTokenExpired = errors.New("expired token") +) + +type Claims struct { + Username string `json:"username"` + UID string `json:"uid"` + // Currently, we are not using any field in jwt.StandardClaims + jwt.StandardClaims +} + +type jwtTokenIssuer struct { + name string + options *auth.AuthenticationOptions + cache cache.Interface + keyFunc jwt.Keyfunc +} + +func (s *jwtTokenIssuer) Verify(tokenString string) (User, *Claims, error) { + if len(tokenString) == 0 { + return nil, nil, errInvalidToken + } + _, err := s.cache.Get(tokenCacheKey(tokenString)) + + if err != nil { + if err == cache.ErrNoSuchKey { + return nil, nil, errTokenExpired + } + return nil, nil, err + } + + clm := &Claims{} + + _, err = jwt.ParseWithClaims(tokenString, clm, s.keyFunc) + if err != nil { + return nil, nil, err + } + + return &iam.User{Name: clm.Username, UID: clm.UID}, clm, nil +} + +func (s *jwtTokenIssuer) IssueTo(user User) (string, *Claims, error) { + clm := &Claims{ + Username: user.GetName(), + UID: user.GetUID(), + StandardClaims: jwt.StandardClaims{ + IssuedAt: time.Now().Unix(), + Issuer: s.name, + NotBefore: time.Now().Unix(), + }, + } + + if s.options.TokenExpiration > 0 { + clm.ExpiresAt = clm.IssuedAt + int64(s.options.TokenExpiration.Seconds()) + } + + token := jwt.NewWithClaims(jwt.SigningMethodHS256, clm) + + tokenString, err := token.SignedString([]byte(s.options.JwtSecret)) + + if err != nil { + return "", nil, err + } + + s.cache.Set(tokenCacheKey(tokenString), tokenString, s.options.TokenExpiration) + + return tokenString, clm, nil +} + +func (s *jwtTokenIssuer) Revoke(token string) error { + return s.cache.Del(tokenCacheKey(token)) +} + +func NewJwtTokenIssuer(issuerName string, options *auth.AuthenticationOptions, cache cache.Interface) Issuer { + return &jwtTokenIssuer{ + name: issuerName, + options: options, + cache: cache, + keyFunc: func(token *jwt.Token) (i interface{}, err error) { + if _, ok := token.Method.(*jwt.SigningMethodHMAC); ok { + return []byte(options.JwtSecret), nil + } else { + return nil, fmt.Errorf("expect token signed with HMAC but got %v", token.Header["alg"]) + } + }, + } +} + +func tokenCacheKey(token string) string { + return fmt.Sprintf("kubesphere:tokens:%s", token) +} diff --git a/pkg/apiserver/authentication/token/jwt_test.go b/pkg/apiserver/authentication/token/jwt_test.go new file mode 100644 index 000000000..bc7c0b380 --- /dev/null +++ b/pkg/apiserver/authentication/token/jwt_test.go @@ -0,0 +1,72 @@ +/* + * + * Copyright 2020 The KubeSphere Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * / + */ + +package token + +import ( + "github.com/google/go-cmp/cmp" + "kubesphere.io/kubesphere/pkg/api/auth" + "kubesphere.io/kubesphere/pkg/api/iam" + "kubesphere.io/kubesphere/pkg/simple/client/cache" + "testing" +) + +func TestJwtTokenIssuer(t *testing.T) { + options := auth.NewAuthenticateOptions() + options.JwtSecret = "kubesphere" + issuer := NewJwtTokenIssuer(DefaultIssuerName, options, cache.NewSimpleCache()) + + testCases := []struct { + description string + name string + uid string + email string + }{ + { + name: "admin", + uid: "b8be6edd-2c92-4535-9b2a-df6326474458", + }, + { + name: "bar", + uid: "b8be6edd-2c92-4535-9b2a-df6326474452", + }, + } + + for _, testCase := range testCases { + user := &iam.User{ + Name: testCase.name, + UID: testCase.uid, + } + + t.Run(testCase.description, func(t *testing.T) { + token, _, err := issuer.IssueTo(user) + if err != nil { + t.Fatal(err) + } + + got, _, err := issuer.Verify(token) + if err != nil { + t.Fatal(err) + } + + if diff := cmp.Diff(user, got); len(diff) != 0 { + t.Errorf("%T differ (-got, +expected), %s", user, diff) + } + }) + } +} diff --git a/pkg/apiserver/authentication/token/token.go b/pkg/apiserver/authentication/token/token.go deleted file mode 100644 index 1765cc067..000000000 --- a/pkg/apiserver/authentication/token/token.go +++ /dev/null @@ -1 +0,0 @@ -package token diff --git a/pkg/apiserver/authentication/token/user.go b/pkg/apiserver/authentication/token/user.go new file mode 100644 index 000000000..ccae5b6ee --- /dev/null +++ b/pkg/apiserver/authentication/token/user.go @@ -0,0 +1,27 @@ +/* + * + * Copyright 2020 The KubeSphere Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * / + */ + +package token + +type User interface { + // Name + GetName() string + + // UID + GetUID() string +} diff --git a/pkg/apiserver/authorization/authorizer/interfaces.go b/pkg/apiserver/authorization/authorizer/interfaces.go new file mode 100644 index 000000000..3c8771328 --- /dev/null +++ b/pkg/apiserver/authorization/authorizer/interfaces.go @@ -0,0 +1,182 @@ +/* +Copyright 2014 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package authorizer + +import ( + "net/http" + + "k8s.io/apiserver/pkg/authentication/user" +) + +// Attributes is an interface used by an Authorizer to get information about a request +// that is used to make an authorization decision. +type Attributes interface { + // GetUser returns the user.Info object to authorize + GetUser() user.Info + + // GetVerb returns the kube verb associated with API requests (this includes get, list, watch, create, update, patch, delete, deletecollection, and proxy), + // or the lowercased HTTP verb associated with non-API requests (this includes get, put, post, patch, and delete) + GetVerb() string + + // When IsReadOnly() == true, the request has no side effects, other than + // caching, logging, and other incidentals. + IsReadOnly() bool + + // Indicates whether or not the request should be handled by kubernetes or kubesphere + IsKubernetesRequest() bool + + // The cluster of the object, if a request is for a REST object. + GetCluster() string + + // The workspace of the object, if a request is for a REST object. + GetWorkspace() string + + // The namespace of the object, if a request is for a REST object. + GetNamespace() string + + // The kind of object, if a request is for a REST object. + GetResource() string + + // GetSubresource returns the subresource being requested, if present + GetSubresource() string + + // GetName returns the name of the object as parsed off the request. This will not be present for all request types, but + // will be present for: get, update, delete + GetName() string + + // The group of the resource, if a request is for a REST object. + GetAPIGroup() string + + // GetAPIVersion returns the version of the group requested, if a request is for a REST object. + GetAPIVersion() string + + // IsResourceRequest returns true for requests to API resources, like /api/v1/nodes, + // and false for non-resource endpoints like /api, /healthz + IsResourceRequest() bool + + // GetPath returns the path of the request + GetPath() string +} + +// Authorizer makes an authorization decision based on information gained by making +// zero or more calls to methods of the Attributes interface. It returns nil when an action is +// authorized, otherwise it returns an error. +type Authorizer interface { + Authorize(a Attributes) (authorized Decision, reason string, err error) +} + +type AuthorizerFunc func(a Attributes) (Decision, string, error) + +func (f AuthorizerFunc) Authorize(a Attributes) (Decision, string, error) { + return f(a) +} + +// RuleResolver provides a mechanism for resolving the list of rules that apply to a given user within a namespace. +type RuleResolver interface { + // RulesFor get the list of cluster wide rules, the list of rules in the specific namespace, incomplete status and errors. + RulesFor(user user.Info, namespace string) ([]ResourceRuleInfo, []NonResourceRuleInfo, bool, error) +} + +// RequestAttributesGetter provides a function that extracts Attributes from an http.Request +type RequestAttributesGetter interface { + GetRequestAttributes(user.Info, *http.Request) Attributes +} + +// AttributesRecord implements Attributes interface. +type AttributesRecord struct { + User user.Info + Verb string + Cluster string + Workspace string + Namespace string + APIGroup string + APIVersion string + Resource string + Subresource string + Name string + KubernetesRequest bool + ResourceRequest bool + Path string +} + +func (a AttributesRecord) GetUser() user.Info { + return a.User +} + +func (a AttributesRecord) GetVerb() string { + return a.Verb +} + +func (a AttributesRecord) IsReadOnly() bool { + return a.Verb == "get" || a.Verb == "list" || a.Verb == "watch" +} + +func (a AttributesRecord) GetCluster() string { + return a.Cluster +} + +func (a AttributesRecord) GetWorkspace() string { + return a.Workspace +} + +func (a AttributesRecord) GetNamespace() string { + return a.Namespace +} + +func (a AttributesRecord) GetResource() string { + return a.Resource +} + +func (a AttributesRecord) GetSubresource() string { + return a.Subresource +} + +func (a AttributesRecord) GetName() string { + return a.Name +} + +func (a AttributesRecord) GetAPIGroup() string { + return a.APIGroup +} + +func (a AttributesRecord) GetAPIVersion() string { + return a.APIVersion +} + +func (a AttributesRecord) IsResourceRequest() bool { + return a.ResourceRequest +} + +func (a AttributesRecord) IsKubernetesRequest() bool { + return a.KubernetesRequest +} + +func (a AttributesRecord) GetPath() string { + return a.Path +} + +type Decision int + +const ( + // DecisionDeny means that an authorizer decided to deny the action. + DecisionDeny Decision = iota + // DecisionAllow means that an authorizer decided to allow the action. + DecisionAllow + // DecisionNoOpionion means that an authorizer has no opinion on whether + // to allow or deny an action. + DecisionNoOpinion +) diff --git a/pkg/apiserver/authorization/authorizer/rule.go b/pkg/apiserver/authorization/authorizer/rule.go new file mode 100644 index 000000000..8f7d9d9ef --- /dev/null +++ b/pkg/apiserver/authorization/authorizer/rule.go @@ -0,0 +1,73 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package authorizer + +type ResourceRuleInfo interface { + // GetVerbs returns a list of kubernetes resource API verbs. + GetVerbs() []string + // GetAPIGroups return the names of the APIGroup that contains the resources. + GetAPIGroups() []string + // GetResources return a list of resources the rule applies to. + GetResources() []string + // GetResourceNames return a white list of names that the rule applies to. + GetResourceNames() []string +} + +// DefaultResourceRuleInfo holds information that describes a rule for the resource +type DefaultResourceRuleInfo struct { + Verbs []string + APIGroups []string + Resources []string + ResourceNames []string +} + +func (i *DefaultResourceRuleInfo) GetVerbs() []string { + return i.Verbs +} + +func (i *DefaultResourceRuleInfo) GetAPIGroups() []string { + return i.APIGroups +} + +func (i *DefaultResourceRuleInfo) GetResources() []string { + return i.Resources +} + +func (i *DefaultResourceRuleInfo) GetResourceNames() []string { + return i.ResourceNames +} + +type NonResourceRuleInfo interface { + // GetVerbs returns a list of kubernetes resource API verbs. + GetVerbs() []string + // GetNonResourceURLs return a set of partial urls that a user should have access to. + GetNonResourceURLs() []string +} + +// DefaultNonResourceRuleInfo holds information that describes a rule for the non-resource +type DefaultNonResourceRuleInfo struct { + Verbs []string + NonResourceURLs []string +} + +func (i *DefaultNonResourceRuleInfo) GetVerbs() []string { + return i.Verbs +} + +func (i *DefaultNonResourceRuleInfo) GetNonResourceURLs() []string { + return i.NonResourceURLs +} diff --git a/pkg/apiserver/authorization/authorizerfactory/opa.go b/pkg/apiserver/authorization/authorizerfactory/opa.go new file mode 100644 index 000000000..609945477 --- /dev/null +++ b/pkg/apiserver/authorization/authorizerfactory/opa.go @@ -0,0 +1,145 @@ +/* + * + * Copyright 2020 The KubeSphere Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * / + */ + +package authorizerfactory + +import ( + "context" + "github.com/open-policy-agent/opa/rego" + "kubesphere.io/kubesphere/pkg/apiserver/authorization/authorizer" + "kubesphere.io/kubesphere/pkg/models/iam/am" +) + +type opaAuthorizer struct { + am am.AccessManagementInterface +} + +// Make decision by request attributes +func (o *opaAuthorizer) Authorize(attr authorizer.Attributes) (authorized authorizer.Decision, reason string, err error) { + + // Make decisions based on the authorization policy of different levels of roles + platformRole, err := o.am.GetPlatformRole(attr.GetUser().GetName()) + if err != nil { + return authorizer.DecisionDeny, "", err + } + + // check platform role policy rules + if authorized, reason, err = makeDecision(platformRole, attr); authorized == authorizer.DecisionAllow { + return authorized, reason, err + } + + // it's not in cluster resource, permission denied + if attr.GetCluster() == "" { + return authorizer.DecisionDeny, "permission undefined", nil + } + + clusterRole, err := o.am.GetClusterRole(attr.GetCluster(), attr.GetUser().GetName()) + if err != nil { + return authorizer.DecisionDeny, "", err + } + + // check cluster role policy rules + if a, r, e := makeDecision(clusterRole, attr); a == authorizer.DecisionAllow { + return a, r, e + } + + // it's not in cluster resource, permission denied + if attr.GetWorkspace() == "" && attr.GetNamespace() == "" { + return authorizer.DecisionDeny, "permission undefined", nil + } + + workspaceRole, err := o.am.GetWorkspaceRole(attr.GetWorkspace(), attr.GetUser().GetName()) + if err != nil { + return authorizer.DecisionDeny, "", err + } + + // check workspace role policy rules + if a, r, e := makeDecision(workspaceRole, attr); a == authorizer.DecisionAllow { + return a, r, e + } + + // it's not in workspace resource, permission denied + if attr.GetNamespace() == "" { + return authorizer.DecisionDeny, "permission undefined", nil + } + + if attr.GetNamespace() != "" { + namespaceRole, err := o.am.GetNamespaceRole(attr.GetCluster(), attr.GetNamespace(), attr.GetUser().GetName()) + if err != nil { + return authorizer.DecisionDeny, "", err + } + // check namespace role policy rules + if a, r, e := makeDecision(namespaceRole, attr); a == authorizer.DecisionAllow { + return a, r, e + } + } + + return authorizer.DecisionDeny, "", nil +} + +// Make decision base on role +func makeDecision(role am.Role, a authorizer.Attributes) (authorized authorizer.Decision, reason string, err error) { + + // Call the rego.New function to create an object that can be prepared or evaluated + // After constructing a new rego.Rego object you can call PrepareForEval() to obtain an executable query + query, err := rego.New(rego.Query("data.authz.allow"), rego.Module("authz.rego", role.GetRego())).PrepareForEval(context.Background()) + + if err != nil { + return authorizer.DecisionDeny, "", err + } + + // data example + //{ + // "User": { + // "Name": "admin", + // "UID": "0", + // "Groups": [ + // "admin" + // ], + // "Extra": null + // }, + // "Verb": "list", + // "Cluster": "cluster1", + // "Workspace": "", + // "Namespace": "", + // "APIGroup": "", + // "APIVersion": "v1", + // "Resource": "nodes", + // "Subresource": "", + // "Name": "", + // "KubernetesRequest": true, + // "ResourceRequest": true, + // "Path": "/api/v1/nodes" + //} + // The policy decision is contained in the results returned by the Eval() call. You can inspect the decision and handle it accordingly. + results, err := query.Eval(context.Background(), rego.EvalInput(a)) + + if err != nil { + return authorizer.DecisionDeny, "", err + } + + if len(results) > 0 && results[0].Expressions[0].Value == true { + return authorizer.DecisionAllow, "", nil + } + + return authorizer.DecisionDeny, "permission undefined", nil +} + +func NewOPAAuthorizer(am am.AccessManagementInterface) *opaAuthorizer { + return &opaAuthorizer{am: am} +} diff --git a/pkg/apiserver/authorization/authorizerfactory/opa_test.go b/pkg/apiserver/authorization/authorizerfactory/opa_test.go new file mode 100644 index 000000000..3781dfae8 --- /dev/null +++ b/pkg/apiserver/authorization/authorizerfactory/opa_test.go @@ -0,0 +1,158 @@ +/* + * + * Copyright 2020 The KubeSphere Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * / + */ + +package authorizerfactory + +import ( + "k8s.io/apiserver/pkg/authentication/user" + "kubesphere.io/kubesphere/pkg/apiserver/authorization/authorizer" + "kubesphere.io/kubesphere/pkg/models/iam/am" + "testing" +) + +func TestPlatformRole(t *testing.T) { + platformRoles := map[string]am.FakeRole{"admin": { + Name: "admin", + Rego: "package authz\ndefault allow = true", + }, "anonymous": { + Name: "anonymous", + Rego: "package authz\ndefault allow = false", + }, "tom": { + Name: "tom", + Rego: `package authz +default allow = false +allow { + resources_in_cluster1 +} +resources_in_cluster1 { + input.Cluster == "cluster1" +}`, + }, + } + + operator := am.NewFakeAMOperator() + operator.Prepare(platformRoles, nil, nil, nil) + + opa := NewOPAAuthorizer(operator) + + tests := []struct { + name string + request authorizer.AttributesRecord + expectedDecision authorizer.Decision + }{ + { + name: "admin can list nodes", + request: authorizer.AttributesRecord{ + User: &user.DefaultInfo{ + Name: "admin", + UID: "0", + Groups: []string{"admin"}, + Extra: nil, + }, + Verb: "list", + Cluster: "", + Workspace: "", + Namespace: "", + APIGroup: "", + APIVersion: "v1", + Resource: "nodes", + Subresource: "", + Name: "", + KubernetesRequest: true, + ResourceRequest: true, + Path: "/api/v1/nodes", + }, + expectedDecision: authorizer.DecisionAllow, + }, + { + name: "anonymous can not list nodes", + request: authorizer.AttributesRecord{ + User: &user.DefaultInfo{ + Name: user.Anonymous, + UID: "0", + Groups: []string{"admin"}, + Extra: nil, + }, + Verb: "list", + Cluster: "", + Workspace: "", + Namespace: "", + APIGroup: "", + APIVersion: "v1", + Resource: "nodes", + Subresource: "", + Name: "", + KubernetesRequest: true, + ResourceRequest: true, + Path: "/api/v1/nodes", + }, + expectedDecision: authorizer.DecisionDeny, + }, { + name: "tom can list nodes in cluster1", + request: authorizer.AttributesRecord{ + User: &user.DefaultInfo{ + Name: "tom", + }, + Verb: "list", + Cluster: "cluster1", + Workspace: "", + Namespace: "", + APIGroup: "", + APIVersion: "v1", + Resource: "nodes", + Subresource: "", + Name: "", + KubernetesRequest: true, + ResourceRequest: true, + Path: "/api/v1/clusters/cluster1/nodes", + }, + expectedDecision: authorizer.DecisionAllow, + }, + { + name: "tom can not list nodes in cluster2", + request: authorizer.AttributesRecord{ + User: &user.DefaultInfo{ + Name: "tom", + }, + Verb: "list", + Cluster: "cluster2", + Workspace: "", + Namespace: "", + APIGroup: "", + APIVersion: "v1", + Resource: "nodes", + Subresource: "", + Name: "", + KubernetesRequest: true, + ResourceRequest: true, + Path: "/api/v1/clusters/cluster2/nodes", + }, + expectedDecision: authorizer.DecisionDeny, + }, + } + + for _, test := range tests { + decision, _, err := opa.Authorize(test.request) + if err != nil { + t.Errorf("test failed: %s, %v", test.name, err) + } + if decision != test.expectedDecision { + t.Errorf("%s: expected decision %v, actual %+v", test.name, test.expectedDecision, decision) + } + } +} diff --git a/pkg/apiserver/authorization/path/doc.go b/pkg/apiserver/authorization/path/doc.go new file mode 100644 index 000000000..654aaeb74 --- /dev/null +++ b/pkg/apiserver/authorization/path/doc.go @@ -0,0 +1,18 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package path contains an authorizer that allows certain paths and path prefixes. +package path // import "k8s.io/apiserver/pkg/authorization/path" diff --git a/pkg/apiserver/authorization/path/path.go b/pkg/apiserver/authorization/path/path.go new file mode 100644 index 000000000..4df9c41a5 --- /dev/null +++ b/pkg/apiserver/authorization/path/path.go @@ -0,0 +1,67 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package path + +import ( + "fmt" + "kubesphere.io/kubesphere/pkg/apiserver/authorization/authorizer" + "strings" + + "k8s.io/apimachinery/pkg/util/sets" +) + +// NewAuthorizer returns an authorizer which accepts a given set of paths. +// Each path is either a fully matching path or it ends in * in case a prefix match is done. A leading / is optional. +func NewAuthorizer(alwaysAllowPaths []string) (authorizer.Authorizer, error) { + var prefixes []string + paths := sets.NewString() + for _, p := range alwaysAllowPaths { + p = strings.TrimPrefix(p, "/") + if len(p) == 0 { + // matches "/" + paths.Insert(p) + continue + } + if strings.ContainsRune(p[:len(p)-1], '*') { + return nil, fmt.Errorf("only trailing * allowed in %q", p) + } + if strings.HasSuffix(p, "*") { + prefixes = append(prefixes, p[:len(p)-1]) + } else { + paths.Insert(p) + } + } + + return authorizer.AuthorizerFunc(func(a authorizer.Attributes) (authorizer.Decision, string, error) { + if a.IsResourceRequest() { + return authorizer.DecisionNoOpinion, "", nil + } + + pth := strings.TrimPrefix(a.GetPath(), "/") + if paths.Has(pth) { + return authorizer.DecisionAllow, "", nil + } + + for _, prefix := range prefixes { + if strings.HasPrefix(pth, prefix) { + return authorizer.DecisionAllow, "", nil + } + } + + return authorizer.DecisionNoOpinion, "", nil + }), nil +} diff --git a/pkg/apiserver/authorization/path/path_test.go b/pkg/apiserver/authorization/path/path_test.go new file mode 100644 index 000000000..73c4eb2f6 --- /dev/null +++ b/pkg/apiserver/authorization/path/path_test.go @@ -0,0 +1,76 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package path + +import ( + "kubesphere.io/kubesphere/pkg/apiserver/authorization/authorizer" + "testing" +) + +func TestNewAuthorizer(t *testing.T) { + tests := []struct { + name string + excludedPaths []string + allowed, denied, noOpinion []string + wantErr bool + }{ + {"inner star", []string{"/foo*bar"}, nil, nil, nil, true}, + {"double star", []string{"/foo**"}, nil, nil, nil, true}, + {"empty", nil, nil, nil, []string{"/"}, false}, + {"slash", []string{"/"}, []string{"/"}, nil, []string{"/foo", "//"}, false}, + {"foo", []string{"/foo"}, []string{"/foo", "foo"}, nil, []string{"/", "", "/bar", "/foo/", "/fooooo", "//foo"}, false}, + {"foo slash", []string{"/foo/"}, []string{"/foo/"}, nil, []string{"/", "", "/bar", "/foo", "/fooooo"}, false}, + {"foo slash star", []string{"/foo/*"}, []string{"/foo/", "/foo/bar/bla"}, nil, []string{"/", "", "/foo", "/bar", "/fooooo"}, false}, + {"foo bar", []string{"/foo", "/bar"}, []string{"/foo", "/bar"}, nil, []string{"/", "", "/foo/", "/bar/", "/fooooo"}, false}, + {"foo star", []string{"/foo*"}, []string{"/foo", "/foooo"}, nil, []string{"/", "", "/fo", "/bar"}, false}, + {"star", []string{"/*"}, []string{"/", "", "/foo", "/foooo"}, nil, nil, false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + a, err := NewAuthorizer(tt.excludedPaths) + if err != nil && !tt.wantErr { + t.Fatalf("unexpected error: %v", err) + } + if err == nil && tt.wantErr { + t.Fatalf("expected error, didn't get any") + } + if err != nil { + return + } + + for _, cases := range []struct { + paths []string + want authorizer.Decision + }{ + {tt.allowed, authorizer.DecisionAllow}, + {tt.denied, authorizer.DecisionDeny}, + {tt.noOpinion, authorizer.DecisionNoOpinion}, + } { + for _, pth := range cases.paths { + info := authorizer.AttributesRecord{ + Path: pth, + } + if got, _, err := a.Authorize(info); err != nil { + t.Errorf("NewAuthorizer(%v).Authorize(%q) return unexpected error: %v", tt.excludedPaths, pth, err) + } else if got != cases.want { + t.Errorf("NewAuthorizer(%v).Authorize(%q) = %v, want %v", tt.excludedPaths, pth, got, cases.want) + } + } + } + }) + } +} diff --git a/pkg/apiserver/authorization/union/union.go b/pkg/apiserver/authorization/union/union.go new file mode 100644 index 000000000..1e3dfac97 --- /dev/null +++ b/pkg/apiserver/authorization/union/union.go @@ -0,0 +1,105 @@ +/* +Copyright 2014 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package union implements an authorizer that combines multiple subauthorizer. +// The union authorizer iterates over each subauthorizer and returns the first +// decision that is either an Allow decision or a Deny decision. If a +// subauthorizer returns a NoOpinion, then the union authorizer moves onto the +// next authorizer or, if the subauthorizer was the last authorizer, returns +// NoOpinion as the aggregate decision. I.e. union authorizer creates an +// aggregate decision and supports short-circuit allows and denies from +// subauthorizers. +package union + +import ( + "kubesphere.io/kubesphere/pkg/apiserver/authorization/authorizer" + "strings" + + utilerrors "k8s.io/apimachinery/pkg/util/errors" + "k8s.io/apiserver/pkg/authentication/user" +) + +// unionAuthzHandler authorizer against a chain of authorizer.Authorizer +type unionAuthzHandler []authorizer.Authorizer + +// New returns an authorizer that authorizes against a chain of authorizer.Authorizer objects +func New(authorizationHandlers ...authorizer.Authorizer) authorizer.Authorizer { + return unionAuthzHandler(authorizationHandlers) +} + +// Authorizes against a chain of authorizer.Authorizer objects and returns nil if successful and returns error if unsuccessful +func (authzHandler unionAuthzHandler) Authorize(a authorizer.Attributes) (authorizer.Decision, string, error) { + var ( + errlist []error + reasonlist []string + ) + + for _, currAuthzHandler := range authzHandler { + decision, reason, err := currAuthzHandler.Authorize(a) + + if err != nil { + errlist = append(errlist, err) + } + if len(reason) != 0 { + reasonlist = append(reasonlist, reason) + } + switch decision { + case authorizer.DecisionAllow, authorizer.DecisionDeny: + return decision, reason, err + case authorizer.DecisionNoOpinion: + // continue to the next authorizer + } + } + + return authorizer.DecisionNoOpinion, strings.Join(reasonlist, "\n"), utilerrors.NewAggregate(errlist) +} + +// unionAuthzRulesHandler authorizer against a chain of authorizer.RuleResolver +type unionAuthzRulesHandler []authorizer.RuleResolver + +// NewRuleResolvers returns an authorizer that authorizes against a chain of authorizer.Authorizer objects +func NewRuleResolvers(authorizationHandlers ...authorizer.RuleResolver) authorizer.RuleResolver { + return unionAuthzRulesHandler(authorizationHandlers) +} + +// RulesFor against a chain of authorizer.RuleResolver objects and returns nil if successful and returns error if unsuccessful +func (authzHandler unionAuthzRulesHandler) RulesFor(user user.Info, namespace string) ([]authorizer.ResourceRuleInfo, []authorizer.NonResourceRuleInfo, bool, error) { + var ( + errList []error + resourceRulesList []authorizer.ResourceRuleInfo + nonResourceRulesList []authorizer.NonResourceRuleInfo + ) + incompleteStatus := false + + for _, currAuthzHandler := range authzHandler { + resourceRules, nonResourceRules, incomplete, err := currAuthzHandler.RulesFor(user, namespace) + + if incomplete == true { + incompleteStatus = true + } + if err != nil { + errList = append(errList, err) + } + if len(resourceRules) > 0 { + resourceRulesList = append(resourceRulesList, resourceRules...) + } + if len(nonResourceRules) > 0 { + nonResourceRulesList = append(nonResourceRulesList, nonResourceRules...) + } + } + + return resourceRulesList, nonResourceRulesList, incompleteStatus, utilerrors.NewAggregate(errList) +} diff --git a/pkg/apiserver/authorization/union/union_test.go b/pkg/apiserver/authorization/union/union_test.go new file mode 100644 index 000000000..592629eeb --- /dev/null +++ b/pkg/apiserver/authorization/union/union_test.go @@ -0,0 +1,265 @@ +/* +Copyright 2014 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package union + +import ( + "errors" + "fmt" + "k8s.io/apiserver/pkg/authentication/user" + "kubesphere.io/kubesphere/pkg/apiserver/authorization/authorizer" + "reflect" + "testing" +) + +type mockAuthzHandler struct { + decision authorizer.Decision + err error +} + +func (mock *mockAuthzHandler) Authorize(a authorizer.Attributes) (authorizer.Decision, string, error) { + return mock.decision, "", mock.err +} + +func TestAuthorizationSecondPasses(t *testing.T) { + handler1 := &mockAuthzHandler{decision: authorizer.DecisionNoOpinion} + handler2 := &mockAuthzHandler{decision: authorizer.DecisionAllow} + authzHandler := New(handler1, handler2) + + authorized, _, _ := authzHandler.Authorize(nil) + if authorized != authorizer.DecisionAllow { + t.Errorf("Unexpected authorization failure") + } +} + +func TestAuthorizationFirstPasses(t *testing.T) { + handler1 := &mockAuthzHandler{decision: authorizer.DecisionAllow} + handler2 := &mockAuthzHandler{decision: authorizer.DecisionNoOpinion} + authzHandler := New(handler1, handler2) + + authorized, _, _ := authzHandler.Authorize(nil) + if authorized != authorizer.DecisionAllow { + t.Errorf("Unexpected authorization failure") + } +} + +func TestAuthorizationNonePasses(t *testing.T) { + handler1 := &mockAuthzHandler{decision: authorizer.DecisionNoOpinion} + handler2 := &mockAuthzHandler{decision: authorizer.DecisionNoOpinion} + authzHandler := New(handler1, handler2) + + authorized, _, _ := authzHandler.Authorize(nil) + if authorized == authorizer.DecisionAllow { + t.Errorf("Expected failed authorization") + } +} + +func TestAuthorizationError(t *testing.T) { + handler1 := &mockAuthzHandler{err: fmt.Errorf("foo")} + handler2 := &mockAuthzHandler{err: fmt.Errorf("foo")} + authzHandler := New(handler1, handler2) + + _, _, err := authzHandler.Authorize(nil) + if err == nil { + t.Errorf("Expected error: %v", err) + } +} + +type mockAuthzRuleHandler struct { + resourceRules []authorizer.ResourceRuleInfo + nonResourceRules []authorizer.NonResourceRuleInfo + err error +} + +func (mock *mockAuthzRuleHandler) RulesFor(user user.Info, namespace string) ([]authorizer.ResourceRuleInfo, []authorizer.NonResourceRuleInfo, bool, error) { + if mock.err != nil { + return []authorizer.ResourceRuleInfo{}, []authorizer.NonResourceRuleInfo{}, false, mock.err + } + return mock.resourceRules, mock.nonResourceRules, false, nil +} + +func TestAuthorizationResourceRules(t *testing.T) { + handler1 := &mockAuthzRuleHandler{ + resourceRules: []authorizer.ResourceRuleInfo{ + &authorizer.DefaultResourceRuleInfo{ + Verbs: []string{"*"}, + APIGroups: []string{"*"}, + Resources: []string{"bindings"}, + }, + &authorizer.DefaultResourceRuleInfo{ + Verbs: []string{"get", "list", "watch"}, + APIGroups: []string{"*"}, + Resources: []string{"*"}, + }, + }, + } + handler2 := &mockAuthzRuleHandler{ + resourceRules: []authorizer.ResourceRuleInfo{ + &authorizer.DefaultResourceRuleInfo{ + Verbs: []string{"*"}, + APIGroups: []string{"*"}, + Resources: []string{"events"}, + }, + &authorizer.DefaultResourceRuleInfo{ + Verbs: []string{"get"}, + APIGroups: []string{"*"}, + Resources: []string{"*"}, + ResourceNames: []string{"foo"}, + }, + }, + } + + expected := []authorizer.DefaultResourceRuleInfo{ + { + Verbs: []string{"*"}, + APIGroups: []string{"*"}, + Resources: []string{"bindings"}, + }, + { + Verbs: []string{"get", "list", "watch"}, + APIGroups: []string{"*"}, + Resources: []string{"*"}, + }, + { + Verbs: []string{"*"}, + APIGroups: []string{"*"}, + Resources: []string{"events"}, + }, + { + Verbs: []string{"get"}, + APIGroups: []string{"*"}, + Resources: []string{"*"}, + ResourceNames: []string{"foo"}, + }, + } + + authzRulesHandler := NewRuleResolvers(handler1, handler2) + + rules, _, _, _ := authzRulesHandler.RulesFor(nil, "") + actual := getResourceRules(rules) + if !reflect.DeepEqual(expected, actual) { + t.Errorf("Expected: \n%#v\n but actual: \n%#v\n", expected, actual) + } +} + +func TestAuthorizationNonResourceRules(t *testing.T) { + handler1 := &mockAuthzRuleHandler{ + nonResourceRules: []authorizer.NonResourceRuleInfo{ + &authorizer.DefaultNonResourceRuleInfo{ + Verbs: []string{"get"}, + NonResourceURLs: []string{"/api"}, + }, + }, + } + + handler2 := &mockAuthzRuleHandler{ + nonResourceRules: []authorizer.NonResourceRuleInfo{ + &authorizer.DefaultNonResourceRuleInfo{ + Verbs: []string{"get"}, + NonResourceURLs: []string{"/api/*"}, + }, + }, + } + + expected := []authorizer.DefaultNonResourceRuleInfo{ + { + Verbs: []string{"get"}, + NonResourceURLs: []string{"/api"}, + }, + { + Verbs: []string{"get"}, + NonResourceURLs: []string{"/api/*"}, + }, + } + + authzRulesHandler := NewRuleResolvers(handler1, handler2) + + _, rules, _, _ := authzRulesHandler.RulesFor(nil, "") + actual := getNonResourceRules(rules) + if !reflect.DeepEqual(expected, actual) { + t.Errorf("Expected: \n%#v\n but actual: \n%#v\n", expected, actual) + } +} + +func getResourceRules(infos []authorizer.ResourceRuleInfo) []authorizer.DefaultResourceRuleInfo { + rules := make([]authorizer.DefaultResourceRuleInfo, len(infos)) + for i, info := range infos { + rules[i] = authorizer.DefaultResourceRuleInfo{ + Verbs: info.GetVerbs(), + APIGroups: info.GetAPIGroups(), + Resources: info.GetResources(), + ResourceNames: info.GetResourceNames(), + } + } + return rules +} + +func getNonResourceRules(infos []authorizer.NonResourceRuleInfo) []authorizer.DefaultNonResourceRuleInfo { + rules := make([]authorizer.DefaultNonResourceRuleInfo, len(infos)) + for i, info := range infos { + rules[i] = authorizer.DefaultNonResourceRuleInfo{ + Verbs: info.GetVerbs(), + NonResourceURLs: info.GetNonResourceURLs(), + } + } + return rules +} + +func TestAuthorizationUnequivocalDeny(t *testing.T) { + cs := []struct { + authorizers []authorizer.Authorizer + decision authorizer.Decision + }{ + { + authorizers: []authorizer.Authorizer{}, + decision: authorizer.DecisionNoOpinion, + }, + { + authorizers: []authorizer.Authorizer{ + &mockAuthzHandler{decision: authorizer.DecisionNoOpinion}, + &mockAuthzHandler{decision: authorizer.DecisionAllow}, + &mockAuthzHandler{decision: authorizer.DecisionDeny}, + }, + decision: authorizer.DecisionAllow, + }, + { + authorizers: []authorizer.Authorizer{ + &mockAuthzHandler{decision: authorizer.DecisionNoOpinion}, + &mockAuthzHandler{decision: authorizer.DecisionDeny}, + &mockAuthzHandler{decision: authorizer.DecisionAllow}, + }, + decision: authorizer.DecisionDeny, + }, + { + authorizers: []authorizer.Authorizer{ + &mockAuthzHandler{decision: authorizer.DecisionNoOpinion}, + &mockAuthzHandler{decision: authorizer.DecisionDeny, err: errors.New("webhook failed closed")}, + &mockAuthzHandler{decision: authorizer.DecisionAllow}, + }, + decision: authorizer.DecisionDeny, + }, + } + for i, c := range cs { + t.Run(fmt.Sprintf("case %v", i), func(t *testing.T) { + authzHandler := New(c.authorizers...) + + decision, _, _ := authzHandler.Authorize(nil) + if decision != c.decision { + t.Errorf("Unexpected authorization failure: %v, expected: %v", decision, c.decision) + } + }) + } +} diff --git a/pkg/apiserver/config/config.go b/pkg/apiserver/config/config.go index 4d6d8d435..d3cfb7d08 100644 --- a/pkg/apiserver/config/config.go +++ b/pkg/apiserver/config/config.go @@ -5,7 +5,7 @@ import ( "github.com/emicklei/go-restful" "github.com/spf13/viper" "k8s.io/apimachinery/pkg/runtime/schema" - "kubesphere.io/kubesphere/pkg/api/iam" + "kubesphere.io/kubesphere/pkg/api/auth" "kubesphere.io/kubesphere/pkg/apiserver/runtime" "kubesphere.io/kubesphere/pkg/simple/client/alerting" "kubesphere.io/kubesphere/pkg/simple/client/cache" @@ -78,7 +78,7 @@ type Config struct { // Options below are only loaded from configuration file, no command line flags for these options now. KubeSphereOptions *kubesphere.Options `json:"-" yaml:"kubesphere,omitempty" mapstructure:"kubesphere"` - AuthenticateOptions *iam.AuthenticationOptions `json:"authentication,omitempty" yaml:"authenticate,omitempty" mapstructure:"authenticate"` + AuthenticateOptions *auth.AuthenticationOptions `json:"authentication,omitempty" yaml:"authenticate,omitempty" mapstructure:"authenticate"` // Options used for enabling components, not actually used now. Once we switch Alerting/Notification API to kubesphere, // we can add these options to kubesphere command lines @@ -103,7 +103,7 @@ func New() *Config { AlertingOptions: alerting.NewAlertingOptions(), NotificationOptions: notification.NewNotificationOptions(), LoggingOptions: elasticsearch.NewElasticSearchOptions(), - AuthenticateOptions: iam.NewAuthenticateOptions(), + AuthenticateOptions: auth.NewAuthenticateOptions(), } } diff --git a/pkg/apiserver/config/config_test.go b/pkg/apiserver/config/config_test.go index cea9d7b77..66d952141 100644 --- a/pkg/apiserver/config/config_test.go +++ b/pkg/apiserver/config/config_test.go @@ -4,7 +4,7 @@ import ( "fmt" "gopkg.in/yaml.v2" "io/ioutil" - iamapi "kubesphere.io/kubesphere/pkg/api/iam" + "kubesphere.io/kubesphere/pkg/api/auth" "kubesphere.io/kubesphere/pkg/simple/client/alerting" "kubesphere.io/kubesphere/pkg/simple/client/cache" "kubesphere.io/kubesphere/pkg/simple/client/devops/jenkins" @@ -64,7 +64,7 @@ func newTestConfig() *Config { GroupSearchBase: "ou=Groups,dc=example,dc=org", }, RedisOptions: &cache.Options{ - Host: "localhost:6379", + Host: "localhost", Port: 6379, Password: "P@88w0rd", DB: 0, @@ -106,7 +106,7 @@ func newTestConfig() *Config { NotificationOptions: ¬ification.Options{ Endpoint: "http://notification.kubesphere-alerting-system.svc:9200", }, - AuthenticateOptions: &iamapi.AuthenticationOptions{ + AuthenticateOptions: &auth.AuthenticationOptions{ AuthenticateRateLimiterMaxTries: 5, AuthenticateRateLimiterDuration: 30 * time.Minute, MaxAuthenticateRetries: 6, diff --git a/pkg/apiserver/filters/authentication.go b/pkg/apiserver/filters/authentication.go index 43d06bb3a..b4a3d92c0 100644 --- a/pkg/apiserver/filters/authentication.go +++ b/pkg/apiserver/filters/authentication.go @@ -1,27 +1,44 @@ package filters import ( + "errors" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/runtime/serializer" "k8s.io/apiserver/pkg/authentication/authenticator" - "k8s.io/apiserver/pkg/endpoints/request" + "k8s.io/apiserver/pkg/endpoints/handlers/responsewriters" "k8s.io/klog" + "kubesphere.io/kubesphere/pkg/apiserver/request" "net/http" ) // WithAuthentication installs authentication handler to handler chain. -func WithAuthentication(handler http.Handler, auth authenticator.Request, failed http.Handler) http.Handler { +func WithAuthentication(handler http.Handler, auth authenticator.Request) http.Handler { if auth == nil { klog.Warningf("Authentication is disabled") return handler } + + s := serializer.NewCodecFactory(runtime.NewScheme()).WithoutConversion() + return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { - //authenticationStart := time.Now() resp, ok, err := auth.AuthenticateRequest(req) if err != nil || !ok { if err != nil { klog.Errorf("Unable to authenticate the request due to error: %v", err) } - failed.ServeHTTP(w, req) + + ctx := req.Context() + requestInfo, found := request.RequestInfoFrom(ctx) + if !found { + responsewriters.InternalError(w, req, errors.New("no RequestInfo found in the context")) + return + } + + gv := schema.GroupVersion{Group: requestInfo.APIGroup, Version: requestInfo.APIVersion} + responsewriters.ErrorNegotiated(apierrors.NewUnauthorized("Unauthorized"), s, gv, w, req) return } diff --git a/pkg/apiserver/filters/authorization.go b/pkg/apiserver/filters/authorization.go index d76d79dab..fb63f97ba 100644 --- a/pkg/apiserver/filters/authorization.go +++ b/pkg/apiserver/filters/authorization.go @@ -3,10 +3,11 @@ package filters import ( "context" "errors" - "k8s.io/apiserver/pkg/authorization/authorizer" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/serializer" "k8s.io/apiserver/pkg/endpoints/handlers/responsewriters" - k8srequest "k8s.io/apiserver/pkg/endpoints/request" "k8s.io/klog" + "kubesphere.io/kubesphere/pkg/apiserver/authorization/authorizer" "kubesphere.io/kubesphere/pkg/apiserver/request" "net/http" ) @@ -18,6 +19,8 @@ func WithAuthorization(handler http.Handler, a authorizer.Authorizer) http.Handl return handler } + serializer := serializer.NewCodecFactory(runtime.NewScheme()).WithoutConversion() + return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { ctx := req.Context() @@ -38,14 +41,14 @@ func WithAuthorization(handler http.Handler, a authorizer.Authorizer) http.Handl } klog.V(4).Infof("Forbidden: %#v, Reason: %q", req.RequestURI, reason) - w.WriteHeader(http.StatusForbidden) + responsewriters.Forbidden(ctx, attributes, w, req, reason, serializer) }) } func GetAuthorizerAttributes(ctx context.Context) (authorizer.Attributes, error) { attribs := authorizer.AttributesRecord{} - user, ok := k8srequest.UserFrom(ctx) + user, ok := request.UserFrom(ctx) if ok { attribs.User = user } @@ -59,6 +62,9 @@ func GetAuthorizerAttributes(ctx context.Context) (authorizer.Attributes, error) attribs.ResourceRequest = requestInfo.IsResourceRequest attribs.Path = requestInfo.Path attribs.Verb = requestInfo.Verb + attribs.Cluster = requestInfo.Cluster + attribs.Workspace = requestInfo.Workspace + attribs.KubernetesRequest = requestInfo.IsKubernetesRequest attribs.APIGroup = requestInfo.APIGroup attribs.APIVersion = requestInfo.APIVersion diff --git a/pkg/apiserver/filters/dispatch.go b/pkg/apiserver/filters/dispatch.go index c893e6dac..1157f4b4c 100644 --- a/pkg/apiserver/filters/dispatch.go +++ b/pkg/apiserver/filters/dispatch.go @@ -7,6 +7,7 @@ import ( "kubesphere.io/kubesphere/pkg/apiserver/dispatch" "kubesphere.io/kubesphere/pkg/apiserver/request" "net/http" + "strings" ) // Multiple cluster dispatcher forward request to desired cluster based on request cluster name @@ -23,9 +24,11 @@ func WithMultipleClusterDispatcher(handler http.Handler, dispatch dispatch.Dispa return } - if info.Cluster == "" { + if info.Cluster == "host-cluster" || info.Cluster == "" { handler.ServeHTTP(w, req) } else { + // remove cluster path + req.URL.Path = strings.Replace(req.URL.Path, fmt.Sprintf("/clusters/%s", info.Cluster), "", 1) dispatch.Dispatch(w, req) } }) diff --git a/pkg/apiserver/filters/kubeapiserver.go b/pkg/apiserver/filters/kubeapiserver.go index c7eb02cfe..009ed5dc6 100644 --- a/pkg/apiserver/filters/kubeapiserver.go +++ b/pkg/apiserver/filters/kubeapiserver.go @@ -1,6 +1,7 @@ package filters import ( + "fmt" "k8s.io/apiserver/pkg/endpoints/handlers/responsewriters" "k8s.io/client-go/rest" "k8s.io/klog" @@ -8,6 +9,7 @@ import ( "kubesphere.io/kubesphere/pkg/server/errors" "net/http" "net/url" + "strings" "k8s.io/apimachinery/pkg/util/proxy" ) @@ -33,6 +35,8 @@ func WithKubeAPIServer(handler http.Handler, config *rest.Config, failed proxy.E s := *req.URL s.Host = kubernetes.Host s.Scheme = kubernetes.Scheme + // remove cluster path + s.Path = strings.Replace(s.Path, fmt.Sprintf("/clusters/%s", info.Cluster), "", 1) httpProxy := proxy.NewUpgradeAwareHandler(&s, defaultTransport, true, false, failed) httpProxy.ServeHTTP(w, req) diff --git a/pkg/apiserver/request/context.go b/pkg/apiserver/request/context.go new file mode 100644 index 000000000..fe3ae38ed --- /dev/null +++ b/pkg/apiserver/request/context.go @@ -0,0 +1,96 @@ +/* +Copyright 2014 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package request + +import ( + "context" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apiserver/pkg/apis/audit" + "k8s.io/apiserver/pkg/authentication/user" +) + +// The key type is unexported to prevent collisions +type key int + +const ( + // namespaceKey is the context key for the request namespace. + namespaceKey key = iota + + // userKey is the context key for the request user. + userKey + + // auditKey is the context key for the audit event. + auditKey + + // audiencesKey is the context key for request audiences. + audiencesKey +) + +// NewContext instantiates a base context object for request flows. +func NewContext() context.Context { + return context.TODO() +} + +// NewDefaultContext instantiates a base context object for request flows in the default namespace +func NewDefaultContext() context.Context { + return WithNamespace(NewContext(), metav1.NamespaceDefault) +} + +// WithValue returns a copy of parent in which the value associated with key is val. +func WithValue(parent context.Context, key interface{}, val interface{}) context.Context { + return context.WithValue(parent, key, val) +} + +// WithNamespace returns a copy of parent in which the namespace value is set +func WithNamespace(parent context.Context, namespace string) context.Context { + return WithValue(parent, namespaceKey, namespace) +} + +// NamespaceFrom returns the value of the namespace key on the ctx +func NamespaceFrom(ctx context.Context) (string, bool) { + namespace, ok := ctx.Value(namespaceKey).(string) + return namespace, ok +} + +// NamespaceValue returns the value of the namespace key on the ctx, or the empty string if none +func NamespaceValue(ctx context.Context) string { + namespace, _ := NamespaceFrom(ctx) + return namespace +} + +// WithUser returns a copy of parent in which the user value is set +func WithUser(parent context.Context, user user.Info) context.Context { + return WithValue(parent, userKey, user) +} + +// UserFrom returns the value of the user key on the ctx +func UserFrom(ctx context.Context) (user.Info, bool) { + user, ok := ctx.Value(userKey).(user.Info) + return user, ok +} + +// WithAuditEvent returns set audit event struct. +func WithAuditEvent(parent context.Context, ev *audit.Event) context.Context { + return WithValue(parent, auditKey, ev) +} + +// AuditEventFrom returns the audit event struct on the ctx +func AuditEventFrom(ctx context.Context) *audit.Event { + ev, _ := ctx.Value(auditKey).(*audit.Event) + return ev +} diff --git a/pkg/apiserver/request/context_test.go b/pkg/apiserver/request/context_test.go new file mode 100644 index 000000000..72b3124b4 --- /dev/null +++ b/pkg/apiserver/request/context_test.go @@ -0,0 +1,93 @@ +/* +Copyright 2014 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package request + +import ( + "testing" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apiserver/pkg/authentication/user" +) + +// TestNamespaceContext validates that a namespace can be get/set on a context object +func TestNamespaceContext(t *testing.T) { + ctx := NewDefaultContext() + result, ok := NamespaceFrom(ctx) + if !ok { + t.Fatalf("Error getting namespace") + } + if metav1.NamespaceDefault != result { + t.Fatalf("Expected: %s, Actual: %s", metav1.NamespaceDefault, result) + } + + ctx = NewContext() + result, ok = NamespaceFrom(ctx) + if ok { + t.Fatalf("Should not be ok because there is no namespace on the context") + } +} + +//TestUserContext validates that a userinfo can be get/set on a context object +func TestUserContext(t *testing.T) { + ctx := NewContext() + _, ok := UserFrom(ctx) + if ok { + t.Fatalf("Should not be ok because there is no user.Info on the context") + } + ctx = WithUser( + ctx, + &user.DefaultInfo{ + Name: "bob", + UID: "123", + Groups: []string{"group1"}, + Extra: map[string][]string{"foo": {"bar"}}, + }, + ) + + result, ok := UserFrom(ctx) + if !ok { + t.Fatalf("Error getting user info") + } + + expectedName := "bob" + if result.GetName() != expectedName { + t.Fatalf("Get user name error, Expected: %s, Actual: %s", expectedName, result.GetName()) + } + + expectedUID := "123" + if result.GetUID() != expectedUID { + t.Fatalf("Get UID error, Expected: %s, Actual: %s", expectedUID, result.GetName()) + } + + expectedGroup := "group1" + actualGroup := result.GetGroups() + if len(actualGroup) != 1 { + t.Fatalf("Get user group number error, Expected: 1, Actual: %d", len(actualGroup)) + } else if actualGroup[0] != expectedGroup { + t.Fatalf("Get user group error, Expected: %s, Actual: %s", expectedGroup, actualGroup[0]) + } + + expectedExtraKey := "foo" + expectedExtraValue := "bar" + actualExtra := result.GetExtra() + if len(actualExtra[expectedExtraKey]) != 1 { + t.Fatalf("Get user extra map number error, Expected: 1, Actual: %d", len(actualExtra[expectedExtraKey])) + } else if actualExtra[expectedExtraKey][0] != expectedExtraValue { + t.Fatalf("Get user extra map value error, Expected: %s, Actual: %s", expectedExtraValue, actualExtra[expectedExtraKey]) + } + +} diff --git a/pkg/apiserver/request/requestinfo.go b/pkg/apiserver/request/requestinfo.go index 4a85ff99a..8f0c53533 100644 --- a/pkg/apiserver/request/requestinfo.go +++ b/pkg/apiserver/request/requestinfo.go @@ -3,7 +3,11 @@ package request import ( "context" "fmt" + "k8s.io/apimachinery/pkg/api/validation/path" + metainternalversion "k8s.io/apimachinery/pkg/apis/meta/internalversion" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/sets" + "k8s.io/klog" "net/http" "strings" @@ -19,6 +23,13 @@ type RequestInfoResolver interface { // master's Mux. var specialVerbs = sets.NewString("proxy", "watch") +// specialVerbsNoSubresources contains root verbs which do not allow subresources +var specialVerbsNoSubresources = sets.NewString("proxy") + +// namespaceSubresources contains subresources of namespace +// this list allows the parser to distinguish between a namespace subresource, and a namespaced resource +var namespaceSubresources = sets.NewString("status", "finalize") + var kubernetesAPIPrefixes = sets.NewString("api", "apis") // RequestInfo holds information parsed from the http.Request, @@ -26,10 +37,10 @@ var kubernetesAPIPrefixes = sets.NewString("api", "apis") type RequestInfo struct { *k8srequest.RequestInfo - // IsKubeSphereRequest indicates whether or not the request should be handled by kubernetes or kubesphere + // IsKubernetesRequest indicates whether or not the request should be handled by kubernetes or kubesphere IsKubernetesRequest bool - // Workspace of requested namespace, for non-workspaced resources, this may be empty + // Workspace of requested resource, for non-workspaced resources, this may be empty Workspace string // Cluster of requested resource, this is empty in single-cluster environment @@ -37,9 +48,8 @@ type RequestInfo struct { } type RequestInfoFactory struct { - APIPrefixes sets.String - GrouplessAPIPrefixes sets.String - k8sRequestInfoFactory *k8srequest.RequestInfoFactory + APIPrefixes sets.String + GrouplessAPIPrefixes sets.String } // NewRequestInfo returns the information from the http request. If error is not nil, RequestInfo holds the information as best it is known before the failure @@ -99,16 +109,9 @@ func (r *RequestInfoFactory) NewRequestInfo(req *http.Request) (*RequestInfo, er currentParts = currentParts[1:] if !r.GrouplessAPIPrefixes.Has(requestInfo.APIPrefix) { - if len(currentParts) < 2 { - return &requestInfo, nil - } - - if currentParts[0] == "clusters" { - requestInfo.Cluster = currentParts[1] - currentParts = currentParts[2:] - } - + // one part (APIPrefix) has already been consumed, so this is actually "do we have four parts?" if len(currentParts) < 3 { + // return a non-resource request return &requestInfo, nil } @@ -120,6 +123,18 @@ func (r *RequestInfoFactory) NewRequestInfo(req *http.Request) (*RequestInfo, er requestInfo.APIVersion = currentParts[0] currentParts = currentParts[1:] + if currentParts[0] == "clusters" { + requestInfo.Cluster = currentParts[1] + currentParts = currentParts[2:] + } else if len(currentParts) > 0 { + requestInfo.Cluster = "host-cluster" + } + + if currentParts[0] == "workspaces" { + requestInfo.Workspace = currentParts[1] + currentParts = currentParts[2:] + } + if specialVerbs.Has(currentParts[0]) { if len(currentParts) < 2 { return &requestInfo, fmt.Errorf("unable to determine kind and namespace from url: %v", req.URL) @@ -144,6 +159,73 @@ func (r *RequestInfoFactory) NewRequestInfo(req *http.Request) (*RequestInfo, er } } + // URL forms: /namespaces/{namespace}/{kind}/*, where parts are adjusted to be relative to kind + if currentParts[0] == "namespaces" { + if len(currentParts) > 1 { + requestInfo.Namespace = currentParts[1] + + // if there is another step after the namespace name and it is not a known namespace subresource + // move currentParts to include it as a resource in its own right + if len(currentParts) > 2 && !namespaceSubresources.Has(currentParts[2]) { + currentParts = currentParts[2:] + } + } + } else { + requestInfo.Namespace = metav1.NamespaceNone + } + + // parsing successful, so we now know the proper value for .Parts + requestInfo.Parts = currentParts + + // parts look like: resource/resourceName/subresource/other/stuff/we/don't/interpret + switch { + case len(requestInfo.Parts) >= 3 && !specialVerbsNoSubresources.Has(requestInfo.Verb): + requestInfo.Subresource = requestInfo.Parts[2] + fallthrough + case len(requestInfo.Parts) >= 2: + requestInfo.Name = requestInfo.Parts[1] + fallthrough + case len(requestInfo.Parts) >= 1: + requestInfo.Resource = requestInfo.Parts[0] + } + + // if there's no name on the request and we thought it was a get before, then the actual verb is a list or a watch + if len(requestInfo.Name) == 0 && requestInfo.Verb == "get" { + opts := metainternalversion.ListOptions{} + if err := metainternalversion.ParameterCodec.DecodeParameters(req.URL.Query(), metav1.SchemeGroupVersion, &opts); err != nil { + // An error in parsing request will result in default to "list" and not setting "name" field. + klog.Errorf("Couldn't parse request %#v: %v", req.URL.Query(), err) + // Reset opts to not rely on partial results from parsing. + // However, if watch is set, let's report it. + opts = metainternalversion.ListOptions{} + if values := req.URL.Query()["watch"]; len(values) > 0 { + switch strings.ToLower(values[0]) { + case "false", "0": + default: + opts.Watch = true + } + } + } + + if opts.Watch { + requestInfo.Verb = "watch" + } else { + requestInfo.Verb = "list" + } + + if opts.FieldSelector != nil { + if name, ok := opts.FieldSelector.RequiresExactMatch("metadata.name"); ok { + if len(path.IsValidPathSegmentName(name)) == 0 { + requestInfo.Name = name + } + } + } + } + // if there's no name on the request and we thought it was a delete before, then the actual verb is deletecollection + if len(requestInfo.Name) == 0 && requestInfo.Verb == "delete" { + requestInfo.Verb = "deletecollection" + } + return &requestInfo, nil } diff --git a/pkg/apiserver/request/requestinfo_test.go b/pkg/apiserver/request/requestinfo_test.go new file mode 100644 index 000000000..3742e180c --- /dev/null +++ b/pkg/apiserver/request/requestinfo_test.go @@ -0,0 +1,178 @@ +/* + * + * Copyright 2020 The KubeSphere Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * / + */ + +package request + +import ( + "k8s.io/apimachinery/pkg/util/sets" + "net/http" + "testing" +) + +func newTestRequestInfoResolver() RequestInfoResolver { + requestInfoResolver := &RequestInfoFactory{ + APIPrefixes: sets.NewString("api", "apis", "kapis", "kapi"), + GrouplessAPIPrefixes: sets.NewString("api", "kapi"), + } + + return requestInfoResolver +} + +func TestRequestInfoFactory_NewRequestInfo(t *testing.T) { + tests := []struct { + name string + url string + method string + expectedErr error + expectedVerb string + expectedResource string + expectedIsResourceRequest bool + expectedCluster string + expectedWorkspace string + exceptedNamespace string + }{ + { + name: "login", + url: "/oauth/authorize?client_id=ks-console&response_type=token", + method: http.MethodPost, + expectedErr: nil, + expectedVerb: "POST", + expectedResource: "", + expectedIsResourceRequest: false, + expectedCluster: "", + }, + { + name: "list cluster roles", + url: "/apis/rbac.authorization.k8s.io/v1/clusters/cluster1/clusterroles", + method: http.MethodGet, + expectedErr: nil, + expectedVerb: "list", + expectedResource: "clusterroles", + expectedIsResourceRequest: true, + expectedCluster: "cluster1", + }, + { + name: "list cluster nodes", + url: "/api/v1/clusters/cluster1/nodes", + method: http.MethodGet, + expectedErr: nil, + expectedVerb: "list", + expectedResource: "nodes", + expectedIsResourceRequest: true, + expectedCluster: "cluster1", + }, + { + name: "list cluster nodes", + url: "/api/v1/clusters/cluster1/nodes", + method: http.MethodGet, + expectedErr: nil, + expectedVerb: "list", + expectedResource: "nodes", + expectedIsResourceRequest: true, + expectedCluster: "cluster1", + }, + { + name: "list cluster nodes", + url: "/api/v1/nodes", + method: http.MethodGet, + expectedErr: nil, + expectedVerb: "list", + expectedResource: "nodes", + expectedIsResourceRequest: true, + expectedCluster: "host-cluster", + }, + { + name: "list roles", + url: "/apis/rbac.authorization.k8s.io/v1/clusters/cluster1/namespaces/namespace1/roles", + method: http.MethodGet, + expectedErr: nil, + expectedVerb: "list", + expectedResource: "roles", + expectedIsResourceRequest: true, + exceptedNamespace: "namespace1", + expectedCluster: "cluster1", + }, + { + name: "list roles", + url: "/apis/rbac.authorization.k8s.io/v1/namespaces/namespace1/roles", + method: http.MethodGet, + expectedErr: nil, + expectedVerb: "list", + expectedResource: "roles", + expectedIsResourceRequest: true, + expectedCluster: "host-cluster", + }, + { + name: "list namespaces", + url: "/kapis/resources.kubesphere.io/v1alpha3/workspaces/workspace1/namespaces", + method: http.MethodGet, + expectedErr: nil, + expectedVerb: "list", + expectedResource: "namespaces", + expectedIsResourceRequest: true, + expectedWorkspace: "workspace1", + expectedCluster: "host-cluster", + }, + { + name: "list namespaces", + url: "/kapis/resources.kubesphere.io/v1alpha3/clusters/cluster1/workspaces/workspace1/namespaces", + method: http.MethodGet, + expectedErr: nil, + expectedVerb: "list", + expectedResource: "namespaces", + expectedIsResourceRequest: true, + expectedWorkspace: "workspace1", + expectedCluster: "cluster1", + }, + } + + requestInfoResolver := newTestRequestInfoResolver() + + for _, test := range tests { + req, err := http.NewRequest(test.method, test.url, nil) + if err != nil { + t.Fatal(err) + } + requestInfo, err := requestInfoResolver.NewRequestInfo(req) + + if err != nil { + if test.expectedErr != err { + t.Errorf("%s: expected error %v, actual %v", test.name, test.expectedErr, err) + } + } else { + if test.expectedVerb != "" && test.expectedVerb != requestInfo.Verb { + t.Errorf("%s: expected verb %v, actual %+v", test.name, test.expectedVerb, requestInfo.Verb) + } + if test.expectedResource != "" && test.expectedResource != requestInfo.Resource { + t.Errorf("%s: expected resource %v, actual %+v", test.name, test.expectedResource, requestInfo.Resource) + } + if test.expectedIsResourceRequest != requestInfo.IsResourceRequest { + t.Errorf("%s: expected is resource request %v, actual %+v", test.name, test.expectedIsResourceRequest, requestInfo.IsResourceRequest) + } + if test.expectedCluster != "" && test.expectedCluster != requestInfo.Cluster { + t.Errorf("%s: expected cluster %v, actual %+v", test.name, test.expectedCluster, requestInfo.Cluster) + } + if test.expectedWorkspace != "" && test.expectedWorkspace != requestInfo.Workspace { + t.Errorf("%s: expected workspace %v, actual %+v", test.name, test.expectedWorkspace, requestInfo.Workspace) + } + if test.exceptedNamespace != "" && test.exceptedNamespace != requestInfo.Namespace { + t.Errorf("%s: expected namespace %v, actual %+v", test.name, test.exceptedNamespace, requestInfo.Namespace) + } + } + } +} diff --git a/pkg/controller/namespace/namespace_controller.go b/pkg/controller/namespace/namespace_controller.go index e0412025d..9a2f38228 100644 --- a/pkg/controller/namespace/namespace_controller.go +++ b/pkg/controller/namespace/namespace_controller.go @@ -32,11 +32,9 @@ import ( "k8s.io/klog" "kubesphere.io/kubesphere/pkg/apis/tenant/v1alpha1" "kubesphere.io/kubesphere/pkg/constants" - "kubesphere.io/kubesphere/pkg/models/iam" "kubesphere.io/kubesphere/pkg/simple/client/openpitrix" "kubesphere.io/kubesphere/pkg/utils/sliceutil" "openpitrix.io/openpitrix/pkg/pb" - "reflect" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" @@ -183,14 +181,6 @@ func (r *ReconcileNamespace) Reconcile(request reconcile.Request) (reconcile.Res return reconcile.Result{}, err } - if err = r.checkAndCreateRoles(instance); err != nil { - return reconcile.Result{}, err - } - - if err = r.checkAndCreateRoleBindings(instance); err != nil { - return reconcile.Result{}, err - } - if err := r.checkAndCreateRuntime(instance); err != nil { return reconcile.Result{}, err } @@ -210,152 +200,6 @@ func (r *ReconcileNamespace) isControlledByWorkspace(namespace *corev1.Namespace return true, nil } -// Create default roles -func (r *ReconcileNamespace) checkAndCreateRoles(namespace *corev1.Namespace) error { - for _, role := range defaultRoles { - found := &rbac.Role{} - err := r.Get(context.TODO(), types.NamespacedName{Namespace: namespace.Name, Name: role.Name}, found) - if err != nil { - if errors.IsNotFound(err) { - role := role.DeepCopy() - role.Namespace = namespace.Name - err = r.Create(context.TODO(), role) - if err != nil { - klog.Error(err) - return err - } - } else { - klog.Error(err) - return err - } - } - if !reflect.DeepEqual(found.Rules, role.Rules) { - found.Rules = role.Rules - if err := r.Update(context.TODO(), found); err != nil { - klog.Error(err) - return err - } - } - } - return nil -} - -func (r *ReconcileNamespace) checkAndCreateRoleBindings(namespace *corev1.Namespace) error { - - workspaceName := namespace.Labels[constants.WorkspaceLabelKey] - creatorName := namespace.Annotations[constants.CreatorAnnotationKey] - - creator := rbac.Subject{APIGroup: "rbac.authorization.k8s.io", Kind: "User", Name: creatorName} - - workspaceAdminBinding := &rbac.ClusterRoleBinding{} - - err := r.Get(context.TODO(), types.NamespacedName{Name: fmt.Sprintf("workspace:%s:admin", workspaceName)}, workspaceAdminBinding) - - if err != nil { - return err - } - - adminBinding := &rbac.RoleBinding{} - adminBinding.Name = admin.Name - adminBinding.Namespace = namespace.Name - adminBinding.RoleRef = rbac.RoleRef{Name: admin.Name, APIGroup: "rbac.authorization.k8s.io", Kind: "Role"} - adminBinding.Subjects = workspaceAdminBinding.Subjects - - if creator.Name != "" { - if adminBinding.Subjects == nil { - adminBinding.Subjects = make([]rbac.Subject, 0) - } - if !iam.ContainsUser(adminBinding.Subjects, creatorName) { - adminBinding.Subjects = append(adminBinding.Subjects, creator) - } - } - - found := &rbac.RoleBinding{} - - err = r.Get(context.TODO(), types.NamespacedName{Namespace: namespace.Name, Name: adminBinding.Name}, found) - - if errors.IsNotFound(err) { - err = r.Create(context.TODO(), adminBinding) - if err != nil { - klog.Errorf("creating role binding namespace: %s,role binding: %s, error: %s", namespace.Name, adminBinding.Name, err) - return err - } - found = adminBinding - } else if err != nil { - klog.Errorf("get role binding namespace: %s,role binding: %s, error: %s", namespace.Name, adminBinding.Name, err) - return err - } - - if !reflect.DeepEqual(found.RoleRef, adminBinding.RoleRef) { - err = r.Delete(context.TODO(), found) - if err != nil { - klog.Errorf("deleting role binding namespace: %s, role binding: %s, error: %s", namespace.Name, adminBinding.Name, err) - return err - } - err = fmt.Errorf("conflict role binding %s.%s, waiting for recreate", namespace.Name, adminBinding.Name) - klog.Errorf("conflict role binding namespace: %s, role binding: %s, error: %s", namespace.Name, adminBinding.Name, err) - return err - } - - if !reflect.DeepEqual(found.Subjects, adminBinding.Subjects) { - found.Subjects = adminBinding.Subjects - err = r.Update(context.TODO(), found) - if err != nil { - klog.Errorf("updating role binding namespace: %s, role binding: %s, error: %s", namespace.Name, adminBinding.Name, err) - return err - } - } - - workspaceViewerBinding := &rbac.ClusterRoleBinding{} - - err = r.Get(context.TODO(), types.NamespacedName{Name: fmt.Sprintf("workspace:%s:viewer", workspaceName)}, workspaceViewerBinding) - - if err != nil { - return err - } - - viewerBinding := &rbac.RoleBinding{} - viewerBinding.Name = viewer.Name - viewerBinding.Namespace = namespace.Name - viewerBinding.RoleRef = rbac.RoleRef{Name: viewer.Name, APIGroup: "rbac.authorization.k8s.io", Kind: "Role"} - viewerBinding.Subjects = workspaceViewerBinding.Subjects - - err = r.Get(context.TODO(), types.NamespacedName{Namespace: namespace.Name, Name: viewerBinding.Name}, found) - - if errors.IsNotFound(err) { - err = r.Create(context.TODO(), viewerBinding) - if err != nil { - klog.Errorf("creating role binding namespace: %s, role binding: %s, error: %s", namespace.Name, viewerBinding.Name, err) - return err - } - found = viewerBinding - } else if err != nil { - return err - } - - if !reflect.DeepEqual(found.RoleRef, viewerBinding.RoleRef) { - err = r.Delete(context.TODO(), found) - if err != nil { - klog.Errorf("deleting conflict role binding namespace: %s, role binding: %s, %s", namespace.Name, viewerBinding.Name, err) - return err - } - err = fmt.Errorf("conflict role binding %s.%s, waiting for recreate", namespace.Name, viewerBinding.Name) - klog.Errorf("conflict role binding namespace: %s, role binding: %s, error: %s", namespace.Name, viewerBinding.Name, err) - return err - } - - if !reflect.DeepEqual(found.Subjects, viewerBinding.Subjects) { - found.Subjects = viewerBinding.Subjects - err = r.Update(context.TODO(), found) - if err != nil { - klog.Errorf("updating role binding namespace: %s, role binding: %s, error: %s", namespace.Name, viewerBinding.Name, err) - return err - } - } - - return nil -} - // Create openpitrix runtime func (r *ReconcileNamespace) checkAndCreateRuntime(namespace *corev1.Namespace) error { diff --git a/pkg/kapis/iam/v1alpha2/handler.go b/pkg/kapis/iam/v1alpha2/handler.go index 121040902..6f53f3070 100644 --- a/pkg/kapis/iam/v1alpha2/handler.go +++ b/pkg/kapis/iam/v1alpha2/handler.go @@ -1,483 +1,79 @@ package v1alpha2 import ( - "errors" - "fmt" "github.com/emicklei/go-restful" - "github.com/go-ldap/ldap" - rbacv1 "k8s.io/api/rbac/v1" - "k8s.io/klog" - "kubesphere.io/kubesphere/pkg/api" - iamv1alpha2 "kubesphere.io/kubesphere/pkg/api/iam/v1alpha2" - "kubesphere.io/kubesphere/pkg/constants" + "kubesphere.io/kubesphere/pkg/api/auth" "kubesphere.io/kubesphere/pkg/informers" - "kubesphere.io/kubesphere/pkg/models/iam" - "kubesphere.io/kubesphere/pkg/models/iam/policy" - "kubesphere.io/kubesphere/pkg/models/resources/v1alpha2" - apierr "kubesphere.io/kubesphere/pkg/server/errors" - "kubesphere.io/kubesphere/pkg/server/params" + "kubesphere.io/kubesphere/pkg/models/iam/am" + "kubesphere.io/kubesphere/pkg/models/iam/im" "kubesphere.io/kubesphere/pkg/simple/client/cache" "kubesphere.io/kubesphere/pkg/simple/client/k8s" ldappool "kubesphere.io/kubesphere/pkg/simple/client/ldap" - "kubesphere.io/kubesphere/pkg/utils/iputil" - "net/http" - - iamapi "kubesphere.io/kubesphere/pkg/api/iam" -) - -const ( - kindTokenReview = "TokenReview" ) type iamHandler struct { - amOperator iam.AccessManagementInterface - imOperator iam.IdentityManagementInterface + amOperator am.AccessManagementInterface + imOperator im.IdentityManagementInterface } -func newIAMHandler(k8sClient k8s.Client, factory informers.InformerFactory, ldapClient ldappool.Interface, cacheClient cache.Interface, options *iamapi.AuthenticationOptions) *iamHandler { +func newIAMHandler(k8sClient k8s.Client, factory informers.InformerFactory, ldapClient ldappool.Interface, cacheClient cache.Interface, options *auth.AuthenticationOptions) *iamHandler { return &iamHandler{ - amOperator: iam.NewAMOperator(k8sClient.Kubernetes(), factory.KubernetesSharedInformerFactory()), - imOperator: iam.NewIMOperator(ldapClient, cacheClient, options), + amOperator: am.NewAMOperator(k8sClient.Kubernetes(), factory.KubernetesSharedInformerFactory()), + imOperator: im.NewLDAPOperator(ldapClient), } } -// Implement webhook authentication interface -// https://kubernetes.io/docs/reference/access-authn-authz/authentication/#webhook-token-authentication -func (h *iamHandler) TokenReviewHandler(req *restful.Request, resp *restful.Response) { - var tokenReview iamv1alpha2.TokenReview - - err := req.ReadEntity(&tokenReview) - - if err != nil { - klog.Error(err) - api.HandleBadRequest(resp, req, err) - return - } - - if err = tokenReview.Validate(); err != nil { - klog.Error(err) - api.HandleBadRequest(resp, req, err) - return - } - - user, err := h.imOperator.VerifyToken(tokenReview.Spec.Token) - - if err != nil { - klog.Errorln(err) - api.HandleInternalError(resp, req, err) - return - } - - success := iamv1alpha2.TokenReview{APIVersion: tokenReview.APIVersion, - Kind: kindTokenReview, - Status: &iamv1alpha2.Status{ - Authenticated: true, - User: map[string]interface{}{"username": user.Username, "uid": user.Username, "groups": user.Groups}, - }, - } - - resp.WriteEntity(success) -} - -func (h *iamHandler) Login(req *restful.Request, resp *restful.Response) { - var loginRequest iamv1alpha2.LoginRequest - - err := req.ReadEntity(&loginRequest) - - if err != nil || loginRequest.Username == "" || loginRequest.Password == "" { - err = errors.New("incorrect username or password") - klog.V(4).Infoln(err) - resp.WriteHeaderAndEntity(http.StatusUnauthorized, err) - return - } - - ip := iputil.RemoteIp(req.Request) - - token, err := h.imOperator.Login(loginRequest.Username, loginRequest.Password, ip) - - if err != nil { - if err == iam.AuthRateLimitExceeded { - klog.V(4).Infoln(err) - resp.WriteHeaderAndEntity(http.StatusTooManyRequests, err) - return - } - klog.V(4).Infoln(err) - resp.WriteHeaderAndEntity(http.StatusUnauthorized, err) - return - } - - resp.WriteEntity(token) -} - func (h *iamHandler) CreateUser(req *restful.Request, resp *restful.Response) { - var createRequest iamv1alpha2.CreateUserRequest - err := req.ReadEntity(&createRequest) - if err != nil { - klog.V(4).Infoln(err) - api.HandleBadRequest(resp, nil, err) - return - } - - if err := createRequest.Validate(); err != nil { - klog.V(4).Infoln(err) - api.HandleBadRequest(resp, nil, err) - return - } - - created, err := h.imOperator.CreateUser(createRequest.User) - - if err != nil { - if err == iam.UserAlreadyExists { - klog.V(4).Infoln(err) - resp.WriteHeaderAndEntity(http.StatusConflict, err) - return - } - klog.Errorln(err) - api.HandleInternalError(resp, nil, err) - return - } - - err = h.amOperator.CreateClusterRoleBinding(created.Username, createRequest.ClusterRole) - - if err != nil { - klog.Errorln(err) - api.HandleInternalError(resp, nil, err) - return - } - - resp.WriteEntity(created) + panic("implement me") } func (h *iamHandler) DeleteUser(req *restful.Request, resp *restful.Response) { - username := req.PathParameter("user") - operator := req.HeaderParameter(constants.UserNameHeader) - - if operator == username { - err := errors.New("cannot delete yourself") - klog.V(4).Infoln(err) - api.HandleForbidden(resp, nil, err) - return - } - - err := h.amOperator.UnBindAllRoles(username) - - if err != nil { - klog.Errorln(err) - api.HandleInternalError(resp, nil, err) - return - } - - err = h.imOperator.DeleteUser(username) - - // TODO release user resources - if err != nil { - klog.Errorln(err) - api.HandleInternalError(resp, nil, err) - return - } - - resp.WriteEntity(apierr.None) + panic("implement me") } func (h *iamHandler) ModifyUser(request *restful.Request, response *restful.Response) { - - username := request.PathParameter("user") - operator := request.HeaderParameter(constants.UserNameHeader) - var modifyUserRequest iamv1alpha2.ModifyUserRequest - - err := request.ReadEntity(&modifyUserRequest) - - if err != nil { - klog.V(4).Infoln(err) - api.HandleBadRequest(response, nil, err) - return - } - - if username != modifyUserRequest.Username { - err = fmt.Errorf("the name of user (%s) does not match the name on the URL (%s)", modifyUserRequest.Username, username) - klog.V(4).Infoln(err) - api.HandleBadRequest(response, nil, err) - return - } - - if err = modifyUserRequest.Validate(); err != nil { - klog.V(4).Infoln(err) - api.HandleBadRequest(response, nil, err) - return - } - - // change password by self - if operator == modifyUserRequest.Username && modifyUserRequest.Password != "" { - - } - - result, err := h.imOperator.ModifyUser(modifyUserRequest.User) - - if err != nil { - klog.Errorln(err) - api.HandleInternalError(response, nil, err) - return - } - - // TODO modify cluster role - - response.WriteEntity(result) + panic("implement me") } func (h *iamHandler) DescribeUser(req *restful.Request, resp *restful.Response) { - username := req.PathParameter("user") - - user, err := h.imOperator.DescribeUser(username) - - if err != nil { - if err == iam.UserNotExists { - klog.V(4).Infoln(err) - api.HandleNotFound(resp, nil, err) - return - } - klog.Errorln(err) - api.HandleInternalError(resp, nil, err) - return - } - - // TODO append more user info - clusterRole, err := h.amOperator.GetClusterRole(username) - - if err != nil { - klog.Errorln(err) - api.HandleInternalError(resp, nil, err) - return - } - - result := iamv1alpha2.UserDetail{ - User: user, - ClusterRole: clusterRole.Name, - } - - resp.WriteEntity(result) + panic("implement me") } func (h *iamHandler) ListUsers(req *restful.Request, resp *restful.Response) { - - limit, offset := params.ParsePaging(req) - orderBy := params.GetStringValueWithDefault(req, params.OrderByParam, v1alpha2.CreateTime) - reverse := params.GetBoolValueWithDefault(req, params.ReverseParam, true) - conditions, err := params.ParseConditions(req) - - if err != nil { - klog.V(4).Infoln(err) - api.HandleBadRequest(resp, nil, err) - return - } - - result, err := h.imOperator.ListUsers(conditions, orderBy, reverse, limit, offset) - - if err != nil { - klog.Errorln(err) - api.HandleInternalError(resp, nil, err) - return - } - - resp.WriteEntity(result) + panic("implement me") } func (h *iamHandler) ListUserRoles(req *restful.Request, resp *restful.Response) { - - username := req.PathParameter("user") - - roles, err := h.imOperator.GetUserRoles(username) - - if err != nil { - klog.Errorln(err) - api.HandleInternalError(resp, nil, err) - return - } - - resp.WriteEntity(roles) + panic("implement me") } func (h *iamHandler) ListRoles(req *restful.Request, resp *restful.Response) { - namespace := req.PathParameter("namespace") - limit, offset := params.ParsePaging(req) - orderBy := params.GetStringValueWithDefault(req, params.OrderByParam, v1alpha2.CreateTime) - reverse := params.GetBoolValueWithDefault(req, params.ReverseParam, true) - conditions, err := params.ParseConditions(req) - - if err != nil { - klog.V(4).Infoln(err) - api.HandleBadRequest(resp, nil, err) - return - } - - result, err := h.amOperator.ListRoles(namespace, conditions, orderBy, reverse, limit, offset) - - if err != nil { - klog.Errorln(err) - api.HandleInternalError(resp, nil, err) - return - } - - resp.WriteAsJson(result) + panic("implement me") } func (h *iamHandler) ListClusterRoles(req *restful.Request, resp *restful.Response) { - limit, offset := params.ParsePaging(req) - orderBy := params.GetStringValueWithDefault(req, params.OrderByParam, v1alpha2.CreateTime) - reverse := params.GetBoolValueWithDefault(req, params.ReverseParam, true) - conditions, err := params.ParseConditions(req) - - if err != nil { - klog.V(4).Infoln(err) - api.HandleBadRequest(resp, nil, err) - return - } - - result, err := h.amOperator.ListClusterRoles(conditions, orderBy, reverse, limit, offset) - - if err != nil { - klog.Errorln(err) - api.HandleInternalError(resp, nil, err) - return - } - - resp.WriteEntity(result) - + panic("implement me") } func (h *iamHandler) ListRoleUsers(req *restful.Request, resp *restful.Response) { - role := req.PathParameter("role") - namespace := req.PathParameter("namespace") - - roleBindings, err := h.amOperator.ListRoleBindings(namespace, role) - - if err != nil { - klog.Errorln(err) - api.HandleInternalError(resp, nil, err) - return - } - result := make([]*iamapi.User, 0) - for _, roleBinding := range roleBindings { - for _, subject := range roleBinding.Subjects { - if subject.Kind == rbacv1.UserKind { - user, err := h.imOperator.DescribeUser(subject.Name) - // skip if user not exist - if ldap.IsErrorWithCode(err, ldap.LDAPResultNoSuchObject) { - continue - } - if err != nil { - klog.Errorln(err) - api.HandleInternalError(resp, nil, err) - return - } - result = append(result, user) - } - } - } - - resp.WriteEntity(result) + panic("implement me") } // List users by namespace func (h *iamHandler) ListNamespaceUsers(req *restful.Request, resp *restful.Response) { - - namespace := req.PathParameter("namespace") - - roleBindings, err := h.amOperator.ListRoleBindings(namespace, "") - - if err != nil { - klog.Errorln(err) - api.HandleInternalError(resp, nil, err) - return - } - - result := make([]*iamapi.User, 0) - for _, roleBinding := range roleBindings { - for _, subject := range roleBinding.Subjects { - if subject.Kind == rbacv1.UserKind { - user, err := h.imOperator.DescribeUser(subject.Name) - // skip if user not exist - if ldap.IsErrorWithCode(err, ldap.LDAPResultNoSuchObject) { - continue - } - if err != nil { - klog.Errorln(err) - api.HandleInternalError(resp, nil, err) - return - } - result = append(result, user) - } - } - } - - resp.WriteEntity(result) + panic("implement me") } func (h *iamHandler) ListClusterRoleUsers(req *restful.Request, resp *restful.Response) { - clusterRole := req.PathParameter("clusterrole") - clusterRoleBindings, err := h.amOperator.ListClusterRoleBindings(clusterRole) - - if err != nil { - klog.Errorln(err) - api.HandleInternalError(resp, nil, err) - return - } - - result := make([]*iamapi.User, 0) - for _, roleBinding := range clusterRoleBindings { - for _, subject := range roleBinding.Subjects { - if subject.Kind == rbacv1.UserKind { - user, err := h.imOperator.DescribeUser(subject.Name) - // skip if user not exist - if ldap.IsErrorWithCode(err, ldap.LDAPResultNoSuchObject) { - continue - } - if err != nil { - klog.Errorln(err) - api.HandleInternalError(resp, nil, err) - return - } - result = append(result, user) - } - } - } - - resp.WriteEntity(result) -} - -func (h *iamHandler) RulesMapping(req *restful.Request, resp *restful.Response) { - rules := policy.RoleRuleMapping - resp.WriteEntity(rules) -} - -func (h *iamHandler) ClusterRulesMapping(req *restful.Request, resp *restful.Response) { - rules := policy.ClusterRoleRuleMapping - resp.WriteEntity(rules) + panic("implement me") } func (h *iamHandler) ListClusterRoleRules(req *restful.Request, resp *restful.Response) { - clusterRole := req.PathParameter("clusterrole") - rules, err := h.amOperator.GetClusterRoleSimpleRules(clusterRole) - if err != nil { - klog.Errorln(err) - api.HandleInternalError(resp, nil, err) - return - } - resp.WriteEntity(rules) + panic("implement me") } func (h *iamHandler) ListRoleRules(req *restful.Request, resp *restful.Response) { - namespace := req.PathParameter("namespace") - role := req.PathParameter("role") - - rules, err := h.amOperator.GetRoleSimpleRules(namespace, role) - - if err != nil { - klog.Errorln(err) - api.HandleInternalError(resp, nil, err) - return - } - - resp.WriteEntity(rules) + panic("implement me") } func (h *iamHandler) ListWorkspaceRoles(request *restful.Request, response *restful.Response) { diff --git a/pkg/kapis/iam/v1alpha2/register.go b/pkg/kapis/iam/v1alpha2/register.go index 424661bfd..f41652a9e 100644 --- a/pkg/kapis/iam/v1alpha2/register.go +++ b/pkg/kapis/iam/v1alpha2/register.go @@ -20,16 +20,13 @@ package v1alpha2 import ( "github.com/emicklei/go-restful" "github.com/emicklei/go-restful-openapi" - rbacv1 "k8s.io/api/rbac/v1" "k8s.io/apimachinery/pkg/runtime/schema" "kubesphere.io/kubesphere/pkg/api" - "kubesphere.io/kubesphere/pkg/api/iam" - iamv1alpha2 "kubesphere.io/kubesphere/pkg/api/iam/v1alpha2" + "kubesphere.io/kubesphere/pkg/api/auth" "kubesphere.io/kubesphere/pkg/apiserver/runtime" "kubesphere.io/kubesphere/pkg/constants" "kubesphere.io/kubesphere/pkg/informers" "kubesphere.io/kubesphere/pkg/models" - "kubesphere.io/kubesphere/pkg/models/iam/policy" "kubesphere.io/kubesphere/pkg/server/errors" "kubesphere.io/kubesphere/pkg/simple/client/cache" "kubesphere.io/kubesphere/pkg/simple/client/k8s" @@ -41,111 +38,44 @@ const groupName = "iam.kubesphere.io" var GroupVersion = schema.GroupVersion{Group: groupName, Version: "v1alpha2"} -func AddToContainer(c *restful.Container, k8sClient k8s.Client, factory informers.InformerFactory, ldapClient ldappool.Interface, cacheClient cache.Interface, options *iam.AuthenticationOptions) error { +func AddToContainer(c *restful.Container, k8sClient k8s.Client, factory informers.InformerFactory, ldapClient ldappool.Interface, cacheClient cache.Interface, options *auth.AuthenticationOptions) error { ws := runtime.NewWebService(GroupVersion) handler := newIAMHandler(k8sClient, factory, ldapClient, cacheClient, options) - ws.Route(ws.POST("/authenticate"). - To(handler.TokenReviewHandler). - Doc("TokenReview attempts to authenticate a token to a known user. Note: TokenReview requests may be cached by the webhook token authenticator plugin in the kube-apiserver."). - Reads(iamv1alpha2.TokenReview{}). - Returns(http.StatusOK, api.StatusOK, iamv1alpha2.TokenReview{}). - Metadata(restfulspec.KeyOpenAPITags, []string{constants.IdentityManagementTag})) - ws.Route(ws.POST("/login"). - To(handler.Login). - Doc("KubeSphere APIs support token-based authentication via the Authtoken request header. The POST Login API is used to retrieve the authentication token. After the authentication token is obtained, it must be inserted into the Authtoken header for all requests."). - Reads(iamv1alpha2.LoginRequest{}). - Returns(http.StatusOK, api.StatusOK, models.AuthGrantResponse{}). - Metadata(restfulspec.KeyOpenAPITags, []string{constants.IdentityManagementTag})) - ws.Route(ws.POST("/users"). - To(handler.CreateUser). - Doc("Create a user account."). - Reads(iamv1alpha2.CreateUserRequest{}). - Returns(http.StatusOK, api.StatusOK, iamv1alpha2.UserDetail{}). - Metadata(restfulspec.KeyOpenAPITags, []string{constants.IdentityManagementTag})) - ws.Route(ws.DELETE("/users/{user}"). - To(handler.DeleteUser). - Doc("Delete the specified user."). - Param(ws.PathParameter("user", "username")). - Returns(http.StatusOK, api.StatusOK, errors.Error{}). - Metadata(restfulspec.KeyOpenAPITags, []string{constants.IdentityManagementTag})) - ws.Route(ws.PUT("/users/{user}"). - To(handler.ModifyUser). - Doc("Update information about the specified user."). - Param(ws.PathParameter("user", "username")). - Reads(iamv1alpha2.ModifyUserRequest{}). - Returns(http.StatusOK, api.StatusOK, errors.Error{}). - Metadata(restfulspec.KeyOpenAPITags, []string{constants.IdentityManagementTag})) - ws.Route(ws.GET("/users/{user}"). - To(handler.DescribeUser). - Doc("Describe the specified user."). - Param(ws.PathParameter("user", "username")). - Returns(http.StatusOK, api.StatusOK, iamv1alpha2.UserDetail{}). - Metadata(restfulspec.KeyOpenAPITags, []string{constants.IdentityManagementTag})) - ws.Route(ws.GET("/users"). - To(handler.ListUsers). - Doc("List all users."). - Returns(http.StatusOK, api.StatusOK, iamv1alpha2.ListUserResponse{}). - Metadata(restfulspec.KeyOpenAPITags, []string{constants.IdentityManagementTag})) - ws.Route(ws.GET("/users/{user}/roles"). - To(handler.ListUserRoles). - Doc("Retrieve all the roles that are assigned to the specified user."). - Param(ws.PathParameter("user", "username")). - Returns(http.StatusOK, api.StatusOK, []*rbacv1.Role{}). - Metadata(restfulspec.KeyOpenAPITags, []string{constants.AccessManagementTag})) + // implemented by create CRD object. + //ws.Route(ws.POST("/users")) + //ws.Route(ws.DELETE("/users/{user}")) + //ws.Route(ws.PUT("/users/{user}")) + //ws.Route(ws.GET("/users/{user}")) + + // TODO move to resources api + //ws.Route(ws.GET("/users")) + ws.Route(ws.GET("/namespaces/{namespace}/roles"). To(handler.ListRoles). Doc("Retrieve the roles that are assigned to the user in the specified namespace."). Param(ws.PathParameter("namespace", "kubernetes namespace")). Returns(http.StatusOK, api.StatusOK, models.PageableResponse{}). Metadata(restfulspec.KeyOpenAPITags, []string{constants.AccessManagementTag})) + ws.Route(ws.GET("/clusterroles"). To(handler.ListClusterRoles). Doc("List all cluster roles."). Returns(http.StatusOK, api.StatusOK, models.PageableResponse{}). Metadata(restfulspec.KeyOpenAPITags, []string{constants.AccessManagementTag})) - ws.Route(ws.GET("/namespaces/{namespace}/roles/{role}/users"). - To(handler.ListRoleUsers). - Doc("Retrieve the users that are bound to the role in the specified namespace."). - Param(ws.PathParameter("namespace", "kubernetes namespace")). - Param(ws.PathParameter("role", "role name")). - Returns(http.StatusOK, api.StatusOK, []iamv1alpha2.ListUserResponse{}). - Metadata(restfulspec.KeyOpenAPITags, []string{constants.AccessManagementTag})) + + // TODO merge + //ws.Route(ws.GET("/namespaces/{namespace}/roles/{role}/users")) ws.Route(ws.GET("/namespaces/{namespace}/users"). To(handler.ListNamespaceUsers). Doc("List all users in the specified namespace."). Param(ws.PathParameter("namespace", "kubernetes namespace")). - Returns(http.StatusOK, api.StatusOK, []iamv1alpha2.ListUserResponse{}). Metadata(restfulspec.KeyOpenAPITags, []string{constants.AccessManagementTag})) ws.Route(ws.GET("/clusterroles/{clusterrole}/users"). To(handler.ListClusterRoleUsers). Doc("List all users that are bound to the specified cluster role."). Param(ws.PathParameter("clusterrole", "cluster role name")). - Returns(http.StatusOK, api.StatusOK, []iamv1alpha2.ListUserResponse{}). - Metadata(restfulspec.KeyOpenAPITags, []string{constants.AccessManagementTag})) - ws.Route(ws.GET("/clusterroles/{clusterrole}/rules"). - To(handler.ListClusterRoleRules). - Doc("List all policy rules of the specified cluster role."). - Param(ws.PathParameter("clusterrole", "cluster role name")). - Returns(http.StatusOK, api.StatusOK, []policy.SimpleRule{}). - Metadata(restfulspec.KeyOpenAPITags, []string{constants.AccessManagementTag})) - ws.Route(ws.GET("/namespaces/{namespace}/roles/{role}/rules"). - To(handler.ListRoleRules). - Doc("List all policy rules of the specified role in the given namespace."). - Param(ws.PathParameter("namespace", "kubernetes namespace")). - Param(ws.PathParameter("role", "role name")). - Returns(http.StatusOK, api.StatusOK, []policy.SimpleRule{}). - Metadata(restfulspec.KeyOpenAPITags, []string{constants.AccessManagementTag})) - ws.Route(ws.GET("/rulesmapping/clusterroles"). - To(handler.ClusterRulesMapping). - Doc("Get the mapping relationships between cluster roles and policy rules."). - Returns(http.StatusOK, api.StatusOK, policy.ClusterRoleRuleMapping). - Metadata(restfulspec.KeyOpenAPITags, []string{constants.AccessManagementTag})) - ws.Route(ws.GET("/rulesmapping/roles"). - To(handler.RulesMapping). - Doc("Get the mapping relationships between namespaced roles and policy rules."). - Returns(http.StatusOK, api.StatusOK, policy.RoleRuleMapping). Metadata(restfulspec.KeyOpenAPITags, []string{constants.AccessManagementTag})) ws.Route(ws.GET("/workspaces/{workspace}/roles"). @@ -153,23 +83,14 @@ func AddToContainer(c *restful.Container, k8sClient k8s.Client, factory informer Doc("List all workspace roles."). Param(ws.PathParameter("workspace", "workspace name")). Metadata(restfulspec.KeyOpenAPITags, []string{constants.AccessManagementTag})) - ws.Route(ws.GET("/workspaces/{workspace}/roles/{role}"). - To(handler.DescribeWorkspaceRole). - Doc("Describe the workspace role."). - Param(ws.PathParameter("workspace", "workspace name")). - Param(ws.PathParameter("role", "workspace role name")). - Metadata(restfulspec.KeyOpenAPITags, []string{constants.AccessManagementTag})) - ws.Route(ws.GET("/workspaces/{workspace}/roles/{role}/rules"). - To(handler.ListWorkspaceRoleRules). - Doc("List all policy rules of the specified workspace role."). - Param(ws.PathParameter("workspace", "workspace name")). - Param(ws.PathParameter("role", "workspace role name")). - Metadata(restfulspec.KeyOpenAPITags, []string{constants.AccessManagementTag})) + ws.Route(ws.GET("/workspaces/{workspace}/members"). To(handler.ListWorkspaceUsers). Doc("List all members in the specified workspace."). Param(ws.PathParameter("workspace", "workspace name")). Metadata(restfulspec.KeyOpenAPITags, []string{constants.AccessManagementTag})) + + // TODO re-design ws.Route(ws.POST("/workspaces/{workspace}/members"). To(handler.InviteUser). Doc("Invite a member to the specified workspace."). @@ -182,12 +103,7 @@ func AddToContainer(c *restful.Container, k8sClient k8s.Client, factory informer Param(ws.PathParameter("member", "username")). Returns(http.StatusOK, api.StatusOK, errors.Error{}). Metadata(restfulspec.KeyOpenAPITags, []string{constants.AccessManagementTag})) - ws.Route(ws.GET("/workspaces/{workspace}/members/{member}"). - To(handler.DescribeWorkspaceUser). - Doc("Describe the specified user in the given workspace."). - Param(ws.PathParameter("workspace", "workspace name")). - Param(ws.PathParameter("member", "username")). - Metadata(restfulspec.KeyOpenAPITags, []string{constants.AccessManagementTag})) + c.Add(ws) return nil } diff --git a/pkg/kapis/oauth/handler.go b/pkg/kapis/oauth/handler.go new file mode 100644 index 000000000..e78260978 --- /dev/null +++ b/pkg/kapis/oauth/handler.go @@ -0,0 +1,121 @@ +/* + * + * Copyright 2020 The KubeSphere Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * / + */ + +package oauth + +import ( + "fmt" + "github.com/emicklei/go-restful" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/klog" + "kubesphere.io/kubesphere/pkg/api" + "kubesphere.io/kubesphere/pkg/api/auth" + "kubesphere.io/kubesphere/pkg/apiserver/authentication/oauth" + "kubesphere.io/kubesphere/pkg/apiserver/authentication/token" + "kubesphere.io/kubesphere/pkg/apiserver/request" + "net/http" +) + +type oauthHandler struct { + issuer token.Issuer + config oauth.Configuration +} + +func newOAUTHHandler(issuer token.Issuer, config oauth.Configuration) *oauthHandler { + return &oauthHandler{issuer: issuer, config: config} +} + +// Implement webhook authentication interface +// https://kubernetes.io/docs/reference/access-authn-authz/authentication/#webhook-token-authentication +func (h *oauthHandler) TokenReviewHandler(req *restful.Request, resp *restful.Response) { + var tokenReview auth.TokenReview + + err := req.ReadEntity(&tokenReview) + + if err != nil { + klog.Error(err) + api.HandleBadRequest(resp, req, err) + return + } + + if err = tokenReview.Validate(); err != nil { + klog.Error(err) + api.HandleBadRequest(resp, req, err) + return + } + + user, _, err := h.issuer.Verify(tokenReview.Spec.Token) + + if err != nil { + klog.Errorln(err) + api.HandleInternalError(resp, req, err) + return + } + + success := auth.TokenReview{APIVersion: tokenReview.APIVersion, + Kind: auth.KindTokenReview, + Status: &auth.Status{ + Authenticated: true, + User: map[string]interface{}{"username": user.GetName(), "uid": user.GetUID()}, + }, + } + + resp.WriteEntity(success) +} + +func (h *oauthHandler) AuthorizeHandler(req *restful.Request, resp *restful.Response) { + user, ok := request.UserFrom(req.Request.Context()) + clientId := req.QueryParameter("client_id") + responseType := req.QueryParameter("response_type") + + conf, err := h.config.Load(clientId) + + if err != nil { + err := apierrors.NewUnauthorized(fmt.Sprintf("Unauthorized: %s", err)) + resp.WriteError(http.StatusUnauthorized, err) + return + } + + if responseType != "token" { + err := apierrors.NewUnauthorized(fmt.Sprintf("Unauthorized: response type %s is not supported", responseType)) + resp.WriteError(http.StatusUnauthorized, err) + return + } + + if !ok { + err := apierrors.NewUnauthorized("Unauthorized") + resp.WriteError(http.StatusUnauthorized, err) + return + } + + accessToken, clm, err := h.issuer.IssueTo(user) + + if err != nil { + err := apierrors.NewUnauthorized(fmt.Sprintf("Unauthorized: %s", err)) + resp.WriteError(http.StatusUnauthorized, err) + return + } + + redirectURL := fmt.Sprintf("%s?access_token=%s&token_type=Bearer", conf.RedirectURL, accessToken) + expiresIn := clm.ExpiresAt - clm.IssuedAt + if expiresIn > 0 { + redirectURL = fmt.Sprintf("%s&expires_in=%v", redirectURL, expiresIn) + } + + http.Redirect(resp, req.Request, redirectURL, http.StatusFound) +} diff --git a/pkg/kapis/oauth/register.go b/pkg/kapis/oauth/register.go new file mode 100644 index 000000000..846cb8420 --- /dev/null +++ b/pkg/kapis/oauth/register.go @@ -0,0 +1,63 @@ +/* + * + * Copyright 2020 The KubeSphere Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * / + */ + +package oauth + +import ( + "github.com/emicklei/go-restful" + restfulspec "github.com/emicklei/go-restful-openapi" + "kubesphere.io/kubesphere/pkg/api" + "kubesphere.io/kubesphere/pkg/api/auth" + "kubesphere.io/kubesphere/pkg/apiserver/authentication/oauth" + "kubesphere.io/kubesphere/pkg/apiserver/authentication/token" + "kubesphere.io/kubesphere/pkg/constants" + "net/http" +) + +func AddToContainer(c *restful.Container, issuer token.Issuer, configuration oauth.Configuration) error { + ws := &restful.WebService{} + ws.Path("/oauth"). + Consumes(restful.MIME_JSON). + Produces(restful.MIME_JSON) + + handler := newOAUTHHandler(issuer, configuration) + + // Implement webhook authentication interface + // https://kubernetes.io/docs/reference/access-authn-authz/authentication/#webhook-token-authentication + ws.Route(ws.POST("/authenticate"). + Doc("TokenReview attempts to authenticate a token to a known user. Note: TokenReview requests may be cached by the webhook token authenticator plugin in the kube-apiserver."). + Reads(auth.TokenReview{}). + To(handler.TokenReviewHandler). + Returns(http.StatusOK, api.StatusOK, auth.TokenReview{}). + Metadata(restfulspec.KeyOpenAPITags, []string{constants.IdentityManagementTag})) + + // TODO Built-in oauth2 server (provider) + // web console use 'Resource Owner Password Credentials Grant' or 'Client Credentials Grant' request for an OAuth token + // https://tools.ietf.org/html/rfc6749#section-4.3 + // https://tools.ietf.org/html/rfc6749#section-4.4 + + // curl -u admin:P@88w0rd 'http://ks-apiserver.kubesphere-system.svc/oauth/authorize?client_id=kubesphere-console-client&response_type=token' -v + ws.Route(ws.GET("/authorize"). + To(handler.AuthorizeHandler)) + //ws.Route(ws.POST("/token")) + //ws.Route(ws.POST("/callback/{callback}")) + + c.Add(ws) + + return nil +} diff --git a/pkg/kapis/tenant/v1alpha2/handler.go b/pkg/kapis/tenant/v1alpha2/handler.go index 9008d78bc..ec4385e6b 100644 --- a/pkg/kapis/tenant/v1alpha2/handler.go +++ b/pkg/kapis/tenant/v1alpha2/handler.go @@ -3,14 +3,12 @@ package v1alpha2 import ( "github.com/emicklei/go-restful" v1 "k8s.io/api/core/v1" - rbacv1 "k8s.io/api/rbac/v1" k8serr "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/util/net" "k8s.io/klog" "kubesphere.io/kubesphere/pkg/api" "kubesphere.io/kubesphere/pkg/constants" "kubesphere.io/kubesphere/pkg/informers" - "kubesphere.io/kubesphere/pkg/models/iam" + "kubesphere.io/kubesphere/pkg/models/iam/am" "kubesphere.io/kubesphere/pkg/models/monitoring" "kubesphere.io/kubesphere/pkg/models/resources/v1alpha2" "kubesphere.io/kubesphere/pkg/models/tenant" @@ -18,39 +16,22 @@ import ( "kubesphere.io/kubesphere/pkg/server/params" "kubesphere.io/kubesphere/pkg/simple/client/k8s" "kubesphere.io/kubesphere/pkg/simple/client/mysql" - "kubesphere.io/kubesphere/pkg/utils/sliceutil" "net/http" - "strings" ) type tenantHandler struct { tenant tenant.Interface - am iam.AccessManagementInterface + am am.AccessManagementInterface } func newTenantHandler(k8sClient k8s.Client, factory informers.InformerFactory, db *mysql.Database) *tenantHandler { return &tenantHandler{ tenant: tenant.New(k8sClient.Kubernetes(), factory.KubernetesSharedInformerFactory(), factory.KubeSphereSharedInformerFactory(), db), - am: iam.NewAMOperator(k8sClient.Kubernetes(), factory.KubernetesSharedInformerFactory()), + am: am.NewAMOperator(k8sClient.Kubernetes(), factory.KubernetesSharedInformerFactory()), } } -func (h *tenantHandler) ListWorkspaceRules(req *restful.Request, resp *restful.Response) { - workspace := req.PathParameter("workspace") - username := req.HeaderParameter(constants.UserNameHeader) - - rules, err := h.tenant.GetWorkspaceSimpleRules(workspace, username) - - if err != nil { - klog.Errorln(err) - api.HandleInternalError(resp, nil, err) - return - } - - resp.WriteEntity(rules) -} - func (h *tenantHandler) ListWorkspaces(req *restful.Request, resp *restful.Response) { username := req.HeaderParameter(constants.UserNameHeader) orderBy := params.GetStringValueWithDefault(req, params.OrderByParam, v1alpha2.CreateTime) @@ -256,136 +237,3 @@ func (h *tenantHandler) DeleteDevopsProject(req *restful.Request, resp *restful. func (h *tenantHandler) CreateDevopsProject(req *restful.Request, resp *restful.Response) { } - -func (h *tenantHandler) ListNamespaceRules(req *restful.Request, resp *restful.Response) { - namespace := req.PathParameter("namespace") - username := req.HeaderParameter(constants.UserNameHeader) - - rules, err := h.tenant.GetNamespaceSimpleRules(namespace, username) - - if err != nil { - api.HandleInternalError(resp, nil, err) - return - } - - resp.WriteAsJson(rules) -} - -func (h *tenantHandler) ListDevopsRules(req *restful.Request, resp *restful.Response) { - - devops := req.PathParameter("devops") - username := req.HeaderParameter(constants.UserNameHeader) - - rules, err := h.tenant.GetUserDevopsSimpleRules(username, devops) - - if err != nil { - api.HandleInternalError(resp, nil, err) - return - } - - resp.WriteAsJson(rules) -} - -//TODO(wansir): We need move this part to logging module -//func (h *tenantHandler) LogQuery(req *restful.Request, resp *restful.Response) { -// operation := req.QueryParameter("operation") -// req, err := h.regenerateLoggingRequest(req) -// switch { -// case err != nil: -// api.HandleInternalError(resp, err) -// case req != nil: -// loggingv1alpha2.Get(req, loggingv1alpha2.LevelCluster, h.k8s, h.lo, resp) -// default: -// if operation == "export" { -// resp.Header().Set(restful.HEADER_ContentType, "text/plain") -// resp.Header().Set("Content-Disposition", "attachment") -// resp.Write(nil) -// } else { -// resp.WriteAsJson(v1alpha2.APIResponse{Logs: new(loggingclient.Logs)}) -// } -// } -//} - -// override namespace query conditions -//TODO(wansir): We need move this part to logging module -func (h *tenantHandler) regenerateLoggingRequest(req *restful.Request) (*restful.Request, error) { - - username := req.HeaderParameter(constants.UserNameHeader) - - // regenerate the request for log query - newUrl := net.FormatURL("http", "127.0.0.1", 80, "/kapis/logging.kubesphere.io/v1alpha2/cluster") - values := req.Request.URL.Query() - - clusterRoleRules, err := h.am.GetClusterPolicyRules(username) - - if err != nil { - klog.Errorln(err) - return nil, err - } - - hasClusterLogAccess := iam.RulesMatchesRequired(clusterRoleRules, rbacv1.PolicyRule{Verbs: []string{"get"}, Resources: []string{"*"}, APIGroups: []string{"logging.kubesphere.io"}}) - // if the user is not a cluster admin - if !hasClusterLogAccess { - queryNamespaces := strings.Split(req.QueryParameter("namespaces"), ",") - // then the user can only view logs of namespaces he belongs to - namespaces := make([]string, 0) - roles, err := h.am.GetRoles("", username) - if err != nil { - klog.Errorln(err) - return nil, err - } - for _, role := range roles { - if !sliceutil.HasString(namespaces, role.Namespace) && iam.RulesMatchesRequired(role.Rules, rbacv1.PolicyRule{Verbs: []string{"get"}, Resources: []string{"*"}, APIGroups: []string{"logging.kubesphere.io"}}) { - namespaces = append(namespaces, role.Namespace) - } - } - - // if the user belongs to no namespace - // then no log visible - if len(namespaces) == 0 { - return nil, nil - } else if len(queryNamespaces) == 1 && queryNamespaces[0] == "" { - values.Set("namespaces", strings.Join(namespaces, ",")) - } else { - inter := intersection(queryNamespaces, namespaces) - if len(inter) == 0 { - return nil, nil - } - values.Set("namespaces", strings.Join(inter, ",")) - } - } - - newUrl.RawQuery = values.Encode() - - // forward the request to logging model - newHttpRequest, _ := http.NewRequest(http.MethodGet, newUrl.String(), nil) - return restful.NewRequest(newHttpRequest), nil -} - -func intersection(s1, s2 []string) (inter []string) { - hash := make(map[string]bool) - for _, e := range s1 { - hash[e] = true - } - for _, e := range s2 { - // If elements present in the hashmap then append intersection list. - if hash[e] { - inter = append(inter, e) - } - } - //Remove dups from slice. - inter = removeDups(inter) - return -} - -//Remove dups from slice. -func removeDups(elements []string) (nodups []string) { - encountered := make(map[string]bool) - for _, element := range elements { - if !encountered[element] { - nodups = append(nodups, element) - encountered[element] = true - } - } - return -} diff --git a/pkg/kapis/tenant/v1alpha2/register.go b/pkg/kapis/tenant/v1alpha2/register.go index 782c582a6..94b65d4d6 100644 --- a/pkg/kapis/tenant/v1alpha2/register.go +++ b/pkg/kapis/tenant/v1alpha2/register.go @@ -29,7 +29,6 @@ import ( "kubesphere.io/kubesphere/pkg/constants" "kubesphere.io/kubesphere/pkg/informers" "kubesphere.io/kubesphere/pkg/models" - "kubesphere.io/kubesphere/pkg/models/iam/policy" "kubesphere.io/kubesphere/pkg/server/errors" "kubesphere.io/kubesphere/pkg/server/params" "kubesphere.io/kubesphere/pkg/simple/client/k8s" @@ -59,24 +58,6 @@ func AddToContainer(c *restful.Container, k8sClient k8s.Client, factory informer Param(ws.PathParameter("workspace", "workspace name")). Returns(http.StatusOK, api.StatusOK, v1alpha1.Workspace{}). Metadata(restfulspec.KeyOpenAPITags, []string{constants.TenantResourcesTag})) - ws.Route(ws.GET("/workspaces/{workspace}/rules"). - To(handler.ListWorkspaceRules). - Param(ws.PathParameter("workspace", "workspace name")). - Doc("List the rules of the specified workspace for the current user"). - Returns(http.StatusOK, api.StatusOK, policy.SimpleRule{}). - Metadata(restfulspec.KeyOpenAPITags, []string{constants.TenantResourcesTag})) - ws.Route(ws.GET("/namespaces/{namespace}/rules"). - To(handler.ListNamespaceRules). - Param(ws.PathParameter("namespace", "the name of the namespace")). - Doc("List the rules of the specified namespace for the current user"). - Returns(http.StatusOK, api.StatusOK, policy.SimpleRule{}). - Metadata(restfulspec.KeyOpenAPITags, []string{constants.TenantResourcesTag})) - ws.Route(ws.GET("/devops/{devops}/rules"). - To(handler.ListDevopsRules). - Param(ws.PathParameter("devops", "devops project ID")). - Doc("List the rules of the specified DevOps project for the current user"). - Returns(http.StatusOK, api.StatusOK, policy.SimpleRule{}). - Metadata(restfulspec.KeyOpenAPITags, []string{constants.TenantResourcesTag})) ws.Route(ws.GET("/workspaces/{workspace}/namespaces"). To(handler.ListNamespaces). Param(ws.PathParameter("workspace", "workspace name")). @@ -151,32 +132,6 @@ func AddToContainer(c *restful.Container, k8sClient k8s.Client, factory informer Doc("Delete the specified devops project from the workspace"). Returns(http.StatusOK, api.StatusOK, devopsv1alpha2.DevOpsProject{}). Metadata(restfulspec.KeyOpenAPITags, []string{constants.TenantResourcesTag})) - //ws.Route(ws.GET("/logs"). - // To(handler.LogQuery). - // Doc("Query cluster-level logs in a multi-tenants environment"). - // Param(ws.QueryParameter("operation", "Operation type. This can be one of four types: query (for querying logs), statistics (for retrieving statistical data), histogram (for displaying log count by time interval) and export (for exporting logs). Defaults to query.").DefaultValue("query").DataType("string").Required(false)). - // Param(ws.QueryParameter("workspaces", "A comma-separated list of workspaces. This field restricts the query to specified workspaces. For example, the following filter matches the workspace my-ws and demo-ws: `my-ws,demo-ws`").DataType("string").Required(false)). - // Param(ws.QueryParameter("workspace_query", "A comma-separated list of keywords. Differing from **workspaces**, this field performs fuzzy matching on workspaces. For example, the following value limits the query to workspaces whose name contains the word my(My,MY,...) *OR* demo(Demo,DemO,...): `my,demo`.").DataType("string").Required(false)). - // Param(ws.QueryParameter("namespaces", "A comma-separated list of namespaces. This field restricts the query to specified namespaces. For example, the following filter matches the namespace my-ns and demo-ns: `my-ns,demo-ns`").DataType("string").Required(false)). - // Param(ws.QueryParameter("namespace_query", "A comma-separated list of keywords. Differing from **namespaces**, this field performs fuzzy matching on namespaces. For example, the following value limits the query to namespaces whose name contains the word my(My,MY,...) *OR* demo(Demo,DemO,...): `my,demo`.").DataType("string").Required(false)). - // Param(ws.QueryParameter("workloads", "A comma-separated list of workloads. This field restricts the query to specified workloads. For example, the following filter matches the workload my-wl and demo-wl: `my-wl,demo-wl`").DataType("string").Required(false)). - // Param(ws.QueryParameter("workload_query", "A comma-separated list of keywords. Differing from **workloads**, this field performs fuzzy matching on workloads. For example, the following value limits the query to workloads whose name contains the word my(My,MY,...) *OR* demo(Demo,DemO,...): `my,demo`.").DataType("string").Required(false)). - // Param(ws.QueryParameter("pods", "A comma-separated list of pods. This field restricts the query to specified pods. For example, the following filter matches the pod my-po and demo-po: `my-po,demo-po`").DataType("string").Required(false)). - // Param(ws.QueryParameter("pod_query", "A comma-separated list of keywords. Differing from **pods**, this field performs fuzzy matching on pods. For example, the following value limits the query to pods whose name contains the word my(My,MY,...) *OR* demo(Demo,DemO,...): `my,demo`.").DataType("string").Required(false)). - // Param(ws.QueryParameter("containers", "A comma-separated list of containers. This field restricts the query to specified containers. For example, the following filter matches the container my-cont and demo-cont: `my-cont,demo-cont`").DataType("string").Required(false)). - // Param(ws.QueryParameter("container_query", "A comma-separated list of keywords. Differing from **containers**, this field performs fuzzy matching on containers. For example, the following value limits the query to containers whose name contains the word my(My,MY,...) *OR* demo(Demo,DemO,...): `my,demo`.").DataType("string").Required(false)). - // Param(ws.QueryParameter("log_query", "A comma-separated list of keywords. The query returns logs which contain at least one keyword. Case-insensitive matching. For example, if the field is set to `err,INFO`, the query returns any log containing err(ERR,Err,...) *OR* INFO(info,InFo,...).").DataType("string").Required(false)). - // Param(ws.QueryParameter("interval", "Time interval. It requires **operation** is set to histogram. The format is [0-9]+[smhdwMqy]. Defaults to 15m (i.e. 15 min).").DefaultValue("15m").DataType("string").Required(false)). - // Param(ws.QueryParameter("start_time", "Start time of query. Default to 0. The format is a string representing milliseconds since the epoch, eg. 1559664000000.").DataType("string").Required(false)). - // Param(ws.QueryParameter("end_time", "End time of query. Default to now. The format is a string representing milliseconds since the epoch, eg. 1559664000000.").DataType("string").Required(false)). - // Param(ws.QueryParameter("sort", "Sort order. One of acs, desc. This field sorts logs by timestamp.").DataType("string").DefaultValue("desc").Required(false)). - // Param(ws.QueryParameter("from", "The offset from the result set. This field returns query results from the specified offset. It requires **operation** is set to query. Defaults to 0 (i.e. from the beginning of the result set).").DataType("integer").DefaultValue("0").Required(false)). - // Param(ws.QueryParameter("size", "Size of result to return. It requires **operation** is set to query. Defaults to 10 (i.e. 10 log records).").DataType("integer").DefaultValue("10").Required(false)). - // Metadata(restfulspec.KeyOpenAPITags, []string{constants.TenantResourcesTag}). - // Writes(v1alpha2.Response{}). - // Returns(http.StatusOK, api.StatusOK, v1alpha2.Response{})). - // Consumes(restful.MIME_JSON, restful.MIME_XML). - // Produces(restful.MIME_JSON, "text/plain") c.Add(ws) return nil diff --git a/pkg/models/iam/am.go b/pkg/models/iam/am.go deleted file mode 100644 index a4c8251ea..000000000 --- a/pkg/models/iam/am.go +++ /dev/null @@ -1,606 +0,0 @@ -/* - - Copyright 2019 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - -*/ -package iam - -import ( - rbacv1 "k8s.io/api/rbac/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/labels" - "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/client-go/informers" - "k8s.io/client-go/kubernetes" - "k8s.io/klog" - "kubesphere.io/kubesphere/pkg/constants" - "kubesphere.io/kubesphere/pkg/models" - "kubesphere.io/kubesphere/pkg/models/iam/policy" - "kubesphere.io/kubesphere/pkg/models/resources/v1alpha2" - "kubesphere.io/kubesphere/pkg/models/resources/v1alpha2/clusterrole" - "kubesphere.io/kubesphere/pkg/models/resources/v1alpha2/resource" - "kubesphere.io/kubesphere/pkg/models/resources/v1alpha2/role" - "kubesphere.io/kubesphere/pkg/server/params" - "kubesphere.io/kubesphere/pkg/utils/k8sutil" -) - -const ( - ClusterRoleKind = "ClusterRole" - NamespaceAdminRoleBindName = "admin" - NamespaceViewerRoleBindName = "viewer" -) - -type AccessManagementInterface interface { - GetClusterRole(username string) (*rbacv1.ClusterRole, error) - UnBindAllRoles(username string) error - ListRoleBindings(namespace string, role string) ([]*rbacv1.RoleBinding, error) - CreateClusterRoleBinding(username string, clusterRole string) error - ListRoles(namespace string, conditions *params.Conditions, orderBy string, reverse bool, limit int, offset int) (*models.PageableResponse, error) - ListClusterRoles(conditions *params.Conditions, orderBy string, reverse bool, limit int, offset int) (*models.PageableResponse, error) - ListClusterRoleBindings(clusterRole string) ([]*rbacv1.ClusterRoleBinding, error) - GetClusterRoleSimpleRules(clusterRole string) ([]policy.SimpleRule, error) - GetRoleSimpleRules(namespace string, role string) ([]policy.SimpleRule, error) - GetRoles(namespace, username string) ([]*rbacv1.Role, error) - GetClusterPolicyRules(username string) ([]rbacv1.PolicyRule, error) - GetPolicyRules(namespace, username string) ([]rbacv1.PolicyRule, error) - GetWorkspaceRoleSimpleRules(workspace, roleName string) []policy.SimpleRule - GetWorkspaceRole(workspace, username string) (*rbacv1.ClusterRole, error) - GetWorkspaceRoleMap(username string) (map[string]string, error) -} - -type amOperator struct { - informers informers.SharedInformerFactory - resources resource.ResourceGetter - kubeClient kubernetes.Interface -} - -func (am *amOperator) ListClusterRoleBindings(clusterRole string) ([]*rbacv1.ClusterRoleBinding, error) { - panic("implement me") -} - -func (am *amOperator) GetRoles(namespace, username string) ([]*rbacv1.Role, error) { - panic("implement me") -} - -func (am *amOperator) GetClusterPolicyRules(username string) ([]rbacv1.PolicyRule, error) { - panic("implement me") -} - -func (am *amOperator) GetPolicyRules(namespace, username string) ([]rbacv1.PolicyRule, error) { - panic("implement me") -} - -func (am *amOperator) GetWorkspaceRole(workspace, username string) (*rbacv1.ClusterRole, error) { - panic("implement me") -} - -func (am *amOperator) UnBindAllRoles(username string) error { - panic("implement me") -} - -func NewAMOperator(kubeClient kubernetes.Interface, informers informers.SharedInformerFactory) *amOperator { - resourceGetter := resource.ResourceGetter{} - resourceGetter.Add(v1alpha2.Role, role.NewRoleSearcher(informers)) - resourceGetter.Add(v1alpha2.ClusterRoles, clusterrole.NewClusterRoleSearcher(informers)) - return &amOperator{ - informers: informers, - resources: resourceGetter, - kubeClient: kubeClient, - } -} - -func (am *amOperator) GetDevopsRoleSimpleRules(role string) []policy.SimpleRule { - var rules []policy.SimpleRule - - switch role { - case "developer": - rules = []policy.SimpleRule{ - {Name: "pipelines", Actions: []string{"view", "trigger"}}, - {Name: "roles", Actions: []string{"view"}}, - {Name: "members", Actions: []string{"view"}}, - {Name: "devops", Actions: []string{"view"}}, - } - break - case "owner": - rules = []policy.SimpleRule{ - {Name: "pipelines", Actions: []string{"create", "edit", "view", "delete", "trigger"}}, - {Name: "roles", Actions: []string{"view"}}, - {Name: "members", Actions: []string{"create", "edit", "view", "delete"}}, - {Name: "credentials", Actions: []string{"create", "edit", "view", "delete"}}, - {Name: "devops", Actions: []string{"edit", "view", "delete"}}, - } - break - case "maintainer": - rules = []policy.SimpleRule{ - {Name: "pipelines", Actions: []string{"create", "edit", "view", "delete", "trigger"}}, - {Name: "roles", Actions: []string{"view"}}, - {Name: "members", Actions: []string{"view"}}, - {Name: "credentials", Actions: []string{"create", "edit", "view", "delete"}}, - {Name: "devops", Actions: []string{"view"}}, - } - break - case "reporter": - fallthrough - default: - rules = []policy.SimpleRule{ - {Name: "pipelines", Actions: []string{"view"}}, - {Name: "roles", Actions: []string{"view"}}, - {Name: "members", Actions: []string{"view"}}, - {Name: "devops", Actions: []string{"view"}}, - } - break - } - return rules -} - -// Get user roles in namespace -func (am *amOperator) GetUserRoles(namespace, username string) ([]*rbacv1.Role, error) { - clusterRoleLister := am.informers.Rbac().V1().ClusterRoles().Lister() - roleBindingLister := am.informers.Rbac().V1().RoleBindings().Lister() - roleLister := am.informers.Rbac().V1().Roles().Lister() - roleBindings, err := roleBindingLister.RoleBindings(namespace).List(labels.Everything()) - - if err != nil { - klog.Errorln(err) - return nil, err - } - - roles := make([]*rbacv1.Role, 0) - - for _, roleBinding := range roleBindings { - if ContainsUser(roleBinding.Subjects, username) { - if roleBinding.RoleRef.Kind == ClusterRoleKind { - clusterRole, err := clusterRoleLister.Get(roleBinding.RoleRef.Name) - if err != nil { - if apierrors.IsNotFound(err) { - klog.Warningf("cluster role %s not found but bind user %s in namespace %s", roleBinding.RoleRef.Name, username, namespace) - continue - } else { - klog.Errorln(err) - return nil, err - } - } - role := rbacv1.Role{} - role.TypeMeta = clusterRole.TypeMeta - role.ObjectMeta = clusterRole.ObjectMeta - role.Rules = clusterRole.Rules - role.Namespace = roleBinding.Namespace - roles = append(roles, &role) - } else { - role, err := roleLister.Roles(roleBinding.Namespace).Get(roleBinding.RoleRef.Name) - - if err != nil { - if apierrors.IsNotFound(err) { - klog.Warningf("namespace %s role %s not found, but bind user %s", namespace, roleBinding.RoleRef.Name, username) - continue - } else { - klog.Errorln(err) - return nil, err - } - } - roles = append(roles, role) - } - } - } - - return roles, nil -} - -func (am *amOperator) GetUserClusterRoles(username string) (*rbacv1.ClusterRole, []*rbacv1.ClusterRole, error) { - clusterRoleLister := am.informers.Rbac().V1().ClusterRoles().Lister() - clusterRoleBindingLister := am.informers.Rbac().V1().ClusterRoleBindings().Lister() - clusterRoleBindings, err := clusterRoleBindingLister.List(labels.Everything()) - - if err != nil { - klog.Errorln(err) - return nil, nil, err - } - - clusterRoles := make([]*rbacv1.ClusterRole, 0) - userFacingClusterRole := &rbacv1.ClusterRole{} - for _, clusterRoleBinding := range clusterRoleBindings { - if ContainsUser(clusterRoleBinding.Subjects, username) { - clusterRole, err := clusterRoleLister.Get(clusterRoleBinding.RoleRef.Name) - if err != nil { - if apierrors.IsNotFound(err) { - klog.Warningf("cluster role %s not found but bind user %s", clusterRoleBinding.RoleRef.Name, username) - continue - } else { - klog.Errorln(err) - return nil, nil, err - } - } - if clusterRoleBinding.Name == username { - userFacingClusterRole = clusterRole - } - clusterRoles = append(clusterRoles, clusterRole) - } - } - - return userFacingClusterRole, clusterRoles, nil -} - -func (am *amOperator) GetClusterRole(username string) (*rbacv1.ClusterRole, error) { - userFacingClusterRole, _, err := am.GetUserClusterRoles(username) - if err != nil { - return nil, err - } - return userFacingClusterRole, nil -} - -func (am *amOperator) GetUserClusterRules(username string) ([]rbacv1.PolicyRule, error) { - _, clusterRoles, err := am.GetUserClusterRoles(username) - - if err != nil { - return nil, err - } - - rules := make([]rbacv1.PolicyRule, 0) - for _, clusterRole := range clusterRoles { - rules = append(rules, clusterRole.Rules...) - } - - return rules, nil -} - -func (am *amOperator) GetUserRules(namespace, username string) ([]rbacv1.PolicyRule, error) { - roles, err := am.GetUserRoles(namespace, username) - - if err != nil { - return nil, err - } - - rules := make([]rbacv1.PolicyRule, 0) - for _, role := range roles { - rules = append(rules, role.Rules...) - } - - return rules, nil -} - -func (am *amOperator) GetWorkspaceRoleBindings(workspace string) ([]*rbacv1.ClusterRoleBinding, error) { - - clusterRoleBindings, err := am.informers.Rbac().V1().ClusterRoleBindings().Lister().List(labels.Everything()) - - if err != nil { - klog.Errorln("get cluster role bindings", err) - return nil, err - } - - result := make([]*rbacv1.ClusterRoleBinding, 0) - - for _, roleBinding := range clusterRoleBindings { - if k8sutil.IsControlledBy(roleBinding.OwnerReferences, "Workspace", workspace) { - result = append(result, roleBinding) - } - } - - return result, nil -} - -//func (am *amOperator) GetWorkspaceRole(workspace, role string) (*rbacv1.ClusterRole, error) { -// if !sliceutil.HasString(constants.WorkSpaceRoles, role) { -// return nil, apierrors.NewNotFound(schema.GroupResource{Resource: "workspace role"}, role) -// } -// role = fmt.Sprintf("workspace:%s:%s", workspace, strings.TrimPrefix(role, "workspace-")) -// return am.informers.Rbac().V1().ClusterRoles().Lister().Get(role) -//} - -func (am *amOperator) GetWorkspaceRoleMap(username string) (map[string]string, error) { - - clusterRoleBindings, err := am.informers.Rbac().V1().ClusterRoleBindings().Lister().List(labels.Everything()) - - if err != nil { - klog.Errorln("get cluster role bindings", err) - return nil, err - } - - result := make(map[string]string, 0) - - for _, roleBinding := range clusterRoleBindings { - if workspace := k8sutil.GetControlledWorkspace(roleBinding.OwnerReferences); workspace != "" && - ContainsUser(roleBinding.Subjects, username) { - result[workspace] = roleBinding.RoleRef.Name - } - } - - return result, nil -} - -func (am *amOperator) GetUserWorkspaceRole(workspace, username string) (*rbacv1.ClusterRole, error) { - workspaceRoleMap, err := am.GetWorkspaceRoleMap(username) - - if err != nil { - return nil, err - } - - if workspaceRole := workspaceRoleMap[workspace]; workspaceRole != "" { - return am.informers.Rbac().V1().ClusterRoles().Lister().Get(workspaceRole) - } - - return nil, apierrors.NewNotFound(schema.GroupResource{Resource: "workspace user"}, username) -} - -func (am *amOperator) GetRoleBindings(namespace string, roleName string) ([]*rbacv1.RoleBinding, error) { - roleBindingLister := am.informers.Rbac().V1().RoleBindings().Lister() - roleBindings, err := roleBindingLister.RoleBindings(namespace).List(labels.Everything()) - - if err != nil { - klog.Errorln(err) - return nil, err - } - - items := make([]*rbacv1.RoleBinding, 0) - - for _, roleBinding := range roleBindings { - if roleName == "" { - items = append(items, roleBinding) - } else if roleBinding.RoleRef.Name == roleName { - items = append(items, roleBinding) - } - } - - return items, nil -} - -func (am *amOperator) GetClusterRoleBindings(clusterRoleName string) ([]*rbacv1.ClusterRoleBinding, error) { - clusterRoleBindingLister := am.informers.Rbac().V1().ClusterRoleBindings().Lister() - roleBindings, err := clusterRoleBindingLister.List(labels.Everything()) - - if err != nil { - klog.Errorln(err) - return nil, err - } - - items := make([]*rbacv1.ClusterRoleBinding, 0) - - for _, roleBinding := range roleBindings { - if roleBinding.RoleRef.Name == clusterRoleName { - items = append(items, roleBinding) - } - } - - return items, nil -} - -func (am *amOperator) ListRoles(namespace string, conditions *params.Conditions, orderBy string, reverse bool, limit, offset int) (*models.PageableResponse, error) { - return am.resources.ListResources(namespace, v1alpha2.Roles, conditions, orderBy, reverse, limit, offset) -} - -func (am *amOperator) ListRoleBindings(namespace string, role string) ([]*rbacv1.RoleBinding, error) { - rbs, err := am.informers.Rbac().V1().RoleBindings().Lister().RoleBindings(namespace).List(labels.Everything()) - if err != nil { - return nil, err - } - result := make([]*rbacv1.RoleBinding, 0) - for _, rb := range rbs { - if rb.RoleRef.Name == role { - result = append(result, rb.DeepCopy()) - } - } - return result, nil -} - -func (am *amOperator) ListWorkspaceRoles(workspace string, conditions *params.Conditions, orderBy string, reverse bool, limit, offset int) (*models.PageableResponse, error) { - conditions.Match[v1alpha2.OwnerName] = workspace - conditions.Match[v1alpha2.OwnerKind] = "Workspace" - result, err := am.resources.ListResources("", v1alpha2.ClusterRoles, conditions, orderBy, reverse, limit, offset) - - if err != nil { - return nil, err - } - - for i, item := range result.Items { - if role, ok := item.(*rbacv1.ClusterRole); ok { - role = role.DeepCopy() - role.Name = role.Annotations[constants.DisplayNameAnnotationKey] - result.Items[i] = role - } - } - return result, nil -} - -func (am *amOperator) ListClusterRoles(conditions *params.Conditions, orderBy string, reverse bool, limit, offset int) (*models.PageableResponse, error) { - return am.resources.ListResources("", v1alpha2.ClusterRoles, conditions, orderBy, reverse, limit, offset) -} - -func (am *amOperator) GetWorkspaceRoleSimpleRules(workspace, roleName string) []policy.SimpleRule { - - workspaceRules := make([]policy.SimpleRule, 0) - - switch roleName { - case constants.WorkspaceAdmin: - workspaceRules = []policy.SimpleRule{ - {Name: "workspaces", Actions: []string{"edit", "delete", "view"}}, - {Name: "members", Actions: []string{"edit", "delete", "create", "view"}}, - {Name: "devops", Actions: []string{"edit", "delete", "create", "view"}}, - {Name: "projects", Actions: []string{"edit", "delete", "create", "view"}}, - {Name: "roles", Actions: []string{"view"}}, - {Name: "apps", Actions: []string{"view", "create", "manage"}}, - {Name: "repos", Actions: []string{"view", "manage"}}, - } - case constants.WorkspaceRegular: - workspaceRules = []policy.SimpleRule{ - {Name: "members", Actions: []string{"view"}}, - {Name: "devops", Actions: []string{"view", "create"}}, - {Name: "projects", Actions: []string{"view", "create"}}, - {Name: "apps", Actions: []string{"view", "create"}}, - {Name: "repos", Actions: []string{"view"}}, - } - case constants.WorkspaceViewer: - workspaceRules = []policy.SimpleRule{ - {Name: "workspaces", Actions: []string{"view"}}, - {Name: "members", Actions: []string{"view"}}, - {Name: "devops", Actions: []string{"view"}}, - {Name: "projects", Actions: []string{"view"}}, - {Name: "roles", Actions: []string{"view"}}, - {Name: "apps", Actions: []string{"view"}}, - {Name: "repos", Actions: []string{"view"}}, - } - case constants.WorkspacesManager: - workspaceRules = []policy.SimpleRule{ - {Name: "workspaces", Actions: []string{"edit", "delete", "view"}}, - {Name: "members", Actions: []string{"edit", "delete", "create", "view"}}, - {Name: "roles", Actions: []string{"view"}}, - } - } - - return workspaceRules -} - -// Convert cluster role to rules -func (am *amOperator) GetClusterRoleSimpleRules(clusterRoleName string) ([]policy.SimpleRule, error) { - - clusterRoleLister := am.informers.Rbac().V1().ClusterRoles().Lister() - clusterRole, err := clusterRoleLister.Get(clusterRoleName) - - if err != nil { - klog.Errorln(err) - return nil, err - } - - return getClusterSimpleRule(clusterRole.Rules), nil -} - -func (am *amOperator) GetUserClusterSimpleRules(username string) ([]policy.SimpleRule, error) { - clusterRules, err := am.GetUserClusterRules(username) - if err != nil { - return nil, err - } - return getClusterSimpleRule(clusterRules), nil -} - -// Convert roles to rules -func (am *amOperator) GetRoleSimpleRules(namespace string, roleName string) ([]policy.SimpleRule, error) { - - roleLister := am.informers.Rbac().V1().Roles().Lister() - role, err := roleLister.Roles(namespace).Get(roleName) - - if err != nil { - klog.Errorln(err) - return nil, err - } - - return ConvertToSimpleRule(role.Rules), nil -} - -func getClusterSimpleRule(policyRules []rbacv1.PolicyRule) []policy.SimpleRule { - rules := make([]policy.SimpleRule, 0) - - for i := 0; i < len(policy.ClusterRoleRuleMapping); i++ { - validActions := make([]string, 0) - for j := 0; j < (len(policy.ClusterRoleRuleMapping[i].Actions)); j++ { - if rulesMatchesAction(policyRules, policy.ClusterRoleRuleMapping[i].Actions[j]) { - validActions = append(validActions, policy.ClusterRoleRuleMapping[i].Actions[j].Name) - } - } - if len(validActions) > 0 { - rules = append(rules, policy.SimpleRule{Name: policy.ClusterRoleRuleMapping[i].Name, Actions: validActions}) - } - } - - return rules -} - -func ConvertToSimpleRule(policyRules []rbacv1.PolicyRule) []policy.SimpleRule { - simpleRules := make([]policy.SimpleRule, 0) - for i := 0; i < len(policy.RoleRuleMapping); i++ { - rule := policy.SimpleRule{Name: policy.RoleRuleMapping[i].Name} - rule.Actions = make([]string, 0) - for j := 0; j < len(policy.RoleRuleMapping[i].Actions); j++ { - if rulesMatchesAction(policyRules, policy.RoleRuleMapping[i].Actions[j]) { - rule.Actions = append(rule.Actions, policy.RoleRuleMapping[i].Actions[j].Name) - } - } - if len(rule.Actions) > 0 { - simpleRules = append(simpleRules, rule) - } - } - return simpleRules -} - -func (am *amOperator) CreateClusterRoleBinding(username string, clusterRoleName string) error { - clusterRoleLister := am.informers.Rbac().V1().ClusterRoles().Lister() - - _, err := clusterRoleLister.Get(clusterRoleName) - - if err != nil { - klog.Errorln(err) - return err - } - - // TODO move to user controller - if clusterRoleName == constants.ClusterAdmin { - // create kubectl pod if cluster role is cluster-admin - //if err := kubectl.CreateKubectlDeploy(username); err != nil { - // klog.Error("create user terminal pod failed", username, err) - //} - - } else { - // delete kubectl pod if cluster role is not cluster-admin, whether it exists or not - //if err := kubectl.DelKubectlDeploy(username); err != nil { - // klog.Error("delete user terminal pod failed", username, err) - //} - } - - clusterRoleBinding := &rbacv1.ClusterRoleBinding{} - clusterRoleBinding.Name = username - clusterRoleBinding.RoleRef = rbacv1.RoleRef{Name: clusterRoleName, Kind: ClusterRoleKind} - clusterRoleBinding.Subjects = []rbacv1.Subject{{Kind: rbacv1.UserKind, Name: username}} - - clusterRoleBindingLister := am.informers.Rbac().V1().ClusterRoleBindings().Lister() - found, err := clusterRoleBindingLister.Get(username) - - if apierrors.IsNotFound(err) { - _, err = am.kubeClient.RbacV1().ClusterRoleBindings().Create(clusterRoleBinding) - if err != nil { - klog.Errorln("create cluster role binding", err) - return err - } - return nil - } else if err != nil { - return err - } - - // cluster role changed - if found.RoleRef.Name != clusterRoleName { - deletePolicy := metav1.DeletePropagationBackground - gracePeriodSeconds := int64(0) - deleteOption := &metav1.DeleteOptions{PropagationPolicy: &deletePolicy, GracePeriodSeconds: &gracePeriodSeconds} - err = am.kubeClient.RbacV1().ClusterRoleBindings().Delete(found.Name, deleteOption) - if err != nil { - klog.Errorln(err) - return err - } - _, err = am.kubeClient.RbacV1().ClusterRoleBindings().Create(clusterRoleBinding) - if err != nil { - klog.Errorln(err) - return err - } - return nil - } - - if !ContainsUser(found.Subjects, username) { - found.Subjects = clusterRoleBinding.Subjects - _, err = am.kubeClient.RbacV1().ClusterRoleBindings().Update(found) - if err != nil { - klog.Errorln("update cluster role binding", err) - return err - } - } - - return nil -} diff --git a/pkg/models/iam/am/am.go b/pkg/models/iam/am/am.go new file mode 100644 index 000000000..9a2da7f84 --- /dev/null +++ b/pkg/models/iam/am/am.go @@ -0,0 +1,78 @@ +/* + * + * Copyright 2020 The KubeSphere Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * / + */ +package am + +import ( + "k8s.io/client-go/informers" + "k8s.io/client-go/kubernetes" + "kubesphere.io/kubesphere/pkg/models/resources/v1alpha2" + "kubesphere.io/kubesphere/pkg/models/resources/v1alpha2/clusterrole" + "kubesphere.io/kubesphere/pkg/models/resources/v1alpha2/resource" + "kubesphere.io/kubesphere/pkg/models/resources/v1alpha2/role" +) + +const ( + ClusterRoleKind = "ClusterRole" + NamespaceAdminRoleBindName = "admin" + NamespaceViewerRoleBindName = "viewer" +) + +type AccessManagementInterface interface { + GetPlatformRole(username string) (Role, error) + GetClusterRole(cluster, username string) (Role, error) + GetWorkspaceRole(workspace, username string) (Role, error) + GetNamespaceRole(cluster, namespace, username string) (Role, error) +} + +type Role interface { + GetName() string + GetRego() string +} + +type amOperator struct { + informers informers.SharedInformerFactory + resources resource.ResourceGetter + kubeClient kubernetes.Interface +} + +func NewAMOperator(kubeClient kubernetes.Interface, informers informers.SharedInformerFactory) AccessManagementInterface { + resourceGetter := resource.ResourceGetter{} + resourceGetter.Add(v1alpha2.Role, role.NewRoleSearcher(informers)) + resourceGetter.Add(v1alpha2.ClusterRoles, clusterrole.NewClusterRoleSearcher(informers)) + return &amOperator{ + informers: informers, + resources: resourceGetter, + kubeClient: kubeClient, + } +} + +func (am *amOperator) GetPlatformRole(username string) (Role, error) { + panic("implement me") +} + +func (am *amOperator) GetClusterRole(cluster, username string) (Role, error) { + panic("implement me") +} + +func (am *amOperator) GetWorkspaceRole(workspace, username string) (Role, error) { + panic("implement me") +} + +func (am *amOperator) GetNamespaceRole(cluster, namespace, username string) (Role, error) { + panic("implement me") +} diff --git a/pkg/models/iam/am/fake_operator.go b/pkg/models/iam/am/fake_operator.go new file mode 100644 index 000000000..13a5c5253 --- /dev/null +++ b/pkg/models/iam/am/fake_operator.go @@ -0,0 +1,139 @@ +/* + * + * Copyright 2020 The KubeSphere Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * / + */ + +package am + +import ( + "encoding/json" + "fmt" + "k8s.io/apiserver/pkg/authentication/user" + "kubesphere.io/kubesphere/pkg/simple/client/cache" +) + +type FakeRole struct { + Name string + Rego string +} +type FakeOperator struct { + cache cache.Interface +} + +func (f FakeOperator) queryFakeRole(cacheKey string) (Role, error) { + data, err := f.cache.Get(cacheKey) + if err != nil { + if err == cache.ErrNoSuchKey { + return &FakeRole{ + Name: "DenyAll", + Rego: "package authz\ndefault allow = false", + }, nil + } + return nil, err + } + var role FakeRole + err = json.Unmarshal([]byte(data), &role) + if err != nil { + return nil, err + } + return role, nil +} + +func (f FakeOperator) saveFakeRole(cacheKey string, role FakeRole) error { + data, err := json.Marshal(role) + if err != nil { + return err + } + return f.cache.Set(cacheKey, string(data), 0) +} + +func (f FakeOperator) GetPlatformRole(username string) (Role, error) { + return f.queryFakeRole(platformRoleCacheKey(username)) +} + +func (f FakeOperator) GetClusterRole(cluster, username string) (Role, error) { + return f.queryFakeRole(clusterRoleCacheKey(cluster, username)) +} + +func (f FakeOperator) GetWorkspaceRole(workspace, username string) (Role, error) { + return f.queryFakeRole(workspaceRoleCacheKey(workspace, username)) +} + +func (f FakeOperator) GetNamespaceRole(cluster, namespace, username string) (Role, error) { + return f.queryFakeRole(namespaceRoleCacheKey(cluster, namespace, username)) +} + +func (f FakeOperator) Prepare(platformRoles map[string]FakeRole, clusterRoles map[string]map[string]FakeRole, workspaceRoles map[string]map[string]FakeRole, namespaceRoles map[string]map[string]map[string]FakeRole) { + + for username, role := range platformRoles { + f.saveFakeRole(platformRoleCacheKey(username), role) + } + for cluster, roles := range clusterRoles { + for username, role := range roles { + f.saveFakeRole(clusterRoleCacheKey(cluster, username), role) + } + } + + for workspace, roles := range workspaceRoles { + for username, role := range roles { + f.saveFakeRole(workspaceRoleCacheKey(workspace, username), role) + } + } + + for cluster, nsRoles := range namespaceRoles { + for namespace, roles := range nsRoles { + for username, role := range roles { + f.saveFakeRole(namespaceRoleCacheKey(cluster, namespace, username), role) + } + } + } +} + +func namespaceRoleCacheKey(cluster, namespace, username string) string { + return fmt.Sprintf("cluster.%s.namespaces.%s.roles.%s", cluster, namespace, username) +} + +func clusterRoleCacheKey(cluster, username string) string { + return fmt.Sprintf("cluster.%s.roles.%s", cluster, username) +} +func workspaceRoleCacheKey(workspace, username string) string { + return fmt.Sprintf("workspace.%s.roles.%s", workspace, username) +} + +func platformRoleCacheKey(username string) string { + return fmt.Sprintf("platform.roles.%s", username) +} + +func (f FakeRole) GetName() string { + return f.Name +} + +func (f FakeRole) GetRego() string { + return f.Rego +} + +func NewFakeAMOperator() *FakeOperator { + operator := &FakeOperator{cache: cache.NewSimpleCache()} + operator.saveFakeRole(platformRoleCacheKey("admin"), FakeRole{ + Name: "admin", + Rego: "package authz\ndefault allow = true", + }) + operator.saveFakeRole(platformRoleCacheKey(user.Anonymous), FakeRole{ + Name: "admin", + Rego: "package authz\ndefault allow = false", + }) + return operator +} diff --git a/pkg/models/iam/im.go b/pkg/models/iam/im.go deleted file mode 100644 index 1f12b9b60..000000000 --- a/pkg/models/iam/im.go +++ /dev/null @@ -1,220 +0,0 @@ -/* - * - * Copyright 2020 The KubeSphere Authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * / - */ -package iam - -import ( - "fmt" - "github.com/pkg/errors" - "golang.org/x/oauth2" - rbacv1 "k8s.io/api/rbac/v1" - "k8s.io/klog" - "kubesphere.io/kubesphere/pkg/api/iam" - "kubesphere.io/kubesphere/pkg/api/iam/token" - "kubesphere.io/kubesphere/pkg/models" - "kubesphere.io/kubesphere/pkg/server/params" - "kubesphere.io/kubesphere/pkg/simple/client/cache" - "kubesphere.io/kubesphere/pkg/simple/client/ldap" - "time" -) - -type IdentityManagementInterface interface { - CreateUser(user *iam.User) (*iam.User, error) - DeleteUser(username string) error - DescribeUser(username string) (*iam.User, error) - Login(username, password, ip string) (*oauth2.Token, error) - ModifyUser(user *iam.User) (*iam.User, error) - ListUsers(conditions *params.Conditions, orderBy string, reverse bool, limit, offset int) (*models.PageableResponse, error) - GetUserRoles(username string) ([]*rbacv1.Role, error) - GetUserRole(namespace string, username string) (*rbacv1.Role, error) - VerifyToken(token string) (*iam.User, error) -} - -type imOperator struct { - authenticateOptions *iam.AuthenticationOptions - ldapClient ldap.Interface - cacheClient cache.Interface - issuer token.Issuer -} - -var ( - AuthRateLimitExceeded = errors.New("user auth rate limit exceeded") - UserAlreadyExists = errors.New("user already exists") - UserNotExists = errors.New("user not exists") -) - -func NewIMOperator(ldapClient ldap.Interface, cacheClient cache.Interface, options *iam.AuthenticationOptions) *imOperator { - return &imOperator{ - ldapClient: ldapClient, - cacheClient: cacheClient, - authenticateOptions: options, - issuer: token.NewJwtTokenIssuer(token.DefaultIssuerName, []byte(options.JwtSecret)), - } - -} - -func (im *imOperator) ModifyUser(user *iam.User) (*iam.User, error) { - err := im.ldapClient.Update(user) - if err != nil { - return nil, err - } - - // clear auth failed record - if user.Password != "" { - records, err := im.cacheClient.Keys(authenticationFailedKeyForUsername(user.Username, "*")) - if err == nil { - im.cacheClient.Del(records...) - } - } - - return im.ldapClient.Get(user.Username) -} - -func (im *imOperator) Login(username, password, ip string) (*oauth2.Token, error) { - - records, err := im.cacheClient.Keys(authenticationFailedKeyForUsername(username, "*")) - if err != nil { - return nil, err - } - - if len(records) > im.authenticateOptions.MaxAuthenticateRetries { - return nil, AuthRateLimitExceeded - } - - user, err := im.ldapClient.Get(username) - if err != nil { - return nil, err - } - - err = im.ldapClient.Verify(user.Username, password) - if err != nil { - if err == ldap.ErrInvalidCredentials { - im.cacheClient.Set(authenticationFailedKeyForUsername(username, fmt.Sprintf("%d", time.Now().UnixNano())), "", 30*time.Minute) - } - return nil, err - } - - issuedToken, err := im.issuer.IssueTo(user) - if err != nil { - return nil, err - } - - // TODO: I think we should come up with a better strategy to prevent multiple login. - tokenKey := tokenKeyForUsername(user.Username, issuedToken) - if !im.authenticateOptions.MultipleLogin { - // multi login not allowed, remove the previous token - sessions, err := im.cacheClient.Keys(tokenKey) - if err != nil { - return nil, err - } - - if len(sessions) > 0 { - klog.V(4).Infoln("revoke token", sessions) - err = im.cacheClient.Del(sessions...) - if err != nil { - return nil, err - } - } - } - - // save token with expiration time - if err = im.cacheClient.Set(tokenKey, issuedToken, im.authenticateOptions.TokenExpiration); err != nil { - return nil, err - } - - im.logLogin(user.Username, ip, time.Now()) - - return &oauth2.Token{AccessToken: issuedToken}, nil -} - -func (im *imOperator) logLogin(username, ip string, loginTime time.Time) { - if ip != "" { - _ = im.cacheClient.Set(loginKeyForUsername(username, loginTime.UTC().Format("2006-01-02T15:04:05Z"), ip), "", 30*24*time.Hour) - } -} - -func (im *imOperator) LoginHistory(username string) ([]string, error) { - keys, err := im.cacheClient.Keys(loginKeyForUsername(username, "*", "*")) - if err != nil { - return nil, err - } - - return keys, nil -} - -func (im *imOperator) ListUsers(conditions *params.Conditions, orderBy string, reverse bool, limit, offset int) (*models.PageableResponse, error) { - panic("implement me") -} - -func (im *imOperator) DescribeUser(username string) (*iam.User, error) { - return im.ldapClient.Get(username) -} - -func (im *imOperator) getLastLoginTime(username string) string { - return "" -} - -func (im *imOperator) DeleteUser(username string) error { - return im.ldapClient.Delete(username) -} - -func (im *imOperator) CreateUser(user *iam.User) (*iam.User, error) { - err := im.ldapClient.Create(user) - if err != nil { - return nil, err - } - - return user, nil -} - -func (im *imOperator) VerifyToken(tokenString string) (*iam.User, error) { - providedUser, err := im.issuer.Verify(tokenString) - if err != nil { - return nil, err - } - - user, err := im.ldapClient.Get(providedUser.Name()) - if err != nil { - return nil, err - } - - return user, nil -} - -func (im *imOperator) uidNumberNext() int { - // TODO fix me - return 0 -} -func (im *imOperator) GetUserRoles(username string) ([]*rbacv1.Role, error) { - panic("implement me") -} - -func (im *imOperator) GetUserRole(namespace string, username string) (*rbacv1.Role, error) { - panic("implement me") -} - -func authenticationFailedKeyForUsername(username, failedTimestamp string) string { - return fmt.Sprintf("kubesphere:authfailed:%s:%s", username, failedTimestamp) -} - -func tokenKeyForUsername(username, token string) string { - return fmt.Sprintf("kubesphere:users:%s:token:%s", username, token) -} - -func loginKeyForUsername(username, loginTimestamp, ip string) string { - return fmt.Sprintf("kubesphere:users:%s:login-log:%s:%s", username, loginTimestamp, ip) -} diff --git a/pkg/models/iam/im/fake_operator.go b/pkg/models/iam/im/fake_operator.go new file mode 100644 index 000000000..e6186e044 --- /dev/null +++ b/pkg/models/iam/im/fake_operator.go @@ -0,0 +1,25 @@ +/* + * + * Copyright 2020 The KubeSphere Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * / + */ + +package im + +import "kubesphere.io/kubesphere/pkg/simple/client/ldap" + +func NewFakeOperator() IdentityManagementInterface { + return NewLDAPOperator(ldap.NewSimpleLdap()) +} diff --git a/pkg/models/iam/im/im.go b/pkg/models/iam/im/im.go new file mode 100644 index 000000000..e9061c67e --- /dev/null +++ b/pkg/models/iam/im/im.go @@ -0,0 +1,94 @@ +/* + * + * Copyright 2020 The KubeSphere Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * / + */ +package im + +import ( + "github.com/pkg/errors" + "kubesphere.io/kubesphere/pkg/api/iam" + "kubesphere.io/kubesphere/pkg/simple/client/ldap" +) + +type IdentityManagementInterface interface { + CreateUser(user *iam.User) (*iam.User, error) + DeleteUser(username string) error + ModifyUser(user *iam.User) (*iam.User, error) + DescribeUser(username string) (*iam.User, error) + Authenticate(username, password string) (*iam.User, error) +} + +type imOperator struct { + ldapClient ldap.Interface +} + +var ( + AuthRateLimitExceeded = errors.New("user auth rate limit exceeded") + UserAlreadyExists = errors.New("user already exists") + UserNotExists = errors.New("user not exists") +) + +func NewLDAPOperator(ldapClient ldap.Interface) IdentityManagementInterface { + return &imOperator{ + ldapClient: ldapClient, + } + +} + +func (im *imOperator) ModifyUser(user *iam.User) (*iam.User, error) { + + err := im.ldapClient.Update(user) + + if err != nil { + return nil, err + } + + return im.ldapClient.Get(user.Name) +} + +func (im *imOperator) Authenticate(username, password string) (*iam.User, error) { + + user, err := im.ldapClient.Get(username) + + if err != nil { + return nil, err + } + + err = im.ldapClient.Authenticate(user.Name, password) + if err != nil { + return nil, err + } + + return user, nil +} + +func (im *imOperator) DescribeUser(username string) (*iam.User, error) { + return im.ldapClient.Get(username) +} + +func (im *imOperator) DeleteUser(username string) error { + return im.ldapClient.Delete(username) +} + +func (im *imOperator) CreateUser(user *iam.User) (*iam.User, error) { + err := im.ldapClient.Create(user) + + if err != nil { + return nil, err + } + + return user, nil +} diff --git a/pkg/models/iam/im_test.go b/pkg/models/iam/im/im_test.go similarity index 98% rename from pkg/models/iam/im_test.go rename to pkg/models/iam/im/im_test.go index eb0d94c0b..2e3a3eccc 100644 --- a/pkg/models/iam/im_test.go +++ b/pkg/models/iam/im/im_test.go @@ -16,4 +16,4 @@ * / */ -package iam +package im diff --git a/pkg/models/iam/policy/policy.go b/pkg/models/iam/policy/policy.go deleted file mode 100644 index 59dbe9c30..000000000 --- a/pkg/models/iam/policy/policy.go +++ /dev/null @@ -1,1101 +0,0 @@ -/* - - Copyright 2019 The KubeSphere Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - -*/ - -package policy - -import ( - "encoding/json" - "io/ioutil" - "k8s.io/api/rbac/v1" -) - -const ( - configPath = "/etc/kubesphere/iam" - rulesConfigPath = configPath + "/rules.json" - clusterRulesConfigPath = configPath + "/clusterrules.json" -) - -func init() { - rulesConfig, err := ioutil.ReadFile(rulesConfigPath) - - if err == nil { - config := &[]Rule{} - json.Unmarshal(rulesConfig, config) - if len(*config) > 0 { - RoleRuleMapping = *config - } - } - - clusterRulesConfig, err := ioutil.ReadFile(clusterRulesConfigPath) - - if err == nil { - config := &[]Rule{} - json.Unmarshal(clusterRulesConfig, config) - if len(*config) > 0 { - ClusterRoleRuleMapping = *config - } - } -} - -var ( - ClusterRoleRuleMapping = []Rule{ - {Name: "workspaces", - Actions: []Action{ - { - Name: "manage", - Rules: []v1.PolicyRule{ - { - Verbs: []string{"*"}, - APIGroups: []string{"*"}, - Resources: []string{"workspaces", "workspaces/*"}, - }, - }, - }, - }, - }, - { - Name: "monitoring", - Actions: []Action{ - {Name: "view", - Rules: []v1.PolicyRule{{ - Verbs: []string{"get", "list"}, - APIGroups: []string{"monitoring.kubesphere.io"}, - Resources: []string{"*"}, - }, { - Verbs: []string{"get", "list"}, - APIGroups: []string{"resources.kubesphere.io"}, - Resources: []string{"health"}, - }}, - }, - }, - }, - { - Name: "alerting", - Actions: []Action{ - {Name: "view", - Rules: []v1.PolicyRule{{ - Verbs: []string{"get", "list"}, - APIGroups: []string{"alerting.kubesphere.io"}, - Resources: []string{"*"}, - }}, - }, - {Name: "create", - Rules: []v1.PolicyRule{{ - Verbs: []string{"create"}, - APIGroups: []string{"alerting.kubesphere.io"}, - Resources: []string{"*"}, - }}, - }, - {Name: "delete", - Rules: []v1.PolicyRule{{ - Verbs: []string{"delete"}, - APIGroups: []string{"alerting.kubesphere.io"}, - Resources: []string{"*"}, - }}, - }, - }, - }, - { - Name: "logging", - Actions: []Action{ - {Name: "view", - Rules: []v1.PolicyRule{{ - Verbs: []string{"get", "list"}, - APIGroups: []string{"logging.kubesphere.io"}, - Resources: []string{"*"}, - }}, - }, - }, - }, - { - Name: "accounts", - Actions: []Action{ - {Name: "view", - Rules: []v1.PolicyRule{ - { - Verbs: []string{"get", "watch", "list"}, - APIGroups: []string{"iam.kubesphere.io"}, - Resources: []string{"users", "users/*"}, - }, - { - Verbs: []string{"get"}, - APIGroups: []string{"iam.kubesphere.io"}, - Resources: []string{"rulesmapping"}, - ResourceNames: []string{"clusterroles"}, - }, - { - Verbs: []string{"get", "watch", "list"}, - APIGroups: []string{"rbac.authorization.k8s.io"}, - Resources: []string{"clusterrolebindings"}, - }, - }, - }, - {Name: "create", - Rules: []v1.PolicyRule{ - { - Verbs: []string{"create", "get", "list"}, - APIGroups: []string{"iam.kubesphere.io"}, - Resources: []string{"users"}, - }, - { - Verbs: []string{"get"}, - APIGroups: []string{"iam.kubesphere.io"}, - Resources: []string{"clusterrules"}, - ResourceNames: []string{"mapping"}, - }, - { - Verbs: []string{"create", "delete", "deletecollection"}, - APIGroups: []string{"rbac.authorization.k8s.io"}, - Resources: []string{"clusterrolebindings"}, - }, - }, - }, - {Name: "edit", - Rules: []v1.PolicyRule{ - { - Verbs: []string{"get", "list", "update", "patch"}, - APIGroups: []string{"iam.kubesphere.io"}, - Resources: []string{"users"}, - }, - { - Verbs: []string{"create", "delete", "deletecollection"}, - APIGroups: []string{"rbac.authorization.k8s.io"}, - Resources: []string{"clusterrolebindings"}, - }, - }, - }, - {Name: "delete", - Rules: []v1.PolicyRule{ - { - Verbs: []string{"delete", "deletecollection"}, - APIGroups: []string{"iam.kubesphere.io"}, - Resources: []string{"users"}, - }, - }, - }, - }, - }, { - Name: "roles", - Actions: []Action{ - {Name: "view", - Rules: []v1.PolicyRule{ - { - Verbs: []string{"get", "watch", "list"}, - APIGroups: []string{"rbac.authorization.k8s.io"}, - Resources: []string{"clusterroles"}, - }, - { - Verbs: []string{"get", "list"}, - APIGroups: []string{"iam.kubesphere.io"}, - Resources: []string{"clusterroles", "clusterroles/*"}, - }, - }, - }, - - {Name: "create", - Rules: []v1.PolicyRule{ - { - Verbs: []string{"create"}, - APIGroups: []string{"rbac.authorization.k8s.io"}, - Resources: []string{"clusterroles"}, - }, - }, - }, - {Name: "edit", - Rules: []v1.PolicyRule{ - { - Verbs: []string{"update", "patch"}, - APIGroups: []string{"rbac.authorization.k8s.io"}, - Resources: []string{"clusterroles"}, - }, - }, - }, - {Name: "delete", - Rules: []v1.PolicyRule{ - { - Verbs: []string{"delete", "deletecollection"}, - APIGroups: []string{"rbac.authorization.k8s.io"}, - Resources: []string{"clusterroles"}, - }, - }, - }, - }, - }, { - Name: "storageclasses", - Actions: []Action{ - {Name: "view", - Rules: []v1.PolicyRule{ - { - Verbs: []string{"get", "watch", "list"}, - APIGroups: []string{"storage.k8s.io"}, - Resources: []string{"storageclasses"}, - }, { - Verbs: []string{"get", "list"}, - APIGroups: []string{"resources.kubesphere.io"}, - Resources: []string{"storageclasses", "storageclasses/*"}, - }, - }, - }, - {Name: "create", - Rules: []v1.PolicyRule{ - { - Verbs: []string{"create"}, - APIGroups: []string{"storage.k8s.io"}, - Resources: []string{"storageclasses"}, - }, - }, - }, - {Name: "edit", - Rules: []v1.PolicyRule{ - { - Verbs: []string{"update", "patch"}, - APIGroups: []string{"storage.k8s.io"}, - Resources: []string{"storageclasses"}, - }, - }, - }, - {Name: "delete", - Rules: []v1.PolicyRule{ - { - Verbs: []string{"delete", "deletecollection"}, - APIGroups: []string{"storage.k8s.io"}, - Resources: []string{"storageclasses"}, - }, - }, - }, - }, - }, { - Name: "nodes", - Actions: []Action{ - {Name: "view", - Rules: []v1.PolicyRule{ - { - Verbs: []string{"get", "watch", "list"}, - APIGroups: []string{""}, - Resources: []string{"nodes", "events"}, - }, - { - Verbs: []string{"get", "list"}, - APIGroups: []string{"resources.kubesphere.io"}, - Resources: []string{"nodes", "nodes/*"}, - }, { - Verbs: []string{"get", "list"}, - APIGroups: []string{"monitoring.kubesphere.io"}, - Resources: []string{"nodes"}, - }, - }, - }, - {Name: "edit", - Rules: []v1.PolicyRule{ - { - Verbs: []string{"update", "patch"}, - APIGroups: []string{""}, - Resources: []string{"nodes"}, - }, - }, - }, - }, - }, { - Name: "repos", - Actions: []Action{ - {Name: "view", - Rules: []v1.PolicyRule{ - { - Verbs: []string{"get", "watch", "list"}, - APIGroups: []string{"openpitrix.io"}, - Resources: []string{"repos"}, - }, - }, - }, - {Name: "create", - Rules: []v1.PolicyRule{ - { - Verbs: []string{"create"}, - APIGroups: []string{"openpitrix.io"}, - Resources: []string{"repos"}, - }, - }, - }, - {Name: "edit", - Rules: []v1.PolicyRule{ - { - Verbs: []string{"update", "patch"}, - APIGroups: []string{"openpitrix.io"}, - Resources: []string{"repos"}, - }, - }, - }, - {Name: "delete", - Rules: []v1.PolicyRule{ - { - Verbs: []string{"delete", "deletecollection"}, - APIGroups: []string{"openpitrix.io"}, - Resources: []string{"repos"}, - }, - }, - }, - }, - }, { - Name: "apps", - Actions: []Action{ - {Name: "view", - Rules: []v1.PolicyRule{ - { - Verbs: []string{"get", "list"}, - APIGroups: []string{"openpitrix.io"}, - Resources: []string{"apps", "clusters", "repos", "app_versions", "app_version/*"}, - }, - }, - }, - }, - }, { - Name: "components", - Actions: []Action{ - {Name: "view", - Rules: []v1.PolicyRule{ - { - Verbs: []string{"list", "get"}, - APIGroups: []string{"resources.kubesphere.io"}, - Resources: []string{"components", "components/*"}, - }, - }, - }, - }, - }} - - RoleRuleMapping = []Rule{{ - Name: "projects", - Actions: []Action{ - {Name: "view", - Rules: []v1.PolicyRule{ - { - Verbs: []string{"get"}, - APIGroups: []string{"*"}, - Resources: []string{"namespaces"}, - }, - { - Verbs: []string{"list"}, - APIGroups: []string{"*"}, - Resources: []string{"events"}, - }, - }, - }, - {Name: "edit", - Rules: []v1.PolicyRule{ - { - Verbs: []string{"update", "patch"}, - APIGroups: []string{""}, - Resources: []string{"namespaces"}, - }, - }, - }, - {Name: "delete", - Rules: []v1.PolicyRule{ - { - Verbs: []string{"delete"}, - APIGroups: []string{""}, - Resources: []string{"namespaces"}, - }, - }, - }, - }, - }, - { - Name: "monitoring", - Actions: []Action{ - {Name: "view", - Rules: []v1.PolicyRule{{ - Verbs: []string{"get", "list"}, - APIGroups: []string{"monitoring.kubesphere.io"}, - Resources: []string{"*"}, - }, { - Verbs: []string{"get", "list"}, - APIGroups: []string{"resources.kubesphere.io"}, - Resources: []string{"health"}, - }}, - }, - }, - }, - - { - Name: "alerting", - Actions: []Action{ - {Name: "view", - Rules: []v1.PolicyRule{{ - Verbs: []string{"get", "list"}, - APIGroups: []string{"alerting.kubesphere.io"}, - Resources: []string{"*"}, - }}, - }, - {Name: "create", - Rules: []v1.PolicyRule{{ - Verbs: []string{"create"}, - APIGroups: []string{"alerting.kubesphere.io"}, - Resources: []string{"*"}, - }}, - }, - {Name: "delete", - Rules: []v1.PolicyRule{{ - Verbs: []string{"delete"}, - APIGroups: []string{"alerting.kubesphere.io"}, - Resources: []string{"*"}, - }}, - }, - }, - }, - { - Name: "members", - Actions: []Action{ - {Name: "view", - Rules: []v1.PolicyRule{ - { - Verbs: []string{"get", "list"}, - APIGroups: []string{"rbac.authorization.k8s.io", "resources.kubesphere.io"}, - Resources: []string{"rolebindings"}, - }, - { - Verbs: []string{"get", "list"}, - APIGroups: []string{"iam.kubesphere.io"}, - Resources: []string{"users"}, - }, - }, - }, - {Name: "create", - Rules: []v1.PolicyRule{ - { - Verbs: []string{"create"}, - APIGroups: []string{"rbac.authorization.k8s.io"}, - Resources: []string{"rolebindings"}, - }, - }, - }, - {Name: "edit", - Rules: []v1.PolicyRule{ - { - Verbs: []string{"get", "watch", "list", "create", "update", "patch"}, - APIGroups: []string{"rbac.authorization.k8s.io"}, - Resources: []string{"rolebindings"}, - }, - }, - }, - {Name: "delete", - Rules: []v1.PolicyRule{ - { - Verbs: []string{"delete"}, - APIGroups: []string{"rbac.authorization.k8s.io"}, - Resources: []string{"rolebindings"}, - }, - }, - }, - }, - }, - { - Name: "roles", - Actions: []Action{ - {Name: "view", - Rules: []v1.PolicyRule{ - { - Verbs: []string{"get", "list"}, - APIGroups: []string{"rbac.authorization.k8s.io", "resources.kubesphere.io"}, - Resources: []string{"roles"}, - }, - }, - }, - {Name: "create", - Rules: []v1.PolicyRule{ - { - Verbs: []string{"create"}, - APIGroups: []string{"rbac.authorization.k8s.io"}, - Resources: []string{"roles"}, - }, - }, - }, - {Name: "edit", - Rules: []v1.PolicyRule{ - { - Verbs: []string{"patch", "update"}, - APIGroups: []string{"rbac.authorization.k8s.io"}, - Resources: []string{"roles"}, - }, - }, - }, - {Name: "delete", - Rules: []v1.PolicyRule{ - { - Verbs: []string{"delete"}, - APIGroups: []string{"rbac.authorization.k8s.io"}, - Resources: []string{"roles"}, - }, - }, - }, - }, - }, - { - Name: "deployments", - Actions: []Action{ - {Name: "view", - Rules: []v1.PolicyRule{ - { - Verbs: []string{"get", "list"}, - APIGroups: []string{"apps", "extensions", "resources.kubesphere.io"}, - Resources: []string{"deployments", "deployments/scale"}, - }, - { - Verbs: []string{"get", "list"}, - APIGroups: []string{""}, - Resources: []string{"pods", "pods/*"}, - }, - }, - }, - {Name: "create", - Rules: []v1.PolicyRule{ - { - Verbs: []string{"create"}, - APIGroups: []string{"apps", "extensions"}, - Resources: []string{"deployments"}, - }, - }, - }, - - {Name: "edit", - Rules: []v1.PolicyRule{ - { - Verbs: []string{"update", "patch"}, - APIGroups: []string{"apps", "extensions"}, - Resources: []string{"deployments", "deployments/*"}, - }, - }, - }, - - {Name: "delete", - Rules: []v1.PolicyRule{ - { - Verbs: []string{"delete"}, - APIGroups: []string{"apps", "extensions"}, - Resources: []string{"deployments"}, - }, - }, - }, - {Name: "scale", - Rules: []v1.PolicyRule{ - { - Verbs: []string{"update", "patch"}, - APIGroups: []string{"apps", "extensions"}, - Resources: []string{"deployments/scale"}, - }, - }, - }, - }, - }, { - Name: "statefulsets", - Actions: []Action{ - {Name: "view", - Rules: []v1.PolicyRule{ - { - Verbs: []string{"get", "list"}, - APIGroups: []string{"apps", "resources.kubesphere.io"}, - Resources: []string{"statefulsets"}, - }, - { - Verbs: []string{"get", "list"}, - APIGroups: []string{""}, - Resources: []string{"pods", "pods/*"}, - }, - }, - }, - {Name: "create", - Rules: []v1.PolicyRule{ - { - Verbs: []string{"create"}, - APIGroups: []string{"apps"}, - Resources: []string{"statefulsets"}, - }, - }, - }, - {Name: "edit", - Rules: []v1.PolicyRule{ - { - Verbs: []string{"update", "patch"}, - APIGroups: []string{"apps"}, - Resources: []string{"statefulsets"}, - }, - }, - }, - {Name: "delete", - Rules: []v1.PolicyRule{ - { - Verbs: []string{"delete"}, - APIGroups: []string{"apps"}, - Resources: []string{"statefulsets"}, - }, - }, - }, - {Name: "scale", - Rules: []v1.PolicyRule{ - { - Verbs: []string{"patch"}, - APIGroups: []string{"apps"}, - Resources: []string{"statefulsets"}, - }, - }, - }, - }, - }, { - Name: "daemonsets", - Actions: []Action{ - {Name: "view", - Rules: []v1.PolicyRule{ - { - Verbs: []string{"get", "list"}, - APIGroups: []string{"apps", "extensions", "resources.kubesphere.io"}, - Resources: []string{"daemonsets"}, - }, - { - Verbs: []string{"get", "list"}, - APIGroups: []string{""}, - Resources: []string{"pods", "pods/*"}, - }, - }, - }, - {Name: "create", - Rules: []v1.PolicyRule{ - { - Verbs: []string{"create"}, - APIGroups: []string{"apps", "extensions"}, - Resources: []string{"daemonsets"}, - }, - }, - }, - {Name: "edit", - Rules: []v1.PolicyRule{ - { - Verbs: []string{"update", "patch"}, - APIGroups: []string{"apps", "extensions"}, - Resources: []string{"daemonsets"}, - }, - }, - }, - {Name: "delete", - Rules: []v1.PolicyRule{ - { - Verbs: []string{"delete"}, - APIGroups: []string{"apps", "extensions"}, - Resources: []string{"daemonsets"}, - }, - }, - }, - }, - }, { - Name: "pods", - Actions: []Action{ - {Name: "terminal", - Rules: []v1.PolicyRule{ - { - Verbs: []string{"get"}, - APIGroups: []string{"terminal.kubesphere.io"}, - Resources: []string{"pods"}, - }, - }, - }, - {Name: "view", - Rules: []v1.PolicyRule{ - { - Verbs: []string{"get", "list"}, - APIGroups: []string{"resources.kubesphere.io"}, - Resources: []string{"pods"}, - }, - }, - }, - {Name: "delete", - Rules: []v1.PolicyRule{ - { - Verbs: []string{"delete"}, - APIGroups: []string{"*"}, - Resources: []string{"pods"}, - }, - }, - }, - }, - }, - { - Name: "services", - Actions: []Action{ - {Name: "view", - Rules: []v1.PolicyRule{ - { - Verbs: []string{"list", "get"}, - APIGroups: []string{"", "resources.kubesphere.io"}, - Resources: []string{"services"}, - }, - }, - }, - {Name: "create", - Rules: []v1.PolicyRule{ - { - Verbs: []string{"create"}, - APIGroups: []string{""}, - Resources: []string{"services"}, - }, - }, - }, - - {Name: "edit", - Rules: []v1.PolicyRule{ - { - Verbs: []string{"update", "patch"}, - APIGroups: []string{""}, - Resources: []string{"services"}, - }, - }, - }, - {Name: "delete", - Rules: []v1.PolicyRule{ - { - Verbs: []string{"delete"}, - APIGroups: []string{""}, - Resources: []string{"services"}, - }, - }, - }, - }, - }, - { - Name: "internet", - Actions: []Action{ - {Name: "view", - Rules: []v1.PolicyRule{ - { - Verbs: []string{"get", "list"}, - APIGroups: []string{"resources.kubesphere.io"}, - Resources: []string{"router"}, - }, - }, - }, - {Name: "create", - Rules: []v1.PolicyRule{ - { - Verbs: []string{"create"}, - APIGroups: []string{"resources.kubesphere.io"}, - Resources: []string{"router"}, - }, - }, - }, - {Name: "edit", - Rules: []v1.PolicyRule{ - { - Verbs: []string{"update", "patch"}, - APIGroups: []string{"resources.kubesphere.io"}, - Resources: []string{"router"}, - }, - }, - }, - {Name: "delete", - Rules: []v1.PolicyRule{ - { - Verbs: []string{"delete"}, - APIGroups: []string{"resources.kubesphere.io"}, - Resources: []string{"router"}, - }, - }, - }, - }, - }, - - { - Name: "routes", - Actions: []Action{ - {Name: "view", - Rules: []v1.PolicyRule{ - { - Verbs: []string{"get", "list"}, - APIGroups: []string{"extensions", "resources.kubesphere.io"}, - Resources: []string{"ingresses"}, - }, - }, - }, - {Name: "create", - Rules: []v1.PolicyRule{ - { - Verbs: []string{"create"}, - APIGroups: []string{"extensions"}, - Resources: []string{"ingresses"}, - }, - }, - }, - {Name: "edit", - Rules: []v1.PolicyRule{ - { - Verbs: []string{"update", "patch"}, - APIGroups: []string{"extensions"}, - Resources: []string{"ingresses"}, - }, - }, - }, - {Name: "delete", - Rules: []v1.PolicyRule{ - { - Verbs: []string{"delete"}, - APIGroups: []string{"extensions"}, - Resources: []string{"ingresses"}, - }, - }, - }, - }, - }, { - Name: "volumes", - Actions: []Action{ - {Name: "view", - Rules: []v1.PolicyRule{ - { - Verbs: []string{"get", "list"}, - APIGroups: []string{"", "resources.kubesphere.io"}, - Resources: []string{"persistentvolumeclaims"}, - }, - }, - }, - {Name: "create", - Rules: []v1.PolicyRule{ - { - Verbs: []string{"create"}, - APIGroups: []string{""}, - Resources: []string{"persistentvolumeclaims"}, - }, - }, - }, - {Name: "edit", - Rules: []v1.PolicyRule{ - { - Verbs: []string{"update", "patch"}, - APIGroups: []string{""}, - Resources: []string{"persistentvolumeclaims"}, - }, - }, - }, - {Name: "delete", - Rules: []v1.PolicyRule{ - { - Verbs: []string{"delete"}, - APIGroups: []string{""}, - Resources: []string{"persistentvolumeclaims"}, - }, - }, - }, - }, - }, { - Name: "applications", - Actions: []Action{ - {Name: "view", - Rules: []v1.PolicyRule{ - { - Verbs: []string{"get", "list"}, - APIGroups: []string{"resources.kubesphere.io", "app.k8s.io"}, - Resources: []string{"applications"}, - }, { - Verbs: []string{"get", "list"}, - APIGroups: []string{"servicemesh.kubesphere.io"}, - Resources: []string{"*"}, - }, - { - Verbs: []string{"list"}, - APIGroups: []string{"openpitrix.io"}, - Resources: []string{"repos", "app_versions"}, - }, { - Verbs: []string{"get"}, - APIGroups: []string{"openpitrix.io"}, - Resources: []string{"app_version/*"}, - }, - }, - }, - {Name: "edit", - Rules: []v1.PolicyRule{ - { - Verbs: []string{"create", "update", "patch"}, - APIGroups: []string{"resources.kubesphere.io", "app.k8s.io"}, - Resources: []string{"applications"}, - }, { - Verbs: []string{"create", "update", "patch"}, - APIGroups: []string{"servicemesh.kubesphere.io"}, - Resources: []string{"*"}, - }, - }, - }, - {Name: "delete", - Rules: []v1.PolicyRule{ - { - Verbs: []string{"delete"}, - APIGroups: []string{"resources.kubesphere.io", "app.k8s.io"}, - Resources: []string{"applications"}, - }, - { - Verbs: []string{"delete"}, - APIGroups: []string{"servicemesh.kubesphere.io"}, - Resources: []string{"*"}, - }, - }, - }, - }, - }, - { - Name: "jobs", - Actions: []Action{ - {Name: "view", Rules: []v1.PolicyRule{ - { - Verbs: []string{"get", "list"}, - APIGroups: []string{"batch", "resources.kubesphere.io"}, - Resources: []string{"jobs"}, - }, - }}, - {Name: "create", Rules: []v1.PolicyRule{ - { - Verbs: []string{"create"}, - APIGroups: []string{"batch"}, - Resources: []string{"jobs"}, - }, - }}, - {Name: "edit", Rules: []v1.PolicyRule{ - { - Verbs: []string{"update", "patch"}, - APIGroups: []string{"batch"}, - Resources: []string{"jobs"}, - }, - }}, - {Name: "delete", Rules: []v1.PolicyRule{ - { - Verbs: []string{"delete"}, - APIGroups: []string{"batch"}, - Resources: []string{"jobs"}, - }, - }}, - }, - }, - { - Name: "cronjobs", - Actions: []Action{ - {Name: "view", Rules: []v1.PolicyRule{ - { - Verbs: []string{"get", "list"}, - APIGroups: []string{"batch", "resources.kubesphere.io"}, - Resources: []string{"cronjobs"}, - }, - }}, - {Name: "create", Rules: []v1.PolicyRule{ - { - Verbs: []string{"create"}, - APIGroups: []string{"batch"}, - Resources: []string{"cronjobs"}, - }, - }}, - {Name: "edit", Rules: []v1.PolicyRule{ - { - Verbs: []string{"update", "patch"}, - APIGroups: []string{"batch"}, - Resources: []string{"cronjobs"}, - }, - }}, - {Name: "delete", Rules: []v1.PolicyRule{ - { - Verbs: []string{"delete"}, - APIGroups: []string{"batch"}, - Resources: []string{"cronjobs"}, - }, - }}, - }, - }, - { - Name: "secrets", - Actions: []Action{ - {Name: "view", Rules: []v1.PolicyRule{ - { - Verbs: []string{"get", "list"}, - APIGroups: []string{"", "resources.kubesphere.io"}, - Resources: []string{"secrets"}, - }, - }}, - {Name: "create", Rules: []v1.PolicyRule{ - { - Verbs: []string{"create"}, - APIGroups: []string{""}, - Resources: []string{"secrets"}, - }, - }}, - {Name: "edit", Rules: []v1.PolicyRule{ - { - Verbs: []string{"update", "patch"}, - APIGroups: []string{""}, - Resources: []string{"secrets"}, - }, - }}, - {Name: "delete", Rules: []v1.PolicyRule{ - { - Verbs: []string{"delete"}, - APIGroups: []string{""}, - Resources: []string{"secrets"}, - }, - }}, - }, - }, - { - Name: "configmaps", - Actions: []Action{ - {Name: "view", Rules: []v1.PolicyRule{ - { - Verbs: []string{"get", "list"}, - APIGroups: []string{"", "resources.kubesphere.io"}, - Resources: []string{"configmaps"}, - }, - }}, - {Name: "create", Rules: []v1.PolicyRule{ - { - Verbs: []string{"create"}, - APIGroups: []string{""}, - Resources: []string{"configmaps"}, - }, - }}, - {Name: "edit", Rules: []v1.PolicyRule{ - { - Verbs: []string{"update", "patch"}, - APIGroups: []string{""}, - Resources: []string{"configmaps"}, - }, - }}, - {Name: "delete", Rules: []v1.PolicyRule{ - { - Verbs: []string{"delete"}, - APIGroups: []string{""}, - Resources: []string{"configmaps"}, - }, - }}, - }, - }, - } -) - -type Action struct { - Name string `json:"name"` - Rules []v1.PolicyRule `json:"rules"` -} - -type Rule struct { - Name string `json:"name"` - Actions []Action `json:"actions"` -} - -type SimpleRule struct { - Name string `json:"name" description:"rule name"` - Actions []string `json:"actions" description:"actions"` -} diff --git a/pkg/models/iam/utils.go b/pkg/models/iam/utils.go deleted file mode 100644 index b2638d678..000000000 --- a/pkg/models/iam/utils.go +++ /dev/null @@ -1,213 +0,0 @@ -/* - * - * Copyright 2020 The KubeSphere Authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - * / - */ - -package iam - -import ( - rbacv1 "k8s.io/api/rbac/v1" - "kubesphere.io/kubesphere/pkg/api/iam" - "kubesphere.io/kubesphere/pkg/models/iam/policy" - "strings" -) - -func RulesMatchesRequired(rules []rbacv1.PolicyRule, required rbacv1.PolicyRule) bool { - for _, rule := range rules { - if ruleMatchesRequired(rule, required) { - return true - } - } - return false -} - -func rulesMatchesAction(rules []rbacv1.PolicyRule, action policy.Action) bool { - - for _, required := range action.Rules { - if !RulesMatchesRequired(rules, required) { - return false - } - } - - return true -} - -func ruleMatchesRequired(rule rbacv1.PolicyRule, required rbacv1.PolicyRule) bool { - - if len(required.NonResourceURLs) == 0 { - for _, apiGroup := range required.APIGroups { - for _, resource := range required.Resources { - resources := strings.Split(resource, "/") - resource = resources[0] - var subsource string - if len(resources) > 1 { - subsource = resources[1] - } - - if len(required.ResourceNames) == 0 { - for _, verb := range required.Verbs { - if !ruleMatchesRequest(rule, apiGroup, "", resource, subsource, "", verb) { - return false - } - } - } else { - for _, resourceName := range required.ResourceNames { - for _, verb := range required.Verbs { - if !ruleMatchesRequest(rule, apiGroup, "", resource, subsource, resourceName, verb) { - return false - } - } - } - } - } - } - } else { - for _, apiGroup := range required.APIGroups { - for _, nonResourceURL := range required.NonResourceURLs { - for _, verb := range required.Verbs { - if !ruleMatchesRequest(rule, apiGroup, nonResourceURL, "", "", "", verb) { - return false - } - } - } - } - } - return true -} - -func ruleMatchesResources(rule rbacv1.PolicyRule, apiGroup string, resource string, subresource string, resourceName string) bool { - - if resource == "" { - return false - } - - if !hasString(rule.APIGroups, apiGroup) && !hasString(rule.APIGroups, rbacv1.ResourceAll) { - return false - } - - if len(rule.ResourceNames) > 0 && !hasString(rule.ResourceNames, resourceName) { - return false - } - - combinedResource := resource - - if subresource != "" { - combinedResource = combinedResource + "/" + subresource - } - - for _, res := range rule.Resources { - - // match "*" - if res == rbacv1.ResourceAll || res == combinedResource { - return true - } - - // match "*/subresource" - if len(subresource) > 0 && strings.HasPrefix(res, "*/") && subresource == strings.TrimLeft(res, "*/") { - return true - } - // match "resource/*" - if strings.HasSuffix(res, "/*") && resource == strings.TrimRight(res, "/*") { - return true - } - } - - return false -} - -func ruleMatchesRequest(rule rbacv1.PolicyRule, apiGroup string, nonResourceURL string, resource string, subresource string, resourceName string, verb string) bool { - - if !hasString(rule.Verbs, verb) && !hasString(rule.Verbs, rbacv1.VerbAll) { - return false - } - - if nonResourceURL == "" { - return ruleMatchesResources(rule, apiGroup, resource, subresource, resourceName) - } else { - return ruleMatchesNonResource(rule, nonResourceURL) - } -} - -func ruleMatchesNonResource(rule rbacv1.PolicyRule, nonResourceURL string) bool { - - if nonResourceURL == "" { - return false - } - - for _, spec := range rule.NonResourceURLs { - if pathMatches(nonResourceURL, spec) { - return true - } - } - - return false -} - -func pathMatches(path, spec string) bool { - // Allow wildcard match - if spec == "*" { - return true - } - // Allow exact match - if spec == path { - return true - } - // Allow a trailing * subpath match - if strings.HasSuffix(spec, "*") && strings.HasPrefix(path, strings.TrimRight(spec, "*")) { - return true - } - return false -} - -func hasString(slice []string, value string) bool { - for _, s := range slice { - if s == value { - return true - } - } - return false -} - -func ContainsUser(subjects interface{}, username string) bool { - switch subjects.(type) { - case []*rbacv1.Subject: - for _, subject := range subjects.([]*rbacv1.Subject) { - if subject.Kind == rbacv1.UserKind && subject.Name == username { - return true - } - } - case []rbacv1.Subject: - for _, subject := range subjects.([]rbacv1.Subject) { - if subject.Kind == rbacv1.UserKind && subject.Name == username { - return true - } - } - case []iam.User: - for _, u := range subjects.([]iam.User) { - if u.Username == username { - return true - } - } - - case []*iam.User: - for _, u := range subjects.([]*iam.User) { - if u.Username == username { - return true - } - } - } - return false -} diff --git a/pkg/models/tenant/devops.go b/pkg/models/tenant/devops.go index 44e1e75ae..b0cdb6b2a 100644 --- a/pkg/models/tenant/devops.go +++ b/pkg/models/tenant/devops.go @@ -26,7 +26,6 @@ import ( "kubesphere.io/kubesphere/pkg/db" "kubesphere.io/kubesphere/pkg/models" "kubesphere.io/kubesphere/pkg/models/devops" - "kubesphere.io/kubesphere/pkg/models/iam/policy" "kubesphere.io/kubesphere/pkg/server/params" dsClient "kubesphere.io/kubesphere/pkg/simple/client/devops" "kubesphere.io/kubesphere/pkg/simple/client/mysql" @@ -38,7 +37,6 @@ type DevOpsProjectOperator interface { CreateDevOpsProject(username string, workspace string, req *v1alpha2.DevOpsProject) (*v1alpha2.DevOpsProject, error) GetDevOpsProjectsCount(username string) (uint32, error) DeleteDevOpsProject(projectId, username string) error - GetUserDevOpsSimpleRules(username, projectId string) ([]policy.SimpleRule, error) } type devopsProjectOperator struct { @@ -208,16 +206,6 @@ func (o *devopsProjectOperator) CreateDevOpsProject(username string, workspace s return project, nil } -func (o *devopsProjectOperator) GetUserDevOpsSimpleRules(username, projectId string) ([]policy.SimpleRule, error) { - - role, err := o.getProjectUserRole(username, projectId) - if err != nil { - klog.Errorf("%+v", err) - return nil, restful.NewError(http.StatusForbidden, err.Error()) - } - return GetDevopsRoleSimpleRules(role), nil -} - func (o *devopsProjectOperator) getProjectUserRole(username, projectId string) (string, error) { if username == devops.KS_ADMIN { return dsClient.ProjectOwner, nil @@ -235,47 +223,3 @@ func (o *devopsProjectOperator) getProjectUserRole(username, projectId string) ( return membership.Role, nil } - -func GetDevopsRoleSimpleRules(role string) []policy.SimpleRule { - var rules []policy.SimpleRule - - switch role { - case "developer": - rules = []policy.SimpleRule{ - {Name: "pipelines", Actions: []string{"view", "trigger"}}, - {Name: "roles", Actions: []string{"view"}}, - {Name: "members", Actions: []string{"view"}}, - {Name: "devops", Actions: []string{"view"}}, - } - break - case "owner": - rules = []policy.SimpleRule{ - {Name: "pipelines", Actions: []string{"create", "edit", "view", "delete", "trigger"}}, - {Name: "roles", Actions: []string{"view"}}, - {Name: "members", Actions: []string{"create", "edit", "view", "delete"}}, - {Name: "credentials", Actions: []string{"create", "edit", "view", "delete"}}, - {Name: "devops", Actions: []string{"edit", "view", "delete"}}, - } - break - case "maintainer": - rules = []policy.SimpleRule{ - {Name: "pipelines", Actions: []string{"create", "edit", "view", "delete", "trigger"}}, - {Name: "roles", Actions: []string{"view"}}, - {Name: "members", Actions: []string{"view"}}, - {Name: "credentials", Actions: []string{"create", "edit", "view", "delete"}}, - {Name: "devops", Actions: []string{"view"}}, - } - break - case "reporter": - fallthrough - default: - rules = []policy.SimpleRule{ - {Name: "pipelines", Actions: []string{"view"}}, - {Name: "roles", Actions: []string{"view"}}, - {Name: "members", Actions: []string{"view"}}, - {Name: "devops", Actions: []string{"view"}}, - } - break - } - return rules -} diff --git a/pkg/models/tenant/namespaces.go b/pkg/models/tenant/namespaces.go index 71e2f5986..b8ea11a5c 100644 --- a/pkg/models/tenant/namespaces.go +++ b/pkg/models/tenant/namespaces.go @@ -19,17 +19,13 @@ package tenant import ( "k8s.io/api/core/v1" - rbacv1 "k8s.io/api/rbac/v1" - "k8s.io/apimachinery/pkg/labels" k8sinformers "k8s.io/client-go/informers" - kubernetes "k8s.io/client-go/kubernetes" - "k8s.io/klog" + "k8s.io/client-go/kubernetes" "kubesphere.io/kubesphere/pkg/constants" - "kubesphere.io/kubesphere/pkg/models/iam" + "kubesphere.io/kubesphere/pkg/models/iam/am" "kubesphere.io/kubesphere/pkg/models/resources/v1alpha2" "kubesphere.io/kubesphere/pkg/server/params" "kubesphere.io/kubesphere/pkg/utils/sliceutil" - "sort" "strings" ) @@ -41,7 +37,7 @@ type NamespaceInterface interface { type namespaceSearcher struct { k8s kubernetes.Interface informers k8sinformers.SharedInformerFactory - am iam.AccessManagementInterface + am am.AccessManagementInterface } func (s *namespaceSearcher) CreateNamespace(workspace string, namespace *v1.Namespace, username string) (*v1.Namespace, error) { @@ -57,7 +53,7 @@ func (s *namespaceSearcher) CreateNamespace(workspace string, namespace *v1.Name return s.k8s.CoreV1().Namespaces().Create(namespace) } -func newNamespaceOperator(k8s kubernetes.Interface, informers k8sinformers.SharedInformerFactory, am iam.AccessManagementInterface) NamespaceInterface { +func newNamespaceOperator(k8s kubernetes.Interface, informers k8sinformers.SharedInformerFactory, am am.AccessManagementInterface) NamespaceInterface { return &namespaceSearcher{k8s: k8s, informers: informers, am: am} } @@ -111,76 +107,9 @@ func (s *namespaceSearcher) compare(a, b *v1.Namespace, orderBy string) bool { } func (s *namespaceSearcher) GetNamespaces(username string) ([]*v1.Namespace, error) { - - roles, err := s.am.GetRoles("", username) - - if err != nil { - return nil, err - } - namespaces := make([]*v1.Namespace, 0) - namespaceLister := s.informers.Core().V1().Namespaces().Lister() - for _, role := range roles { - namespace, err := namespaceLister.Get(role.Namespace) - if err != nil { - klog.Errorf("get namespace failed: %+v", err) - return nil, err - } - if !containsNamespace(namespaces, namespace) { - namespaces = append(namespaces, namespace) - } - } - - return namespaces, nil -} - -func containsNamespace(namespaces []*v1.Namespace, namespace *v1.Namespace) bool { - for _, item := range namespaces { - if item.Name == namespace.Name { - return true - } - } - return false + panic("implement me") } func (s *namespaceSearcher) Search(username string, conditions *params.Conditions, orderBy string, reverse bool) ([]*v1.Namespace, error) { - - rules, err := s.am.GetClusterPolicyRules(username) - - if err != nil { - return nil, err - } - - namespaces := make([]*v1.Namespace, 0) - - if iam.RulesMatchesRequired(rules, rbacv1.PolicyRule{Verbs: []string{"list"}, APIGroups: []string{"tenant.kubesphere.io"}, Resources: []string{"namespaces"}}) { - namespaces, err = s.informers.Core().V1().Namespaces().Lister().List(labels.Everything()) - } else { - namespaces, err = s.GetNamespaces(username) - } - - if err != nil { - return nil, err - } - - result := make([]*v1.Namespace, 0) - - for _, namespace := range namespaces { - if s.match(conditions.Match, namespace) && s.fuzzy(conditions.Fuzzy, namespace) { - result = append(result, namespace) - } - } - - // order & reverse - sort.Slice(result, func(i, j int) bool { - if reverse { - i, j = j, i - } - return s.compare(result[i], result[j], orderBy) - }) - - return result, nil -} - -func CreateNamespace() { - + panic("implement me") } diff --git a/pkg/models/tenant/tenant.go b/pkg/models/tenant/tenant.go index 1c8d91f77..dd3ace488 100644 --- a/pkg/models/tenant/tenant.go +++ b/pkg/models/tenant/tenant.go @@ -19,20 +19,14 @@ package tenant import ( "k8s.io/api/core/v1" - rbacv1 "k8s.io/api/rbac/v1" - apierrors "k8s.io/apimachinery/pkg/api/errors" k8sinformers "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes" - "k8s.io/klog" "kubesphere.io/kubesphere/pkg/apis/tenant/v1alpha1" ksinformers "kubesphere.io/kubesphere/pkg/client/informers/externalversions" - "kubesphere.io/kubesphere/pkg/constants" "kubesphere.io/kubesphere/pkg/models" - "kubesphere.io/kubesphere/pkg/models/iam" - "kubesphere.io/kubesphere/pkg/models/iam/policy" + "kubesphere.io/kubesphere/pkg/models/iam/am" "kubesphere.io/kubesphere/pkg/server/params" "kubesphere.io/kubesphere/pkg/simple/client/mysql" - "strconv" ) type Interface interface { @@ -42,17 +36,14 @@ type Interface interface { ListWorkspaces(username string, conditions *params.Conditions, orderBy string, reverse bool, limit, offset int) (*models.PageableResponse, error) ListNamespaces(username string, conditions *params.Conditions, orderBy string, reverse bool, limit, offset int) (*models.PageableResponse, error) ListDevopsProjects(username string, conditions *params.Conditions, orderBy string, reverse bool, limit int, offset int) (*models.PageableResponse, error) - GetWorkspaceSimpleRules(workspace, username string) ([]policy.SimpleRule, error) - GetNamespaceSimpleRules(namespace, username string) ([]policy.SimpleRule, error) CountDevOpsProjects(username string) (uint32, error) DeleteDevOpsProject(username, projectId string) error - GetUserDevopsSimpleRules(username string, devops string) (interface{}, error) } type tenantOperator struct { workspaces WorkspaceInterface namespaces NamespaceInterface - am iam.AccessManagementInterface + am am.AccessManagementInterface devops DevOpsProjectOperator } @@ -65,7 +56,7 @@ func (t *tenantOperator) DeleteDevOpsProject(username, projectId string) error { } func (t *tenantOperator) GetUserDevopsSimpleRules(username string, projectId string) (interface{}, error) { - return t.devops.GetUserDevOpsSimpleRules(username, projectId) + panic("implement me") } func (t *tenantOperator) ListDevopsProjects(username string, conditions *params.Conditions, orderBy string, reverse bool, limit int, offset int) (*models.PageableResponse, error) { @@ -77,7 +68,7 @@ func (t *tenantOperator) DeleteNamespace(workspace, namespace string) error { } func New(client kubernetes.Interface, informers k8sinformers.SharedInformerFactory, ksinformers ksinformers.SharedInformerFactory, db *mysql.Database) Interface { - amOperator := iam.NewAMOperator(client, informers) + amOperator := am.NewAMOperator(client, informers) return &tenantOperator{ workspaces: newWorkspaceOperator(client, informers, ksinformers, amOperator, db), namespaces: newNamespaceOperator(client, informers, amOperator), @@ -96,105 +87,11 @@ func (t *tenantOperator) DescribeWorkspace(username, workspaceName string) (*v1a return nil, err } - if username != "" { - workspace = t.appendAnnotations(username, workspace) - } - return workspace, nil } func (t *tenantOperator) ListWorkspaces(username string, conditions *params.Conditions, orderBy string, reverse bool, limit, offset int) (*models.PageableResponse, error) { - - workspaces, err := t.workspaces.SearchWorkspace(username, conditions, orderBy, reverse) - - if err != nil { - return nil, err - } - - // limit offset - result := make([]interface{}, 0) - for i, workspace := range workspaces { - if len(result) < limit && i >= offset { - workspace := t.appendAnnotations(username, workspace) - result = append(result, workspace) - } - } - - return &models.PageableResponse{Items: result, TotalCount: len(workspaces)}, nil -} - -func (t *tenantOperator) GetWorkspaceSimpleRules(workspace, username string) ([]policy.SimpleRule, error) { - clusterRules, err := t.am.GetClusterPolicyRules(username) - if err != nil { - return nil, err - } - - // cluster-admin - if iam.RulesMatchesRequired(clusterRules, rbacv1.PolicyRule{ - Verbs: []string{"*"}, - APIGroups: []string{"*"}, - Resources: []string{"*"}, - }) { - return t.am.GetWorkspaceRoleSimpleRules(workspace, constants.WorkspaceAdmin), nil - } - - workspaceRole, err := t.am.GetWorkspaceRole(workspace, username) - - // workspaces-manager - if iam.RulesMatchesRequired(clusterRules, rbacv1.PolicyRule{ - Verbs: []string{"*"}, - APIGroups: []string{"*"}, - Resources: []string{"workspaces", "workspaces/*"}, - }) { - return t.am.GetWorkspaceRoleSimpleRules(workspace, constants.WorkspacesManager), nil - } - - if err != nil { - if apierrors.IsNotFound(err) { - return []policy.SimpleRule{}, nil - } - - klog.Error(err) - return nil, err - } - - return t.am.GetWorkspaceRoleSimpleRules(workspace, workspaceRole.Annotations[constants.DisplayNameAnnotationKey]), nil -} - -func (t *tenantOperator) GetNamespaceSimpleRules(namespace, username string) ([]policy.SimpleRule, error) { - clusterRules, err := t.am.GetClusterPolicyRules(username) - if err != nil { - return nil, err - } - rules, err := t.am.GetPolicyRules(namespace, username) - if err != nil { - return nil, err - } - rules = append(rules, clusterRules...) - - return iam.ConvertToSimpleRule(rules), nil -} - -func (t *tenantOperator) appendAnnotations(username string, workspace *v1alpha1.Workspace) *v1alpha1.Workspace { - workspace = workspace.DeepCopy() - if workspace.Annotations == nil { - workspace.Annotations = make(map[string]string) - } - - if ns, err := t.ListNamespaces(username, ¶ms.Conditions{Match: map[string]string{constants.WorkspaceLabelKey: workspace.Name}}, "", false, 1, 0); err == nil { - workspace.Annotations["kubesphere.io/namespace-count"] = strconv.Itoa(ns.TotalCount) - } - - if devops, err := t.ListDevopsProjects(username, ¶ms.Conditions{Match: map[string]string{"workspace": workspace.Name}}, "", false, 1, 0); err == nil { - workspace.Annotations["kubesphere.io/devops-count"] = strconv.Itoa(devops.TotalCount) - } - - userCount, err := t.workspaces.CountUsersInWorkspace(workspace.Name) - - if err == nil { - workspace.Annotations["kubesphere.io/member-count"] = strconv.Itoa(userCount) - } - return workspace + panic("implement me") } func (t *tenantOperator) ListNamespaces(username string, conditions *params.Conditions, orderBy string, reverse bool, limit, offset int) (*models.PageableResponse, error) { diff --git a/pkg/models/tenant/workspaces.go b/pkg/models/tenant/workspaces.go index 2dd646977..f56103177 100644 --- a/pkg/models/tenant/workspaces.go +++ b/pkg/models/tenant/workspaces.go @@ -18,27 +18,22 @@ package tenant import ( - "fmt" core "k8s.io/api/core/v1" - rbacv1 "k8s.io/api/rbac/v1" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes" - "k8s.io/klog" "kubesphere.io/kubesphere/pkg/apis/tenant/v1alpha1" "kubesphere.io/kubesphere/pkg/client/informers/externalversions" "kubesphere.io/kubesphere/pkg/constants" "kubesphere.io/kubesphere/pkg/db" "kubesphere.io/kubesphere/pkg/models/devops" - "kubesphere.io/kubesphere/pkg/models/iam" + "kubesphere.io/kubesphere/pkg/models/iam/am" "kubesphere.io/kubesphere/pkg/models/resources/v1alpha2" "kubesphere.io/kubesphere/pkg/server/params" "kubesphere.io/kubesphere/pkg/simple/client/mysql" "kubesphere.io/kubesphere/pkg/utils/sliceutil" - "sort" "strings" - "k8s.io/api/rbac/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" @@ -69,14 +64,14 @@ type workspaceOperator struct { client kubernetes.Interface informers informers.SharedInformerFactory ksInformers externalversions.SharedInformerFactory - am iam.AccessManagementInterface + am am.AccessManagementInterface // TODO: use db interface instead of mysql client // we can refactor this after rewrite devops using crd db *mysql.Database } -func newWorkspaceOperator(client kubernetes.Interface, informers informers.SharedInformerFactory, ksinformers externalversions.SharedInformerFactory, am iam.AccessManagementInterface, db *mysql.Database) WorkspaceInterface { +func newWorkspaceOperator(client kubernetes.Interface, informers informers.SharedInformerFactory, ksinformers externalversions.SharedInformerFactory, am am.AccessManagementInterface, db *mysql.Database) WorkspaceInterface { return &workspaceOperator{ client: client, informers: informers, @@ -111,96 +106,12 @@ func (w *workspaceOperator) DeleteNamespace(workspace string, namespace string) } func (w *workspaceOperator) RemoveUser(workspace string, username string) error { - workspaceRole, err := w.am.GetWorkspaceRole(workspace, username) - if err != nil { - return err - } - - err = w.deleteWorkspaceRoleBinding(workspace, username, workspaceRole.Annotations[constants.DisplayNameAnnotationKey]) - if err != nil { - return err - } - - return nil + panic("implement me") } func (w *workspaceOperator) AddUser(workspaceName string, user *InWorkspaceUser) error { - workspaceRole, err := w.am.GetWorkspaceRole(workspaceName, user.Username) - - if err != nil && !apierrors.IsNotFound(err) { - klog.Errorf("get workspace role failed: %+v", err) - return err - } - - workspaceRoleName := fmt.Sprintf("workspace:%s:%s", workspaceName, strings.TrimPrefix(user.WorkspaceRole, "workspace-")) - var currentWorkspaceRoleName string - if workspaceRole != nil { - currentWorkspaceRoleName = workspaceRole.Name - } - - if currentWorkspaceRoleName != workspaceRoleName && currentWorkspaceRoleName != "" { - err := w.deleteWorkspaceRoleBinding(workspaceName, user.Username, workspaceRole.Annotations[constants.DisplayNameAnnotationKey]) - if err != nil { - klog.Errorf("delete workspace role binding failed: %+v", err) - return err - } - } else if currentWorkspaceRoleName != "" { - return nil - } - - return w.createWorkspaceRoleBinding(workspaceName, user.Username, user.WorkspaceRole) -} - -func (w *workspaceOperator) createWorkspaceRoleBinding(workspace, username string, role string) error { - - if !sliceutil.HasString(constants.WorkSpaceRoles, role) { - return apierrors.NewNotFound(schema.GroupResource{Resource: "workspace role"}, role) - } - - roleBindingName := fmt.Sprintf("workspace:%s:%s", workspace, strings.TrimPrefix(role, "workspace-")) - workspaceRoleBinding, err := w.informers.Rbac().V1().ClusterRoleBindings().Lister().Get(roleBindingName) - if err != nil { - return err - } - - if !iam.ContainsUser(workspaceRoleBinding.Subjects, username) { - workspaceRoleBinding = workspaceRoleBinding.DeepCopy() - workspaceRoleBinding.Subjects = append(workspaceRoleBinding.Subjects, v1.Subject{APIGroup: "rbac.authorization.k8s.io", Kind: "User", Name: username}) - _, err = w.client.RbacV1().ClusterRoleBindings().Update(workspaceRoleBinding) - if err != nil { - klog.Errorf("update workspace role binding failed: %+v", err) - return err - } - } - - return nil -} - -func (w *workspaceOperator) deleteWorkspaceRoleBinding(workspace, username string, role string) error { - - if !sliceutil.HasString(constants.WorkSpaceRoles, role) { - return apierrors.NewNotFound(schema.GroupResource{Resource: "workspace role"}, role) - } - - roleBindingName := fmt.Sprintf("workspace:%s:%s", workspace, strings.TrimPrefix(role, "workspace-")) - - workspaceRoleBinding, err := w.informers.Rbac().V1().ClusterRoleBindings().Lister().Get(roleBindingName) - if err != nil { - return err - } - workspaceRoleBinding = workspaceRoleBinding.DeepCopy() - - for i, v := range workspaceRoleBinding.Subjects { - if v.Kind == v1.UserKind && v.Name == username { - workspaceRoleBinding.Subjects = append(workspaceRoleBinding.Subjects[:i], workspaceRoleBinding.Subjects[i+1:]...) - i-- - } - } - - workspaceRoleBinding, err = w.client.RbacV1().ClusterRoleBindings().Update(workspaceRoleBinding) - - return err + panic("implement me") } func (w *workspaceOperator) CountDevopsProjectsInWorkspace(workspaceName string) (int, error) { @@ -288,50 +199,7 @@ func (*workspaceOperator) compare(a, b *v1alpha1.Workspace, orderBy string) bool } func (w *workspaceOperator) SearchWorkspace(username string, conditions *params.Conditions, orderBy string, reverse bool) ([]*v1alpha1.Workspace, error) { - rules, err := w.am.GetClusterPolicyRules(username) - - if err != nil { - return nil, err - } - - workspaces := make([]*v1alpha1.Workspace, 0) - - if iam.RulesMatchesRequired(rules, rbacv1.PolicyRule{Verbs: []string{"list"}, APIGroups: []string{"tenant.kubesphere.io"}, Resources: []string{"workspaces"}}) { - workspaces, err = w.ksInformers.Tenant().V1alpha1().Workspaces().Lister().List(labels.Everything()) - if err != nil { - return nil, err - } - } else { - workspaceRoles, err := w.am.GetWorkspaceRoleMap(username) - if err != nil { - return nil, err - } - for k := range workspaceRoles { - workspace, err := w.ksInformers.Tenant().V1alpha1().Workspaces().Lister().Get(k) - if err != nil { - return nil, err - } - workspaces = append(workspaces, workspace) - } - } - - result := make([]*v1alpha1.Workspace, 0) - - for _, workspace := range workspaces { - if w.match(conditions.Match, workspace) && w.fuzzy(conditions.Fuzzy, workspace) { - result = append(result, workspace) - } - } - - // order & reverse - sort.Slice(result, func(i, j int) bool { - if reverse { - i, j = j, i - } - return w.compare(result[i], result[j], orderBy) - }) - - return result, nil + panic("implement me") } func (w *workspaceOperator) GetWorkspace(workspaceName string) (*v1alpha1.Workspace, error) { diff --git a/pkg/simple/client/ldap/interface.go b/pkg/simple/client/ldap/interface.go index 5b3e3e912..1f780d012 100644 --- a/pkg/simple/client/ldap/interface.go +++ b/pkg/simple/client/ldap/interface.go @@ -16,6 +16,6 @@ type Interface interface { // Get gets a user by its username from ldap, return ErrUserNotExists if user not exists Get(name string) (*iam.User, error) - // Verify checks if (name, password) is valid, return ErrInvalidCredentials if not - Verify(name string, password string) error + // Authenticate checks if (name, password) is valid, return ErrInvalidCredentials if not + Authenticate(name string, password string) error } diff --git a/pkg/simple/client/ldap/ldap.go b/pkg/simple/client/ldap/ldap.go index 025887c1d..bb88aced7 100644 --- a/pkg/simple/client/ldap/ldap.go +++ b/pkg/simple/client/ldap/ldap.go @@ -216,7 +216,7 @@ func (l *ldapInterfaceImpl) Get(name string) (*iam.User, error) { userEntry := searchResults.Entries[0] user := &iam.User{ - Username: userEntry.GetAttributeValue(ldapAttributeUserID), + Name: userEntry.GetAttributeValue(ldapAttributeUserID), Email: userEntry.GetAttributeValue(ldapAttributeMail), Lang: userEntry.GetAttributeValue(ldapAttributePreferredLanguage), Description: userEntry.GetAttributeValue(ldapAttributeDescription), @@ -229,12 +229,12 @@ func (l *ldapInterfaceImpl) Get(name string) (*iam.User, error) { } func (l *ldapInterfaceImpl) Create(user *iam.User) error { - if _, err := l.Get(user.Username); err != nil { + if _, err := l.Get(user.Name); err != nil { return ErrUserAlreadyExisted } createRequest := &ldap.AddRequest{ - DN: l.dnForUsername(user.Username), + DN: l.dnForUsername(user.Name), Attributes: []ldap.Attribute{ { Type: ldapAttributeObjectClass, @@ -242,7 +242,7 @@ func (l *ldapInterfaceImpl) Create(user *iam.User) error { }, { Type: ldapAttributeCommonName, - Vals: []string{user.Username}, + Vals: []string{user.Name}, }, { Type: ldapAttributeSerialNumber, @@ -254,11 +254,11 @@ func (l *ldapInterfaceImpl) Create(user *iam.User) error { }, { Type: ldapAttributeHomeDirectory, - Vals: []string{"/home/" + user.Username}, + Vals: []string{"/home/" + user.Name}, }, { Type: ldapAttributeUserID, - Vals: []string{user.Username}, + Vals: []string{user.Name}, }, { Type: ldapAttributeUserIDNumber, @@ -322,13 +322,13 @@ func (l *ldapInterfaceImpl) Update(newUser *iam.User) error { defer conn.Close() // check user existed - _, err = l.Get(newUser.Username) + _, err = l.Get(newUser.Name) if err != nil { return err } modifyRequest := &ldap.ModifyRequest{ - DN: l.dnForUsername(newUser.Username), + DN: l.dnForUsername(newUser.Name), } if newUser.Description != "" { @@ -347,7 +347,7 @@ func (l *ldapInterfaceImpl) Update(newUser *iam.User) error { } -func (l *ldapInterfaceImpl) Verify(username, password string) error { +func (l *ldapInterfaceImpl) Authenticate(username, password string) error { conn, err := l.newConn() if err != nil { return err diff --git a/pkg/simple/client/ldap/simple_ldap.go b/pkg/simple/client/ldap/simple_ldap.go index 78a82b4c6..2dc8fe8bf 100644 --- a/pkg/simple/client/ldap/simple_ldap.go +++ b/pkg/simple/client/ldap/simple_ldap.go @@ -17,7 +17,7 @@ func NewSimpleLdap() Interface { // initialize with a admin user admin := &iam.User{ - Username: "admin", + Name: "admin", Email: "admin@kubesphere.io", Lang: "eng", Description: "administrator", @@ -25,21 +25,21 @@ func NewSimpleLdap() Interface { Groups: nil, Password: "P@88w0rd", } - sl.store[admin.Username] = admin + sl.store[admin.Name] = admin return sl } func (s simpleLdap) Create(user *iam.User) error { - s.store[user.Username] = user + s.store[user.Name] = user return nil } func (s simpleLdap) Update(user *iam.User) error { - _, err := s.Get(user.Username) + _, err := s.Get(user.Name) if err != nil { return err } - s.store[user.Username] = user + s.store[user.Name] = user return nil } @@ -60,7 +60,7 @@ func (s simpleLdap) Get(name string) (*iam.User, error) { } } -func (s simpleLdap) Verify(name string, password string) error { +func (s simpleLdap) Authenticate(name string, password string) error { if user, err := s.Get(name); err != nil { return err } else { diff --git a/pkg/simple/client/ldap/simple_ldap_test.go b/pkg/simple/client/ldap/simple_ldap_test.go index c50096b2c..34dd9fb31 100644 --- a/pkg/simple/client/ldap/simple_ldap_test.go +++ b/pkg/simple/client/ldap/simple_ldap_test.go @@ -11,7 +11,7 @@ func TestSimpleLdap(t *testing.T) { ldapClient := NewSimpleLdap() foo := &iam.User{ - Username: "jerry", + Name: "jerry", Email: "jerry@kubesphere.io", Lang: "en", Description: "Jerry is kind and gentle.", @@ -27,7 +27,7 @@ func TestSimpleLdap(t *testing.T) { } // check if user really created - user, err := ldapClient.Get(foo.Username) + user, err := ldapClient.Get(foo.Name) if err != nil { t.Fatal(err) } @@ -35,7 +35,7 @@ func TestSimpleLdap(t *testing.T) { t.Fatalf("%T differ (-got, +want): %s", user, diff) } - _ = ldapClient.Delete(foo.Username) + _ = ldapClient.Delete(foo.Name) }) t.Run("should update user", func(t *testing.T) { @@ -51,7 +51,7 @@ func TestSimpleLdap(t *testing.T) { } // check if user really created - user, err := ldapClient.Get(foo.Username) + user, err := ldapClient.Get(foo.Name) if err != nil { t.Fatal(err) } @@ -59,7 +59,7 @@ func TestSimpleLdap(t *testing.T) { t.Fatalf("%T differ (-got, +want): %s", user, diff) } - _ = ldapClient.Delete(foo.Username) + _ = ldapClient.Delete(foo.Name) }) t.Run("should delete user", func(t *testing.T) { @@ -68,12 +68,12 @@ func TestSimpleLdap(t *testing.T) { t.Fatal(err) } - err = ldapClient.Delete(foo.Username) + err = ldapClient.Delete(foo.Name) if err != nil { t.Fatal(err) } - _, err = ldapClient.Get(foo.Username) + _, err = ldapClient.Get(foo.Name) if err == nil || err != ErrUserNotExists { t.Fatalf("expected ErrUserNotExists error, got %v", err) } @@ -85,12 +85,12 @@ func TestSimpleLdap(t *testing.T) { t.Fatal(err) } - err = ldapClient.Verify(foo.Username, foo.Password) + err = ldapClient.Authenticate(foo.Name, foo.Password) if err != nil { t.Fatalf("should pass but got an error %v", err) } - err = ldapClient.Verify(foo.Username, "gibberish") + err = ldapClient.Authenticate(foo.Name, "gibberish") if err == nil || err != ErrInvalidCredentials { t.Fatalf("expected error ErrInvalidCrenentials but got %v", err) } diff --git a/vendor/github.com/OneOfOne/xxhash/.gitignore b/vendor/github.com/OneOfOne/xxhash/.gitignore new file mode 100644 index 000000000..f4faa7f8f --- /dev/null +++ b/vendor/github.com/OneOfOne/xxhash/.gitignore @@ -0,0 +1,4 @@ +*.txt +*.pprof +cmap2/ +cache/ diff --git a/vendor/github.com/OneOfOne/xxhash/.travis.yml b/vendor/github.com/OneOfOne/xxhash/.travis.yml new file mode 100644 index 000000000..9b5bb481e --- /dev/null +++ b/vendor/github.com/OneOfOne/xxhash/.travis.yml @@ -0,0 +1,12 @@ +language: go +sudo: false + +go: + - 1.8 + - 1.9 + - "1.10" + - master + +script: + - go test -tags safe ./... + - go test ./... diff --git a/vendor/github.com/OneOfOne/xxhash/LICENSE b/vendor/github.com/OneOfOne/xxhash/LICENSE new file mode 100644 index 000000000..9e30b4f34 --- /dev/null +++ b/vendor/github.com/OneOfOne/xxhash/LICENSE @@ -0,0 +1,187 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. diff --git a/vendor/github.com/OneOfOne/xxhash/README.md b/vendor/github.com/OneOfOne/xxhash/README.md new file mode 100644 index 000000000..23174eb56 --- /dev/null +++ b/vendor/github.com/OneOfOne/xxhash/README.md @@ -0,0 +1,75 @@ +# xxhash [![GoDoc](https://godoc.org/github.com/OneOfOne/xxhash?status.svg)](https://godoc.org/github.com/OneOfOne/xxhash) [![Build Status](https://travis-ci.org/OneOfOne/xxhash.svg?branch=master)](https://travis-ci.org/OneOfOne/xxhash) [![Coverage](https://gocover.io/_badge/github.com/OneOfOne/xxhash)](https://gocover.io/github.com/OneOfOne/xxhash) + +This is a native Go implementation of the excellent [xxhash](https://github.com/Cyan4973/xxHash)* algorithm, an extremely fast non-cryptographic Hash algorithm, working at speeds close to RAM limits. + +* The C implementation is ([Copyright](https://github.com/Cyan4973/xxHash/blob/master/LICENSE) (c) 2012-2014, Yann Collet) + +## Install + + go get github.com/OneOfOne/xxhash + +## Features + +* On Go 1.7+ the pure go version is faster than CGO for all inputs. +* Supports ChecksumString{32,64} xxhash{32,64}.WriteString, which uses no copies when it can, falls back to copy on appengine. +* The native version falls back to a less optimized version on appengine due to the lack of unsafe. +* Almost as fast as the mostly pure assembly version written by the brilliant [cespare](https://github.com/cespare/xxhash), while also supporting seeds. +* To manually toggle the appengine version build with `-tags safe`. + +## Benchmark + +### Core i7-4790 @ 3.60GHz, Linux 4.12.6-1-ARCH (64bit), Go tip (+ff90f4af66 2017-08-19) + +```bash +➤ go test -bench '64' -count 5 -tags cespare | benchstat /dev/stdin +name time/op + +# https://github.com/cespare/xxhash +XXSum64Cespare/Func-8 160ns ± 2% +XXSum64Cespare/Struct-8 173ns ± 1% +XXSum64ShortCespare/Func-8 6.78ns ± 1% +XXSum64ShortCespare/Struct-8 19.6ns ± 2% + +# this package (default mode, using unsafe) +XXSum64/Func-8 170ns ± 1% +XXSum64/Struct-8 182ns ± 1% +XXSum64Short/Func-8 13.5ns ± 3% +XXSum64Short/Struct-8 20.4ns ± 0% + +# this package (appengine, *not* using unsafe) +XXSum64/Func-8 241ns ± 5% +XXSum64/Struct-8 243ns ± 6% +XXSum64Short/Func-8 15.2ns ± 2% +XXSum64Short/Struct-8 23.7ns ± 5% + +CRC64ISO-8 1.23µs ± 1% +CRC64ISOString-8 2.71µs ± 4% +CRC64ISOShort-8 22.2ns ± 3% + +Fnv64-8 2.34µs ± 1% +Fnv64Short-8 74.7ns ± 8% +# +``` + +## Usage + +```go + h := xxhash.New64() + // r, err := os.Open("......") + // defer f.Close() + r := strings.NewReader(F) + io.Copy(h, r) + fmt.Println("xxhash.Backend:", xxhash.Backend) + fmt.Println("File checksum:", h.Sum64()) +``` + +[playground](http://play.golang.org/p/rhRN3RdQyd) + +## TODO + +* Rewrite the 32bit version to be more optimized. +* General cleanup as the Go inliner gets smarter. + +## License + +This project is released under the Apache v2. licence. See [LICENCE](LICENCE) for more details. diff --git a/vendor/github.com/OneOfOne/xxhash/go.mod b/vendor/github.com/OneOfOne/xxhash/go.mod new file mode 100644 index 000000000..2f3334267 --- /dev/null +++ b/vendor/github.com/OneOfOne/xxhash/go.mod @@ -0,0 +1 @@ +module github.com/OneOfOne/xxhash diff --git a/vendor/github.com/OneOfOne/xxhash/xxhash.go b/vendor/github.com/OneOfOne/xxhash/xxhash.go new file mode 100644 index 000000000..2387d6593 --- /dev/null +++ b/vendor/github.com/OneOfOne/xxhash/xxhash.go @@ -0,0 +1,189 @@ +package xxhash + +const ( + prime32x1 uint32 = 2654435761 + prime32x2 uint32 = 2246822519 + prime32x3 uint32 = 3266489917 + prime32x4 uint32 = 668265263 + prime32x5 uint32 = 374761393 + + prime64x1 uint64 = 11400714785074694791 + prime64x2 uint64 = 14029467366897019727 + prime64x3 uint64 = 1609587929392839161 + prime64x4 uint64 = 9650029242287828579 + prime64x5 uint64 = 2870177450012600261 + + maxInt32 int32 = (1<<31 - 1) + + // precomputed zero Vs for seed 0 + zero64x1 = 0x60ea27eeadc0b5d6 + zero64x2 = 0xc2b2ae3d27d4eb4f + zero64x3 = 0x0 + zero64x4 = 0x61c8864e7a143579 +) + +// Checksum32 returns the checksum of the input data with the seed set to 0. +func Checksum32(in []byte) uint32 { + return Checksum32S(in, 0) +} + +// ChecksumString32 returns the checksum of the input data, without creating a copy, with the seed set to 0. +func ChecksumString32(s string) uint32 { + return ChecksumString32S(s, 0) +} + +type XXHash32 struct { + mem [16]byte + ln, memIdx int32 + v1, v2, v3, v4 uint32 + seed uint32 +} + +// Size returns the number of bytes Sum will return. +func (xx *XXHash32) Size() int { + return 4 +} + +// BlockSize returns the hash's underlying block size. +// The Write method must be able to accept any amount +// of data, but it may operate more efficiently if all writes +// are a multiple of the block size. +func (xx *XXHash32) BlockSize() int { + return 16 +} + +// NewS32 creates a new hash.Hash32 computing the 32bit xxHash checksum starting with the specific seed. +func NewS32(seed uint32) (xx *XXHash32) { + xx = &XXHash32{ + seed: seed, + } + xx.Reset() + return +} + +// New32 creates a new hash.Hash32 computing the 32bit xxHash checksum starting with the seed set to 0. +func New32() *XXHash32 { + return NewS32(0) +} + +func (xx *XXHash32) Reset() { + xx.v1 = xx.seed + prime32x1 + prime32x2 + xx.v2 = xx.seed + prime32x2 + xx.v3 = xx.seed + xx.v4 = xx.seed - prime32x1 + xx.ln, xx.memIdx = 0, 0 +} + +// Sum appends the current hash to b and returns the resulting slice. +// It does not change the underlying hash state. +func (xx *XXHash32) Sum(in []byte) []byte { + s := xx.Sum32() + return append(in, byte(s>>24), byte(s>>16), byte(s>>8), byte(s)) +} + +// Checksum64 an alias for Checksum64S(in, 0) +func Checksum64(in []byte) uint64 { + return Checksum64S(in, 0) +} + +// ChecksumString64 returns the checksum of the input data, without creating a copy, with the seed set to 0. +func ChecksumString64(s string) uint64 { + return ChecksumString64S(s, 0) +} + +type XXHash64 struct { + v1, v2, v3, v4 uint64 + seed uint64 + ln uint64 + mem [32]byte + memIdx int8 +} + +// Size returns the number of bytes Sum will return. +func (xx *XXHash64) Size() int { + return 8 +} + +// BlockSize returns the hash's underlying block size. +// The Write method must be able to accept any amount +// of data, but it may operate more efficiently if all writes +// are a multiple of the block size. +func (xx *XXHash64) BlockSize() int { + return 32 +} + +// NewS64 creates a new hash.Hash64 computing the 64bit xxHash checksum starting with the specific seed. +func NewS64(seed uint64) (xx *XXHash64) { + xx = &XXHash64{ + seed: seed, + } + xx.Reset() + return +} + +// New64 creates a new hash.Hash64 computing the 64bit xxHash checksum starting with the seed set to 0x0. +func New64() *XXHash64 { + return NewS64(0) +} + +func (xx *XXHash64) Reset() { + xx.ln, xx.memIdx = 0, 0 + xx.v1, xx.v2, xx.v3, xx.v4 = resetVs64(xx.seed) +} + +// Sum appends the current hash to b and returns the resulting slice. +// It does not change the underlying hash state. +func (xx *XXHash64) Sum(in []byte) []byte { + s := xx.Sum64() + return append(in, byte(s>>56), byte(s>>48), byte(s>>40), byte(s>>32), byte(s>>24), byte(s>>16), byte(s>>8), byte(s)) +} + +// force the compiler to use ROTL instructions + +func rotl32_1(x uint32) uint32 { return (x << 1) | (x >> (32 - 1)) } +func rotl32_7(x uint32) uint32 { return (x << 7) | (x >> (32 - 7)) } +func rotl32_11(x uint32) uint32 { return (x << 11) | (x >> (32 - 11)) } +func rotl32_12(x uint32) uint32 { return (x << 12) | (x >> (32 - 12)) } +func rotl32_13(x uint32) uint32 { return (x << 13) | (x >> (32 - 13)) } +func rotl32_17(x uint32) uint32 { return (x << 17) | (x >> (32 - 17)) } +func rotl32_18(x uint32) uint32 { return (x << 18) | (x >> (32 - 18)) } + +func rotl64_1(x uint64) uint64 { return (x << 1) | (x >> (64 - 1)) } +func rotl64_7(x uint64) uint64 { return (x << 7) | (x >> (64 - 7)) } +func rotl64_11(x uint64) uint64 { return (x << 11) | (x >> (64 - 11)) } +func rotl64_12(x uint64) uint64 { return (x << 12) | (x >> (64 - 12)) } +func rotl64_18(x uint64) uint64 { return (x << 18) | (x >> (64 - 18)) } +func rotl64_23(x uint64) uint64 { return (x << 23) | (x >> (64 - 23)) } +func rotl64_27(x uint64) uint64 { return (x << 27) | (x >> (64 - 27)) } +func rotl64_31(x uint64) uint64 { return (x << 31) | (x >> (64 - 31)) } + +func mix64(h uint64) uint64 { + h ^= h >> 33 + h *= prime64x2 + h ^= h >> 29 + h *= prime64x3 + h ^= h >> 32 + return h +} + +func resetVs64(seed uint64) (v1, v2, v3, v4 uint64) { + if seed == 0 { + return zero64x1, zero64x2, zero64x3, zero64x4 + } + return (seed + prime64x1 + prime64x2), (seed + prime64x2), (seed), (seed - prime64x1) +} + +// borrowed from cespare +func round64(h, v uint64) uint64 { + h += v * prime64x2 + h = rotl64_31(h) + h *= prime64x1 + return h +} + +func mergeRound64(h, v uint64) uint64 { + v = round64(0, v) + h ^= v + h = h*prime64x1 + prime64x4 + return h +} diff --git a/vendor/github.com/OneOfOne/xxhash/xxhash_go17.go b/vendor/github.com/OneOfOne/xxhash/xxhash_go17.go new file mode 100644 index 000000000..ae48e0c5c --- /dev/null +++ b/vendor/github.com/OneOfOne/xxhash/xxhash_go17.go @@ -0,0 +1,161 @@ +package xxhash + +func u32(in []byte) uint32 { + return uint32(in[0]) | uint32(in[1])<<8 | uint32(in[2])<<16 | uint32(in[3])<<24 +} + +func u64(in []byte) uint64 { + return uint64(in[0]) | uint64(in[1])<<8 | uint64(in[2])<<16 | uint64(in[3])<<24 | uint64(in[4])<<32 | uint64(in[5])<<40 | uint64(in[6])<<48 | uint64(in[7])<<56 +} + +// Checksum32S returns the checksum of the input bytes with the specific seed. +func Checksum32S(in []byte, seed uint32) (h uint32) { + var i int + + if len(in) > 15 { + var ( + v1 = seed + prime32x1 + prime32x2 + v2 = seed + prime32x2 + v3 = seed + 0 + v4 = seed - prime32x1 + ) + for ; i < len(in)-15; i += 16 { + in := in[i : i+16 : len(in)] + v1 += u32(in[0:4:len(in)]) * prime32x2 + v1 = rotl32_13(v1) * prime32x1 + + v2 += u32(in[4:8:len(in)]) * prime32x2 + v2 = rotl32_13(v2) * prime32x1 + + v3 += u32(in[8:12:len(in)]) * prime32x2 + v3 = rotl32_13(v3) * prime32x1 + + v4 += u32(in[12:16:len(in)]) * prime32x2 + v4 = rotl32_13(v4) * prime32x1 + } + + h = rotl32_1(v1) + rotl32_7(v2) + rotl32_12(v3) + rotl32_18(v4) + + } else { + h = seed + prime32x5 + } + + h += uint32(len(in)) + for ; i <= len(in)-4; i += 4 { + in := in[i : i+4 : len(in)] + h += u32(in[0:4:len(in)]) * prime32x3 + h = rotl32_17(h) * prime32x4 + } + + for ; i < len(in); i++ { + h += uint32(in[i]) * prime32x5 + h = rotl32_11(h) * prime32x1 + } + + h ^= h >> 15 + h *= prime32x2 + h ^= h >> 13 + h *= prime32x3 + h ^= h >> 16 + + return +} + +func (xx *XXHash32) Write(in []byte) (n int, err error) { + i, ml := 0, int(xx.memIdx) + n = len(in) + xx.ln += int32(n) + + if d := 16 - ml; ml > 0 && ml+len(in) > 16 { + xx.memIdx += int32(copy(xx.mem[xx.memIdx:], in[:d])) + ml, in = 16, in[d:len(in):len(in)] + } else if ml+len(in) < 16 { + xx.memIdx += int32(copy(xx.mem[xx.memIdx:], in)) + return + } + + if ml > 0 { + i += 16 - ml + xx.memIdx += int32(copy(xx.mem[xx.memIdx:len(xx.mem):len(xx.mem)], in)) + in := xx.mem[:16:len(xx.mem)] + + xx.v1 += u32(in[0:4:len(in)]) * prime32x2 + xx.v1 = rotl32_13(xx.v1) * prime32x1 + + xx.v2 += u32(in[4:8:len(in)]) * prime32x2 + xx.v2 = rotl32_13(xx.v2) * prime32x1 + + xx.v3 += u32(in[8:12:len(in)]) * prime32x2 + xx.v3 = rotl32_13(xx.v3) * prime32x1 + + xx.v4 += u32(in[12:16:len(in)]) * prime32x2 + xx.v4 = rotl32_13(xx.v4) * prime32x1 + + xx.memIdx = 0 + } + + for ; i <= len(in)-16; i += 16 { + in := in[i : i+16 : len(in)] + xx.v1 += u32(in[0:4:len(in)]) * prime32x2 + xx.v1 = rotl32_13(xx.v1) * prime32x1 + + xx.v2 += u32(in[4:8:len(in)]) * prime32x2 + xx.v2 = rotl32_13(xx.v2) * prime32x1 + + xx.v3 += u32(in[8:12:len(in)]) * prime32x2 + xx.v3 = rotl32_13(xx.v3) * prime32x1 + + xx.v4 += u32(in[12:16:len(in)]) * prime32x2 + xx.v4 = rotl32_13(xx.v4) * prime32x1 + } + + if len(in)-i != 0 { + xx.memIdx += int32(copy(xx.mem[xx.memIdx:], in[i:len(in):len(in)])) + } + + return +} + +func (xx *XXHash32) Sum32() (h uint32) { + var i int32 + if xx.ln > 15 { + h = rotl32_1(xx.v1) + rotl32_7(xx.v2) + rotl32_12(xx.v3) + rotl32_18(xx.v4) + } else { + h = xx.seed + prime32x5 + } + + h += uint32(xx.ln) + + if xx.memIdx > 0 { + for ; i < xx.memIdx-3; i += 4 { + in := xx.mem[i : i+4 : len(xx.mem)] + h += u32(in[0:4:len(in)]) * prime32x3 + h = rotl32_17(h) * prime32x4 + } + + for ; i < xx.memIdx; i++ { + h += uint32(xx.mem[i]) * prime32x5 + h = rotl32_11(h) * prime32x1 + } + } + h ^= h >> 15 + h *= prime32x2 + h ^= h >> 13 + h *= prime32x3 + h ^= h >> 16 + + return +} + +// Checksum64S returns the 64bit xxhash checksum for a single input +func Checksum64S(in []byte, seed uint64) uint64 { + if len(in) == 0 && seed == 0 { + return 0xef46db3751d8e999 + } + + if len(in) > 31 { + return checksum64(in, seed) + } + + return checksum64Short(in, seed) +} diff --git a/vendor/github.com/OneOfOne/xxhash/xxhash_safe.go b/vendor/github.com/OneOfOne/xxhash/xxhash_safe.go new file mode 100644 index 000000000..7532c2d31 --- /dev/null +++ b/vendor/github.com/OneOfOne/xxhash/xxhash_safe.go @@ -0,0 +1,183 @@ +// +build appengine safe ppc64le ppc64be mipsle mipsbe + +package xxhash + +// Backend returns the current version of xxhash being used. +const Backend = "GoSafe" + +func ChecksumString32S(s string, seed uint32) uint32 { + return Checksum32S([]byte(s), seed) +} + +func (xx *XXHash32) WriteString(s string) (int, error) { + if len(s) == 0 { + return 0, nil + } + return xx.Write([]byte(s)) +} + +func ChecksumString64S(s string, seed uint64) uint64 { + return Checksum64S([]byte(s), seed) +} + +func (xx *XXHash64) WriteString(s string) (int, error) { + if len(s) == 0 { + return 0, nil + } + return xx.Write([]byte(s)) +} + +func checksum64(in []byte, seed uint64) (h uint64) { + var ( + v1, v2, v3, v4 = resetVs64(seed) + + i int + ) + + for ; i < len(in)-31; i += 32 { + in := in[i : i+32 : len(in)] + v1 = round64(v1, u64(in[0:8:len(in)])) + v2 = round64(v2, u64(in[8:16:len(in)])) + v3 = round64(v3, u64(in[16:24:len(in)])) + v4 = round64(v4, u64(in[24:32:len(in)])) + } + + h = rotl64_1(v1) + rotl64_7(v2) + rotl64_12(v3) + rotl64_18(v4) + + h = mergeRound64(h, v1) + h = mergeRound64(h, v2) + h = mergeRound64(h, v3) + h = mergeRound64(h, v4) + + h += uint64(len(in)) + + for ; i < len(in)-7; i += 8 { + h ^= round64(0, u64(in[i:len(in):len(in)])) + h = rotl64_27(h)*prime64x1 + prime64x4 + } + + for ; i < len(in)-3; i += 4 { + h ^= uint64(u32(in[i:len(in):len(in)])) * prime64x1 + h = rotl64_23(h)*prime64x2 + prime64x3 + } + + for ; i < len(in); i++ { + h ^= uint64(in[i]) * prime64x5 + h = rotl64_11(h) * prime64x1 + } + + return mix64(h) +} + +func checksum64Short(in []byte, seed uint64) uint64 { + var ( + h = seed + prime64x5 + uint64(len(in)) + i int + ) + + for ; i < len(in)-7; i += 8 { + k := u64(in[i : i+8 : len(in)]) + h ^= round64(0, k) + h = rotl64_27(h)*prime64x1 + prime64x4 + } + + for ; i < len(in)-3; i += 4 { + h ^= uint64(u32(in[i:i+4:len(in)])) * prime64x1 + h = rotl64_23(h)*prime64x2 + prime64x3 + } + + for ; i < len(in); i++ { + h ^= uint64(in[i]) * prime64x5 + h = rotl64_11(h) * prime64x1 + } + + return mix64(h) +} + +func (xx *XXHash64) Write(in []byte) (n int, err error) { + var ( + ml = int(xx.memIdx) + d = 32 - ml + ) + + n = len(in) + xx.ln += uint64(n) + + if ml+len(in) < 32 { + xx.memIdx += int8(copy(xx.mem[xx.memIdx:len(xx.mem):len(xx.mem)], in)) + return + } + + i, v1, v2, v3, v4 := 0, xx.v1, xx.v2, xx.v3, xx.v4 + if ml > 0 && ml+len(in) > 32 { + xx.memIdx += int8(copy(xx.mem[xx.memIdx:len(xx.mem):len(xx.mem)], in[:d:len(in)])) + in = in[d:len(in):len(in)] + + in := xx.mem[0:32:len(xx.mem)] + + v1 = round64(v1, u64(in[0:8:len(in)])) + v2 = round64(v2, u64(in[8:16:len(in)])) + v3 = round64(v3, u64(in[16:24:len(in)])) + v4 = round64(v4, u64(in[24:32:len(in)])) + + xx.memIdx = 0 + } + + for ; i < len(in)-31; i += 32 { + in := in[i : i+32 : len(in)] + v1 = round64(v1, u64(in[0:8:len(in)])) + v2 = round64(v2, u64(in[8:16:len(in)])) + v3 = round64(v3, u64(in[16:24:len(in)])) + v4 = round64(v4, u64(in[24:32:len(in)])) + } + + if len(in)-i != 0 { + xx.memIdx += int8(copy(xx.mem[xx.memIdx:], in[i:len(in):len(in)])) + } + + xx.v1, xx.v2, xx.v3, xx.v4 = v1, v2, v3, v4 + + return +} + +func (xx *XXHash64) Sum64() (h uint64) { + var i int + if xx.ln > 31 { + v1, v2, v3, v4 := xx.v1, xx.v2, xx.v3, xx.v4 + h = rotl64_1(v1) + rotl64_7(v2) + rotl64_12(v3) + rotl64_18(v4) + + h = mergeRound64(h, v1) + h = mergeRound64(h, v2) + h = mergeRound64(h, v3) + h = mergeRound64(h, v4) + } else { + h = xx.seed + prime64x5 + } + + h += uint64(xx.ln) + if xx.memIdx > 0 { + in := xx.mem[:xx.memIdx] + for ; i < int(xx.memIdx)-7; i += 8 { + in := in[i : i+8 : len(in)] + k := u64(in[0:8:len(in)]) + k *= prime64x2 + k = rotl64_31(k) + k *= prime64x1 + h ^= k + h = rotl64_27(h)*prime64x1 + prime64x4 + } + + for ; i < int(xx.memIdx)-3; i += 4 { + in := in[i : i+4 : len(in)] + h ^= uint64(u32(in[0:4:len(in)])) * prime64x1 + h = rotl64_23(h)*prime64x2 + prime64x3 + } + + for ; i < int(xx.memIdx); i++ { + h ^= uint64(in[i]) * prime64x5 + h = rotl64_11(h) * prime64x1 + } + } + + return mix64(h) +} diff --git a/vendor/github.com/OneOfOne/xxhash/xxhash_unsafe.go b/vendor/github.com/OneOfOne/xxhash/xxhash_unsafe.go new file mode 100644 index 000000000..caacdc8b5 --- /dev/null +++ b/vendor/github.com/OneOfOne/xxhash/xxhash_unsafe.go @@ -0,0 +1,238 @@ +// +build !safe +// +build !appengine +// +build !ppc64le +// +build !mipsle +// +build !ppc64be +// +build !mipsbe + +package xxhash + +import ( + "reflect" + "unsafe" +) + +// Backend returns the current version of xxhash being used. +const Backend = "GoUnsafe" + +// ChecksumString32S returns the checksum of the input data, without creating a copy, with the specific seed. +func ChecksumString32S(s string, seed uint32) uint32 { + if len(s) == 0 { + return Checksum32S(nil, seed) + } + ss := (*reflect.StringHeader)(unsafe.Pointer(&s)) + return Checksum32S((*[maxInt32]byte)(unsafe.Pointer(ss.Data))[:len(s):len(s)], seed) +} + +func (xx *XXHash32) WriteString(s string) (int, error) { + if len(s) == 0 { + return 0, nil + } + + ss := (*reflect.StringHeader)(unsafe.Pointer(&s)) + return xx.Write((*[maxInt32]byte)(unsafe.Pointer(ss.Data))[:len(s):len(s)]) +} + +// ChecksumString64S returns the checksum of the input data, without creating a copy, with the specific seed. +func ChecksumString64S(s string, seed uint64) uint64 { + if len(s) == 0 { + return Checksum64S(nil, seed) + } + + ss := (*reflect.StringHeader)(unsafe.Pointer(&s)) + return Checksum64S((*[maxInt32]byte)(unsafe.Pointer(ss.Data))[:len(s):len(s)], seed) +} + +func (xx *XXHash64) WriteString(s string) (int, error) { + if len(s) == 0 { + return 0, nil + } + ss := (*reflect.StringHeader)(unsafe.Pointer(&s)) + return xx.Write((*[maxInt32]byte)(unsafe.Pointer(ss.Data))[:len(s)]) +} + +func checksum64(in []byte, seed uint64) uint64 { + var ( + wordsLen = len(in) >> 3 + words = ((*[maxInt32 / 8]uint64)(unsafe.Pointer(&in[0])))[:wordsLen:wordsLen] + + h uint64 = prime64x5 + + v1, v2, v3, v4 = resetVs64(seed) + + i int + ) + + for ; i < len(words)-3; i += 4 { + words := (*[4]uint64)(unsafe.Pointer(&words[i])) + + v1 = round64(v1, words[0]) + v2 = round64(v2, words[1]) + v3 = round64(v3, words[2]) + v4 = round64(v4, words[3]) + } + + h = rotl64_1(v1) + rotl64_7(v2) + rotl64_12(v3) + rotl64_18(v4) + + h = mergeRound64(h, v1) + h = mergeRound64(h, v2) + h = mergeRound64(h, v3) + h = mergeRound64(h, v4) + + h += uint64(len(in)) + + for _, k := range words[i:] { + h ^= round64(0, k) + h = rotl64_27(h)*prime64x1 + prime64x4 + } + + if in = in[wordsLen<<3 : len(in) : len(in)]; len(in) > 3 { + words := (*[1]uint32)(unsafe.Pointer(&in[0])) + h ^= uint64(words[0]) * prime64x1 + h = rotl64_23(h)*prime64x2 + prime64x3 + + in = in[4:len(in):len(in)] + } + + for _, b := range in { + h ^= uint64(b) * prime64x5 + h = rotl64_11(h) * prime64x1 + } + + return mix64(h) +} + +func checksum64Short(in []byte, seed uint64) uint64 { + var ( + h = seed + prime64x5 + uint64(len(in)) + i int + ) + + if len(in) > 7 { + var ( + wordsLen = len(in) >> 3 + words = ((*[maxInt32 / 8]uint64)(unsafe.Pointer(&in[0])))[:wordsLen:wordsLen] + ) + + for i := range words { + h ^= round64(0, words[i]) + h = rotl64_27(h)*prime64x1 + prime64x4 + } + + i = wordsLen << 3 + } + + if in = in[i:len(in):len(in)]; len(in) > 3 { + words := (*[1]uint32)(unsafe.Pointer(&in[0])) + h ^= uint64(words[0]) * prime64x1 + h = rotl64_23(h)*prime64x2 + prime64x3 + + in = in[4:len(in):len(in)] + } + + for _, b := range in { + h ^= uint64(b) * prime64x5 + h = rotl64_11(h) * prime64x1 + } + + return mix64(h) +} + +func (xx *XXHash64) Write(in []byte) (n int, err error) { + mem, idx := xx.mem[:], int(xx.memIdx) + + xx.ln, n = xx.ln+uint64(len(in)), len(in) + + if idx+len(in) < 32 { + xx.memIdx += int8(copy(mem[idx:len(mem):len(mem)], in)) + return + } + + var ( + v1, v2, v3, v4 = xx.v1, xx.v2, xx.v3, xx.v4 + + i int + ) + + if d := 32 - int(idx); d > 0 && int(idx)+len(in) > 31 { + copy(mem[idx:len(mem):len(mem)], in[:len(in):len(in)]) + + words := (*[4]uint64)(unsafe.Pointer(&mem[0])) + + v1 = round64(v1, words[0]) + v2 = round64(v2, words[1]) + v3 = round64(v3, words[2]) + v4 = round64(v4, words[3]) + + if in, xx.memIdx = in[d:len(in):len(in)], 0; len(in) == 0 { + goto RET + } + } + + for ; i < len(in)-31; i += 32 { + words := (*[4]uint64)(unsafe.Pointer(&in[i])) + + v1 = round64(v1, words[0]) + v2 = round64(v2, words[1]) + v3 = round64(v3, words[2]) + v4 = round64(v4, words[3]) + } + + if len(in)-i != 0 { + xx.memIdx += int8(copy(mem[xx.memIdx:len(mem):len(mem)], in[i:len(in):len(in)])) + } + +RET: + xx.v1, xx.v2, xx.v3, xx.v4 = v1, v2, v3, v4 + + return +} + +func (xx *XXHash64) Sum64() (h uint64) { + if seed := xx.seed; xx.ln > 31 { + v1, v2, v3, v4 := xx.v1, xx.v2, xx.v3, xx.v4 + h = rotl64_1(v1) + rotl64_7(v2) + rotl64_12(v3) + rotl64_18(v4) + + h = mergeRound64(h, v1) + h = mergeRound64(h, v2) + h = mergeRound64(h, v3) + h = mergeRound64(h, v4) + } else if seed == 0 { + h = prime64x5 + } else { + h = seed + prime64x5 + } + + h += uint64(xx.ln) + + if xx.memIdx == 0 { + return mix64(h) + } + + var ( + in = xx.mem[:xx.memIdx:xx.memIdx] + wordsLen = len(in) >> 3 + words = ((*[maxInt32 / 8]uint64)(unsafe.Pointer(&in[0])))[:wordsLen:wordsLen] + ) + + for _, k := range words { + h ^= round64(0, k) + h = rotl64_27(h)*prime64x1 + prime64x4 + } + + if in = in[wordsLen<<3 : len(in) : len(in)]; len(in) > 3 { + words := (*[1]uint32)(unsafe.Pointer(&in[0])) + + h ^= uint64(words[0]) * prime64x1 + h = rotl64_23(h)*prime64x2 + prime64x3 + + in = in[4:len(in):len(in)] + } + + for _, b := range in { + h ^= uint64(b) * prime64x5 + h = rotl64_11(h) * prime64x1 + } + + return mix64(h) +} diff --git a/vendor/github.com/globalsign/mgo/bson/bson_corpus_spec_test_generator.go b/vendor/github.com/globalsign/mgo/bson/bson_corpus_spec_test_generator.go new file mode 100644 index 000000000..3525a004b --- /dev/null +++ b/vendor/github.com/globalsign/mgo/bson/bson_corpus_spec_test_generator.go @@ -0,0 +1,294 @@ +// +build ignore + +package main + +import ( + "bytes" + "fmt" + "go/format" + "html/template" + "io/ioutil" + "log" + "path/filepath" + "strings" + + "github.com/globalsign/mgo/internal/json" +) + +func main() { + log.SetFlags(0) + log.SetPrefix(name + ": ") + + var g Generator + + fmt.Fprintf(&g, "// Code generated by \"%s.go\"; DO NOT EDIT\n\n", name) + + src := g.generate() + + err := ioutil.WriteFile(fmt.Sprintf("%s.go", strings.TrimSuffix(name, "_generator")), src, 0644) + if err != nil { + log.Fatalf("writing output: %s", err) + } +} + +// Generator holds the state of the analysis. Primarily used to buffer +// the output for format.Source. +type Generator struct { + bytes.Buffer // Accumulated output. +} + +// format returns the gofmt-ed contents of the Generator's buffer. +func (g *Generator) format() []byte { + src, err := format.Source(g.Bytes()) + if err != nil { + // Should never happen, but can arise when developing this code. + // The user can compile the output to see the error. + log.Printf("warning: internal error: invalid Go generated: %s", err) + log.Printf("warning: compile the package to analyze the error") + return g.Bytes() + } + return src +} + +// EVERYTHING ABOVE IS CONSTANT BETWEEN THE GENERATORS + +const name = "bson_corpus_spec_test_generator" + +func (g *Generator) generate() []byte { + + testFiles, err := filepath.Glob("./specdata/specifications/source/bson-corpus/tests/*.json") + if err != nil { + log.Fatalf("error reading bson-corpus files: %s", err) + } + + tests, err := g.loadTests(testFiles) + if err != nil { + log.Fatalf("error loading tests: %s", err) + } + + tmpl, err := g.getTemplate() + if err != nil { + log.Fatalf("error loading template: %s", err) + } + + tmpl.Execute(&g.Buffer, tests) + + return g.format() +} + +func (g *Generator) loadTests(filenames []string) ([]*testDef, error) { + var tests []*testDef + for _, filename := range filenames { + test, err := g.loadTest(filename) + if err != nil { + return nil, err + } + + tests = append(tests, test) + } + + return tests, nil +} + +func (g *Generator) loadTest(filename string) (*testDef, error) { + content, err := ioutil.ReadFile(filename) + if err != nil { + return nil, err + } + + var testDef testDef + err = json.Unmarshal(content, &testDef) + if err != nil { + return nil, err + } + + names := make(map[string]struct{}) + + for i := len(testDef.Valid) - 1; i >= 0; i-- { + if testDef.BsonType == "0x05" && testDef.Valid[i].Description == "subtype 0x02" { + testDef.Valid = append(testDef.Valid[:i], testDef.Valid[i+1:]...) + continue + } + + name := cleanupFuncName(testDef.Description + "_" + testDef.Valid[i].Description) + nameIdx := name + j := 1 + for { + if _, ok := names[nameIdx]; !ok { + break + } + + nameIdx = fmt.Sprintf("%s_%d", name, j) + } + + names[nameIdx] = struct{}{} + + testDef.Valid[i].TestDef = &testDef + testDef.Valid[i].Name = nameIdx + testDef.Valid[i].StructTest = testDef.TestKey != "" && + (testDef.BsonType != "0x05" || strings.Contains(testDef.Valid[i].Description, "0x00")) && + !testDef.Deprecated + } + + for i := len(testDef.DecodeErrors) - 1; i >= 0; i-- { + if strings.Contains(testDef.DecodeErrors[i].Description, "UTF-8") { + testDef.DecodeErrors = append(testDef.DecodeErrors[:i], testDef.DecodeErrors[i+1:]...) + continue + } + + name := cleanupFuncName(testDef.Description + "_" + testDef.DecodeErrors[i].Description) + nameIdx := name + j := 1 + for { + if _, ok := names[nameIdx]; !ok { + break + } + + nameIdx = fmt.Sprintf("%s_%d", name, j) + } + names[nameIdx] = struct{}{} + + testDef.DecodeErrors[i].Name = nameIdx + } + + return &testDef, nil +} + +func (g *Generator) getTemplate() (*template.Template, error) { + content := `package bson_test + +import ( + "encoding/hex" + "time" + + . "gopkg.in/check.v1" + "github.com/globalsign/mgo/bson" +) + +func testValid(c *C, in []byte, expected []byte, result interface{}) { + err := bson.Unmarshal(in, result) + c.Assert(err, IsNil) + + out, err := bson.Marshal(result) + c.Assert(err, IsNil) + + c.Assert(string(expected), Equals, string(out), Commentf("roundtrip failed for %T, expected '%x' but got '%x'", result, expected, out)) +} + +func testDecodeSkip(c *C, in []byte) { + err := bson.Unmarshal(in, &struct{}{}) + c.Assert(err, IsNil) +} + +func testDecodeError(c *C, in []byte, result interface{}) { + err := bson.Unmarshal(in, result) + c.Assert(err, Not(IsNil)) +} + +{{range .}} +{{range .Valid}} +func (s *S) Test{{.Name}}(c *C) { + b, err := hex.DecodeString("{{.Bson}}") + c.Assert(err, IsNil) + + {{if .CanonicalBson}} + cb, err := hex.DecodeString("{{.CanonicalBson}}") + c.Assert(err, IsNil) + {{else}} + cb := b + {{end}} + + var resultD bson.D + testValid(c, b, cb, &resultD) + {{if .StructTest}}var resultS struct { + Element {{.TestDef.GoType}} ` + "`bson:\"{{.TestDef.TestKey}}\"`" + ` + } + testValid(c, b, cb, &resultS){{end}} + + testDecodeSkip(c, b) +} +{{end}} + +{{range .DecodeErrors}} +func (s *S) Test{{.Name}}(c *C) { + b, err := hex.DecodeString("{{.Bson}}") + c.Assert(err, IsNil) + + var resultD bson.D + testDecodeError(c, b, &resultD) +} +{{end}} +{{end}} +` + tmpl, err := template.New("").Parse(content) + if err != nil { + return nil, err + } + return tmpl, nil +} + +func cleanupFuncName(name string) string { + return strings.Map(func(r rune) rune { + if (r >= 48 && r <= 57) || (r >= 65 && r <= 90) || (r >= 97 && r <= 122) { + return r + } + return '_' + }, name) +} + +type testDef struct { + Description string `json:"description"` + BsonType string `json:"bson_type"` + TestKey string `json:"test_key"` + Valid []*valid `json:"valid"` + DecodeErrors []*decodeError `json:"decodeErrors"` + Deprecated bool `json:"deprecated"` +} + +func (t *testDef) GoType() string { + switch t.BsonType { + case "0x01": + return "float64" + case "0x02": + return "string" + case "0x03": + return "bson.D" + case "0x04": + return "[]interface{}" + case "0x05": + return "[]byte" + case "0x07": + return "bson.ObjectId" + case "0x08": + return "bool" + case "0x09": + return "time.Time" + case "0x0E": + return "string" + case "0x10": + return "int32" + case "0x12": + return "int64" + case "0x13": + return "bson.Decimal" + default: + return "interface{}" + } +} + +type valid struct { + Description string `json:"description"` + Bson string `json:"bson"` + CanonicalBson string `json:"canonical_bson"` + + Name string + StructTest bool + TestDef *testDef +} + +type decodeError struct { + Description string `json:"description"` + Bson string `json:"bson"` + + Name string +} diff --git a/vendor/github.com/gobwas/glob/.gitignore b/vendor/github.com/gobwas/glob/.gitignore new file mode 100644 index 000000000..b4ae623be --- /dev/null +++ b/vendor/github.com/gobwas/glob/.gitignore @@ -0,0 +1,8 @@ +glob.iml +.idea +*.cpu +*.mem +*.test +*.dot +*.png +*.svg diff --git a/vendor/github.com/gobwas/glob/.travis.yml b/vendor/github.com/gobwas/glob/.travis.yml new file mode 100644 index 000000000..e8a276826 --- /dev/null +++ b/vendor/github.com/gobwas/glob/.travis.yml @@ -0,0 +1,9 @@ +sudo: false + +language: go + +go: + - 1.5.3 + +script: + - go test -v ./... diff --git a/vendor/github.com/gobwas/glob/LICENSE b/vendor/github.com/gobwas/glob/LICENSE new file mode 100644 index 000000000..9d4735cad --- /dev/null +++ b/vendor/github.com/gobwas/glob/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2016 Sergey Kamardin + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/vendor/github.com/gobwas/glob/bench.sh b/vendor/github.com/gobwas/glob/bench.sh new file mode 100644 index 000000000..804cf22e6 --- /dev/null +++ b/vendor/github.com/gobwas/glob/bench.sh @@ -0,0 +1,26 @@ +#! /bin/bash + +bench() { + filename="/tmp/$1-$2.bench" + if test -e "${filename}"; + then + echo "Already exists ${filename}" + else + backup=`git rev-parse --abbrev-ref HEAD` + git checkout $1 + echo -n "Creating ${filename}... " + go test ./... -run=NONE -bench=$2 > "${filename}" -benchmem + echo "OK" + git checkout ${backup} + sleep 5 + fi +} + + +to=$1 +current=`git rev-parse --abbrev-ref HEAD` + +bench ${to} $2 +bench ${current} $2 + +benchcmp $3 "/tmp/${to}-$2.bench" "/tmp/${current}-$2.bench" diff --git a/vendor/github.com/gobwas/glob/compiler/compiler.go b/vendor/github.com/gobwas/glob/compiler/compiler.go new file mode 100644 index 000000000..02e7de80a --- /dev/null +++ b/vendor/github.com/gobwas/glob/compiler/compiler.go @@ -0,0 +1,525 @@ +package compiler + +// TODO use constructor with all matchers, and to their structs private +// TODO glue multiple Text nodes (like after QuoteMeta) + +import ( + "fmt" + "reflect" + + "github.com/gobwas/glob/match" + "github.com/gobwas/glob/syntax/ast" + "github.com/gobwas/glob/util/runes" +) + +func optimizeMatcher(matcher match.Matcher) match.Matcher { + switch m := matcher.(type) { + + case match.Any: + if len(m.Separators) == 0 { + return match.NewSuper() + } + + case match.AnyOf: + if len(m.Matchers) == 1 { + return m.Matchers[0] + } + + return m + + case match.List: + if m.Not == false && len(m.List) == 1 { + return match.NewText(string(m.List)) + } + + return m + + case match.BTree: + m.Left = optimizeMatcher(m.Left) + m.Right = optimizeMatcher(m.Right) + + r, ok := m.Value.(match.Text) + if !ok { + return m + } + + var ( + leftNil = m.Left == nil + rightNil = m.Right == nil + ) + if leftNil && rightNil { + return match.NewText(r.Str) + } + + _, leftSuper := m.Left.(match.Super) + lp, leftPrefix := m.Left.(match.Prefix) + la, leftAny := m.Left.(match.Any) + + _, rightSuper := m.Right.(match.Super) + rs, rightSuffix := m.Right.(match.Suffix) + ra, rightAny := m.Right.(match.Any) + + switch { + case leftSuper && rightSuper: + return match.NewContains(r.Str, false) + + case leftSuper && rightNil: + return match.NewSuffix(r.Str) + + case rightSuper && leftNil: + return match.NewPrefix(r.Str) + + case leftNil && rightSuffix: + return match.NewPrefixSuffix(r.Str, rs.Suffix) + + case rightNil && leftPrefix: + return match.NewPrefixSuffix(lp.Prefix, r.Str) + + case rightNil && leftAny: + return match.NewSuffixAny(r.Str, la.Separators) + + case leftNil && rightAny: + return match.NewPrefixAny(r.Str, ra.Separators) + } + + return m + } + + return matcher +} + +func compileMatchers(matchers []match.Matcher) (match.Matcher, error) { + if len(matchers) == 0 { + return nil, fmt.Errorf("compile error: need at least one matcher") + } + if len(matchers) == 1 { + return matchers[0], nil + } + if m := glueMatchers(matchers); m != nil { + return m, nil + } + + idx := -1 + maxLen := -1 + var val match.Matcher + for i, matcher := range matchers { + if l := matcher.Len(); l != -1 && l >= maxLen { + maxLen = l + idx = i + val = matcher + } + } + + if val == nil { // not found matcher with static length + r, err := compileMatchers(matchers[1:]) + if err != nil { + return nil, err + } + return match.NewBTree(matchers[0], nil, r), nil + } + + left := matchers[:idx] + var right []match.Matcher + if len(matchers) > idx+1 { + right = matchers[idx+1:] + } + + var l, r match.Matcher + var err error + if len(left) > 0 { + l, err = compileMatchers(left) + if err != nil { + return nil, err + } + } + + if len(right) > 0 { + r, err = compileMatchers(right) + if err != nil { + return nil, err + } + } + + return match.NewBTree(val, l, r), nil +} + +func glueMatchers(matchers []match.Matcher) match.Matcher { + if m := glueMatchersAsEvery(matchers); m != nil { + return m + } + if m := glueMatchersAsRow(matchers); m != nil { + return m + } + return nil +} + +func glueMatchersAsRow(matchers []match.Matcher) match.Matcher { + if len(matchers) <= 1 { + return nil + } + + var ( + c []match.Matcher + l int + ) + for _, matcher := range matchers { + if ml := matcher.Len(); ml == -1 { + return nil + } else { + c = append(c, matcher) + l += ml + } + } + return match.NewRow(l, c...) +} + +func glueMatchersAsEvery(matchers []match.Matcher) match.Matcher { + if len(matchers) <= 1 { + return nil + } + + var ( + hasAny bool + hasSuper bool + hasSingle bool + min int + separator []rune + ) + + for i, matcher := range matchers { + var sep []rune + + switch m := matcher.(type) { + case match.Super: + sep = []rune{} + hasSuper = true + + case match.Any: + sep = m.Separators + hasAny = true + + case match.Single: + sep = m.Separators + hasSingle = true + min++ + + case match.List: + if !m.Not { + return nil + } + sep = m.List + hasSingle = true + min++ + + default: + return nil + } + + // initialize + if i == 0 { + separator = sep + } + + if runes.Equal(sep, separator) { + continue + } + + return nil + } + + if hasSuper && !hasAny && !hasSingle { + return match.NewSuper() + } + + if hasAny && !hasSuper && !hasSingle { + return match.NewAny(separator) + } + + if (hasAny || hasSuper) && min > 0 && len(separator) == 0 { + return match.NewMin(min) + } + + every := match.NewEveryOf() + + if min > 0 { + every.Add(match.NewMin(min)) + + if !hasAny && !hasSuper { + every.Add(match.NewMax(min)) + } + } + + if len(separator) > 0 { + every.Add(match.NewContains(string(separator), true)) + } + + return every +} + +func minimizeMatchers(matchers []match.Matcher) []match.Matcher { + var done match.Matcher + var left, right, count int + + for l := 0; l < len(matchers); l++ { + for r := len(matchers); r > l; r-- { + if glued := glueMatchers(matchers[l:r]); glued != nil { + var swap bool + + if done == nil { + swap = true + } else { + cl, gl := done.Len(), glued.Len() + swap = cl > -1 && gl > -1 && gl > cl + swap = swap || count < r-l + } + + if swap { + done = glued + left = l + right = r + count = r - l + } + } + } + } + + if done == nil { + return matchers + } + + next := append(append([]match.Matcher{}, matchers[:left]...), done) + if right < len(matchers) { + next = append(next, matchers[right:]...) + } + + if len(next) == len(matchers) { + return next + } + + return minimizeMatchers(next) +} + +// minimizeAnyOf tries to apply some heuristics to minimize number of nodes in given tree +func minimizeTree(tree *ast.Node) *ast.Node { + switch tree.Kind { + case ast.KindAnyOf: + return minimizeTreeAnyOf(tree) + default: + return nil + } +} + +// minimizeAnyOf tries to find common children of given node of AnyOf pattern +// it searches for common children from left and from right +// if any common children are found – then it returns new optimized ast tree +// else it returns nil +func minimizeTreeAnyOf(tree *ast.Node) *ast.Node { + if !areOfSameKind(tree.Children, ast.KindPattern) { + return nil + } + + commonLeft, commonRight := commonChildren(tree.Children) + commonLeftCount, commonRightCount := len(commonLeft), len(commonRight) + if commonLeftCount == 0 && commonRightCount == 0 { // there are no common parts + return nil + } + + var result []*ast.Node + if commonLeftCount > 0 { + result = append(result, ast.NewNode(ast.KindPattern, nil, commonLeft...)) + } + + var anyOf []*ast.Node + for _, child := range tree.Children { + reuse := child.Children[commonLeftCount : len(child.Children)-commonRightCount] + var node *ast.Node + if len(reuse) == 0 { + // this pattern is completely reduced by commonLeft and commonRight patterns + // so it become nothing + node = ast.NewNode(ast.KindNothing, nil) + } else { + node = ast.NewNode(ast.KindPattern, nil, reuse...) + } + anyOf = appendIfUnique(anyOf, node) + } + switch { + case len(anyOf) == 1 && anyOf[0].Kind != ast.KindNothing: + result = append(result, anyOf[0]) + case len(anyOf) > 1: + result = append(result, ast.NewNode(ast.KindAnyOf, nil, anyOf...)) + } + + if commonRightCount > 0 { + result = append(result, ast.NewNode(ast.KindPattern, nil, commonRight...)) + } + + return ast.NewNode(ast.KindPattern, nil, result...) +} + +func commonChildren(nodes []*ast.Node) (commonLeft, commonRight []*ast.Node) { + if len(nodes) <= 1 { + return + } + + // find node that has least number of children + idx := leastChildren(nodes) + if idx == -1 { + return + } + tree := nodes[idx] + treeLength := len(tree.Children) + + // allocate max able size for rightCommon slice + // to get ability insert elements in reverse order (from end to start) + // without sorting + commonRight = make([]*ast.Node, treeLength) + lastRight := treeLength // will use this to get results as commonRight[lastRight:] + + var ( + breakLeft bool + breakRight bool + commonTotal int + ) + for i, j := 0, treeLength-1; commonTotal < treeLength && j >= 0 && !(breakLeft && breakRight); i, j = i+1, j-1 { + treeLeft := tree.Children[i] + treeRight := tree.Children[j] + + for k := 0; k < len(nodes) && !(breakLeft && breakRight); k++ { + // skip least children node + if k == idx { + continue + } + + restLeft := nodes[k].Children[i] + restRight := nodes[k].Children[j+len(nodes[k].Children)-treeLength] + + breakLeft = breakLeft || !treeLeft.Equal(restLeft) + + // disable searching for right common parts, if left part is already overlapping + breakRight = breakRight || (!breakLeft && j <= i) + breakRight = breakRight || !treeRight.Equal(restRight) + } + + if !breakLeft { + commonTotal++ + commonLeft = append(commonLeft, treeLeft) + } + if !breakRight { + commonTotal++ + lastRight = j + commonRight[j] = treeRight + } + } + + commonRight = commonRight[lastRight:] + + return +} + +func appendIfUnique(target []*ast.Node, val *ast.Node) []*ast.Node { + for _, n := range target { + if reflect.DeepEqual(n, val) { + return target + } + } + return append(target, val) +} + +func areOfSameKind(nodes []*ast.Node, kind ast.Kind) bool { + for _, n := range nodes { + if n.Kind != kind { + return false + } + } + return true +} + +func leastChildren(nodes []*ast.Node) int { + min := -1 + idx := -1 + for i, n := range nodes { + if idx == -1 || (len(n.Children) < min) { + min = len(n.Children) + idx = i + } + } + return idx +} + +func compileTreeChildren(tree *ast.Node, sep []rune) ([]match.Matcher, error) { + var matchers []match.Matcher + for _, desc := range tree.Children { + m, err := compile(desc, sep) + if err != nil { + return nil, err + } + matchers = append(matchers, optimizeMatcher(m)) + } + return matchers, nil +} + +func compile(tree *ast.Node, sep []rune) (m match.Matcher, err error) { + switch tree.Kind { + case ast.KindAnyOf: + // todo this could be faster on pattern_alternatives_combine_lite (see glob_test.go) + if n := minimizeTree(tree); n != nil { + return compile(n, sep) + } + matchers, err := compileTreeChildren(tree, sep) + if err != nil { + return nil, err + } + return match.NewAnyOf(matchers...), nil + + case ast.KindPattern: + if len(tree.Children) == 0 { + return match.NewNothing(), nil + } + matchers, err := compileTreeChildren(tree, sep) + if err != nil { + return nil, err + } + m, err = compileMatchers(minimizeMatchers(matchers)) + if err != nil { + return nil, err + } + + case ast.KindAny: + m = match.NewAny(sep) + + case ast.KindSuper: + m = match.NewSuper() + + case ast.KindSingle: + m = match.NewSingle(sep) + + case ast.KindNothing: + m = match.NewNothing() + + case ast.KindList: + l := tree.Value.(ast.List) + m = match.NewList([]rune(l.Chars), l.Not) + + case ast.KindRange: + r := tree.Value.(ast.Range) + m = match.NewRange(r.Lo, r.Hi, r.Not) + + case ast.KindText: + t := tree.Value.(ast.Text) + m = match.NewText(t.Text) + + default: + return nil, fmt.Errorf("could not compile tree: unknown node type") + } + + return optimizeMatcher(m), nil +} + +func Compile(tree *ast.Node, sep []rune) (match.Matcher, error) { + m, err := compile(tree, sep) + if err != nil { + return nil, err + } + + return m, nil +} diff --git a/vendor/github.com/gobwas/glob/glob.go b/vendor/github.com/gobwas/glob/glob.go new file mode 100644 index 000000000..2afde343a --- /dev/null +++ b/vendor/github.com/gobwas/glob/glob.go @@ -0,0 +1,80 @@ +package glob + +import ( + "github.com/gobwas/glob/compiler" + "github.com/gobwas/glob/syntax" +) + +// Glob represents compiled glob pattern. +type Glob interface { + Match(string) bool +} + +// Compile creates Glob for given pattern and strings (if any present after pattern) as separators. +// The pattern syntax is: +// +// pattern: +// { term } +// +// term: +// `*` matches any sequence of non-separator characters +// `**` matches any sequence of characters +// `?` matches any single non-separator character +// `[` [ `!` ] { character-range } `]` +// character class (must be non-empty) +// `{` pattern-list `}` +// pattern alternatives +// c matches character c (c != `*`, `**`, `?`, `\`, `[`, `{`, `}`) +// `\` c matches character c +// +// character-range: +// c matches character c (c != `\\`, `-`, `]`) +// `\` c matches character c +// lo `-` hi matches character c for lo <= c <= hi +// +// pattern-list: +// pattern { `,` pattern } +// comma-separated (without spaces) patterns +// +func Compile(pattern string, separators ...rune) (Glob, error) { + ast, err := syntax.Parse(pattern) + if err != nil { + return nil, err + } + + matcher, err := compiler.Compile(ast, separators) + if err != nil { + return nil, err + } + + return matcher, nil +} + +// MustCompile is the same as Compile, except that if Compile returns error, this will panic +func MustCompile(pattern string, separators ...rune) Glob { + g, err := Compile(pattern, separators...) + if err != nil { + panic(err) + } + + return g +} + +// QuoteMeta returns a string that quotes all glob pattern meta characters +// inside the argument text; For example, QuoteMeta(`{foo*}`) returns `\[foo\*\]`. +func QuoteMeta(s string) string { + b := make([]byte, 2*len(s)) + + // a byte loop is correct because all meta characters are ASCII + j := 0 + for i := 0; i < len(s); i++ { + if syntax.Special(s[i]) { + b[j] = '\\' + j++ + } + b[j] = s[i] + j++ + } + + return string(b[0:j]) +} diff --git a/vendor/github.com/gobwas/glob/match/any.go b/vendor/github.com/gobwas/glob/match/any.go new file mode 100644 index 000000000..514a9a5c4 --- /dev/null +++ b/vendor/github.com/gobwas/glob/match/any.go @@ -0,0 +1,45 @@ +package match + +import ( + "fmt" + "github.com/gobwas/glob/util/strings" +) + +type Any struct { + Separators []rune +} + +func NewAny(s []rune) Any { + return Any{s} +} + +func (self Any) Match(s string) bool { + return strings.IndexAnyRunes(s, self.Separators) == -1 +} + +func (self Any) Index(s string) (int, []int) { + found := strings.IndexAnyRunes(s, self.Separators) + switch found { + case -1: + case 0: + return 0, segments0 + default: + s = s[:found] + } + + segments := acquireSegments(len(s)) + for i := range s { + segments = append(segments, i) + } + segments = append(segments, len(s)) + + return 0, segments +} + +func (self Any) Len() int { + return lenNo +} + +func (self Any) String() string { + return fmt.Sprintf("", string(self.Separators)) +} diff --git a/vendor/github.com/gobwas/glob/match/any_of.go b/vendor/github.com/gobwas/glob/match/any_of.go new file mode 100644 index 000000000..8e65356cd --- /dev/null +++ b/vendor/github.com/gobwas/glob/match/any_of.go @@ -0,0 +1,82 @@ +package match + +import "fmt" + +type AnyOf struct { + Matchers Matchers +} + +func NewAnyOf(m ...Matcher) AnyOf { + return AnyOf{Matchers(m)} +} + +func (self *AnyOf) Add(m Matcher) error { + self.Matchers = append(self.Matchers, m) + return nil +} + +func (self AnyOf) Match(s string) bool { + for _, m := range self.Matchers { + if m.Match(s) { + return true + } + } + + return false +} + +func (self AnyOf) Index(s string) (int, []int) { + index := -1 + + segments := acquireSegments(len(s)) + for _, m := range self.Matchers { + idx, seg := m.Index(s) + if idx == -1 { + continue + } + + if index == -1 || idx < index { + index = idx + segments = append(segments[:0], seg...) + continue + } + + if idx > index { + continue + } + + // here idx == index + segments = appendMerge(segments, seg) + } + + if index == -1 { + releaseSegments(segments) + return -1, nil + } + + return index, segments +} + +func (self AnyOf) Len() (l int) { + l = -1 + for _, m := range self.Matchers { + ml := m.Len() + switch { + case l == -1: + l = ml + continue + + case ml == -1: + return -1 + + case l != ml: + return -1 + } + } + + return +} + +func (self AnyOf) String() string { + return fmt.Sprintf("", self.Matchers) +} diff --git a/vendor/github.com/gobwas/glob/match/btree.go b/vendor/github.com/gobwas/glob/match/btree.go new file mode 100644 index 000000000..a8130e93e --- /dev/null +++ b/vendor/github.com/gobwas/glob/match/btree.go @@ -0,0 +1,146 @@ +package match + +import ( + "fmt" + "unicode/utf8" +) + +type BTree struct { + Value Matcher + Left Matcher + Right Matcher + ValueLengthRunes int + LeftLengthRunes int + RightLengthRunes int + LengthRunes int +} + +func NewBTree(Value, Left, Right Matcher) (tree BTree) { + tree.Value = Value + tree.Left = Left + tree.Right = Right + + lenOk := true + if tree.ValueLengthRunes = Value.Len(); tree.ValueLengthRunes == -1 { + lenOk = false + } + + if Left != nil { + if tree.LeftLengthRunes = Left.Len(); tree.LeftLengthRunes == -1 { + lenOk = false + } + } + + if Right != nil { + if tree.RightLengthRunes = Right.Len(); tree.RightLengthRunes == -1 { + lenOk = false + } + } + + if lenOk { + tree.LengthRunes = tree.LeftLengthRunes + tree.ValueLengthRunes + tree.RightLengthRunes + } else { + tree.LengthRunes = -1 + } + + return tree +} + +func (self BTree) Len() int { + return self.LengthRunes +} + +// todo? +func (self BTree) Index(s string) (int, []int) { + return -1, nil +} + +func (self BTree) Match(s string) bool { + inputLen := len(s) + + // self.Length, self.RLen and self.LLen are values meaning the length of runes for each part + // here we manipulating byte length for better optimizations + // but these checks still works, cause minLen of 1-rune string is 1 byte. + if self.LengthRunes != -1 && self.LengthRunes > inputLen { + return false + } + + // try to cut unnecessary parts + // by knowledge of length of right and left part + var offset, limit int + if self.LeftLengthRunes >= 0 { + offset = self.LeftLengthRunes + } + if self.RightLengthRunes >= 0 { + limit = inputLen - self.RightLengthRunes + } else { + limit = inputLen + } + + for offset < limit { + // search for matching part in substring + index, segments := self.Value.Index(s[offset:limit]) + if index == -1 { + releaseSegments(segments) + return false + } + + l := s[:offset+index] + var left bool + if self.Left != nil { + left = self.Left.Match(l) + } else { + left = l == "" + } + + if left { + for i := len(segments) - 1; i >= 0; i-- { + length := segments[i] + + var right bool + var r string + // if there is no string for the right branch + if inputLen <= offset+index+length { + r = "" + } else { + r = s[offset+index+length:] + } + + if self.Right != nil { + right = self.Right.Match(r) + } else { + right = r == "" + } + + if right { + releaseSegments(segments) + return true + } + } + } + + _, step := utf8.DecodeRuneInString(s[offset+index:]) + offset += index + step + + releaseSegments(segments) + } + + return false +} + +func (self BTree) String() string { + const n string = "" + var l, r string + if self.Left == nil { + l = n + } else { + l = self.Left.String() + } + if self.Right == nil { + r = n + } else { + r = self.Right.String() + } + + return fmt.Sprintf("%s]>", l, self.Value, r) +} diff --git a/vendor/github.com/gobwas/glob/match/contains.go b/vendor/github.com/gobwas/glob/match/contains.go new file mode 100644 index 000000000..0998e95b0 --- /dev/null +++ b/vendor/github.com/gobwas/glob/match/contains.go @@ -0,0 +1,58 @@ +package match + +import ( + "fmt" + "strings" +) + +type Contains struct { + Needle string + Not bool +} + +func NewContains(needle string, not bool) Contains { + return Contains{needle, not} +} + +func (self Contains) Match(s string) bool { + return strings.Contains(s, self.Needle) != self.Not +} + +func (self Contains) Index(s string) (int, []int) { + var offset int + + idx := strings.Index(s, self.Needle) + + if !self.Not { + if idx == -1 { + return -1, nil + } + + offset = idx + len(self.Needle) + if len(s) <= offset { + return 0, []int{offset} + } + s = s[offset:] + } else if idx != -1 { + s = s[:idx] + } + + segments := acquireSegments(len(s) + 1) + for i := range s { + segments = append(segments, offset+i) + } + + return 0, append(segments, offset+len(s)) +} + +func (self Contains) Len() int { + return lenNo +} + +func (self Contains) String() string { + var not string + if self.Not { + not = "!" + } + return fmt.Sprintf("", not, self.Needle) +} diff --git a/vendor/github.com/gobwas/glob/match/every_of.go b/vendor/github.com/gobwas/glob/match/every_of.go new file mode 100644 index 000000000..7c968ee36 --- /dev/null +++ b/vendor/github.com/gobwas/glob/match/every_of.go @@ -0,0 +1,99 @@ +package match + +import ( + "fmt" +) + +type EveryOf struct { + Matchers Matchers +} + +func NewEveryOf(m ...Matcher) EveryOf { + return EveryOf{Matchers(m)} +} + +func (self *EveryOf) Add(m Matcher) error { + self.Matchers = append(self.Matchers, m) + return nil +} + +func (self EveryOf) Len() (l int) { + for _, m := range self.Matchers { + if ml := m.Len(); l > 0 { + l += ml + } else { + return -1 + } + } + + return +} + +func (self EveryOf) Index(s string) (int, []int) { + var index int + var offset int + + // make `in` with cap as len(s), + // cause it is the maximum size of output segments values + next := acquireSegments(len(s)) + current := acquireSegments(len(s)) + + sub := s + for i, m := range self.Matchers { + idx, seg := m.Index(sub) + if idx == -1 { + releaseSegments(next) + releaseSegments(current) + return -1, nil + } + + if i == 0 { + // we use copy here instead of `current = seg` + // cause seg is a slice from reusable buffer `in` + // and it could be overwritten in next iteration + current = append(current, seg...) + } else { + // clear the next + next = next[:0] + + delta := index - (idx + offset) + for _, ex := range current { + for _, n := range seg { + if ex+delta == n { + next = append(next, n) + } + } + } + + if len(next) == 0 { + releaseSegments(next) + releaseSegments(current) + return -1, nil + } + + current = append(current[:0], next...) + } + + index = idx + offset + sub = s[index:] + offset += idx + } + + releaseSegments(next) + + return index, current +} + +func (self EveryOf) Match(s string) bool { + for _, m := range self.Matchers { + if !m.Match(s) { + return false + } + } + + return true +} + +func (self EveryOf) String() string { + return fmt.Sprintf("", self.Matchers) +} diff --git a/vendor/github.com/gobwas/glob/match/list.go b/vendor/github.com/gobwas/glob/match/list.go new file mode 100644 index 000000000..7fd763ecd --- /dev/null +++ b/vendor/github.com/gobwas/glob/match/list.go @@ -0,0 +1,49 @@ +package match + +import ( + "fmt" + "github.com/gobwas/glob/util/runes" + "unicode/utf8" +) + +type List struct { + List []rune + Not bool +} + +func NewList(list []rune, not bool) List { + return List{list, not} +} + +func (self List) Match(s string) bool { + r, w := utf8.DecodeRuneInString(s) + if len(s) > w { + return false + } + + inList := runes.IndexRune(self.List, r) != -1 + return inList == !self.Not +} + +func (self List) Len() int { + return lenOne +} + +func (self List) Index(s string) (int, []int) { + for i, r := range s { + if self.Not == (runes.IndexRune(self.List, r) == -1) { + return i, segmentsByRuneLength[utf8.RuneLen(r)] + } + } + + return -1, nil +} + +func (self List) String() string { + var not string + if self.Not { + not = "!" + } + + return fmt.Sprintf("", not, string(self.List)) +} diff --git a/vendor/github.com/gobwas/glob/match/match.go b/vendor/github.com/gobwas/glob/match/match.go new file mode 100644 index 000000000..f80e007fb --- /dev/null +++ b/vendor/github.com/gobwas/glob/match/match.go @@ -0,0 +1,81 @@ +package match + +// todo common table of rune's length + +import ( + "fmt" + "strings" +) + +const lenOne = 1 +const lenZero = 0 +const lenNo = -1 + +type Matcher interface { + Match(string) bool + Index(string) (int, []int) + Len() int + String() string +} + +type Matchers []Matcher + +func (m Matchers) String() string { + var s []string + for _, matcher := range m { + s = append(s, fmt.Sprint(matcher)) + } + + return fmt.Sprintf("%s", strings.Join(s, ",")) +} + +// appendMerge merges and sorts given already SORTED and UNIQUE segments. +func appendMerge(target, sub []int) []int { + lt, ls := len(target), len(sub) + out := make([]int, 0, lt+ls) + + for x, y := 0, 0; x < lt || y < ls; { + if x >= lt { + out = append(out, sub[y:]...) + break + } + + if y >= ls { + out = append(out, target[x:]...) + break + } + + xValue := target[x] + yValue := sub[y] + + switch { + + case xValue == yValue: + out = append(out, xValue) + x++ + y++ + + case xValue < yValue: + out = append(out, xValue) + x++ + + case yValue < xValue: + out = append(out, yValue) + y++ + + } + } + + target = append(target[:0], out...) + + return target +} + +func reverseSegments(input []int) { + l := len(input) + m := l / 2 + + for i := 0; i < m; i++ { + input[i], input[l-i-1] = input[l-i-1], input[i] + } +} diff --git a/vendor/github.com/gobwas/glob/match/max.go b/vendor/github.com/gobwas/glob/match/max.go new file mode 100644 index 000000000..d72f69eff --- /dev/null +++ b/vendor/github.com/gobwas/glob/match/max.go @@ -0,0 +1,49 @@ +package match + +import ( + "fmt" + "unicode/utf8" +) + +type Max struct { + Limit int +} + +func NewMax(l int) Max { + return Max{l} +} + +func (self Max) Match(s string) bool { + var l int + for range s { + l += 1 + if l > self.Limit { + return false + } + } + + return true +} + +func (self Max) Index(s string) (int, []int) { + segments := acquireSegments(self.Limit + 1) + segments = append(segments, 0) + var count int + for i, r := range s { + count++ + if count > self.Limit { + break + } + segments = append(segments, i+utf8.RuneLen(r)) + } + + return 0, segments +} + +func (self Max) Len() int { + return lenNo +} + +func (self Max) String() string { + return fmt.Sprintf("", self.Limit) +} diff --git a/vendor/github.com/gobwas/glob/match/min.go b/vendor/github.com/gobwas/glob/match/min.go new file mode 100644 index 000000000..db57ac8eb --- /dev/null +++ b/vendor/github.com/gobwas/glob/match/min.go @@ -0,0 +1,57 @@ +package match + +import ( + "fmt" + "unicode/utf8" +) + +type Min struct { + Limit int +} + +func NewMin(l int) Min { + return Min{l} +} + +func (self Min) Match(s string) bool { + var l int + for range s { + l += 1 + if l >= self.Limit { + return true + } + } + + return false +} + +func (self Min) Index(s string) (int, []int) { + var count int + + c := len(s) - self.Limit + 1 + if c <= 0 { + return -1, nil + } + + segments := acquireSegments(c) + for i, r := range s { + count++ + if count >= self.Limit { + segments = append(segments, i+utf8.RuneLen(r)) + } + } + + if len(segments) == 0 { + return -1, nil + } + + return 0, segments +} + +func (self Min) Len() int { + return lenNo +} + +func (self Min) String() string { + return fmt.Sprintf("", self.Limit) +} diff --git a/vendor/github.com/gobwas/glob/match/nothing.go b/vendor/github.com/gobwas/glob/match/nothing.go new file mode 100644 index 000000000..0d4ecd36b --- /dev/null +++ b/vendor/github.com/gobwas/glob/match/nothing.go @@ -0,0 +1,27 @@ +package match + +import ( + "fmt" +) + +type Nothing struct{} + +func NewNothing() Nothing { + return Nothing{} +} + +func (self Nothing) Match(s string) bool { + return len(s) == 0 +} + +func (self Nothing) Index(s string) (int, []int) { + return 0, segments0 +} + +func (self Nothing) Len() int { + return lenZero +} + +func (self Nothing) String() string { + return fmt.Sprintf("") +} diff --git a/vendor/github.com/gobwas/glob/match/prefix.go b/vendor/github.com/gobwas/glob/match/prefix.go new file mode 100644 index 000000000..a7347250e --- /dev/null +++ b/vendor/github.com/gobwas/glob/match/prefix.go @@ -0,0 +1,50 @@ +package match + +import ( + "fmt" + "strings" + "unicode/utf8" +) + +type Prefix struct { + Prefix string +} + +func NewPrefix(p string) Prefix { + return Prefix{p} +} + +func (self Prefix) Index(s string) (int, []int) { + idx := strings.Index(s, self.Prefix) + if idx == -1 { + return -1, nil + } + + length := len(self.Prefix) + var sub string + if len(s) > idx+length { + sub = s[idx+length:] + } else { + sub = "" + } + + segments := acquireSegments(len(sub) + 1) + segments = append(segments, length) + for i, r := range sub { + segments = append(segments, length+i+utf8.RuneLen(r)) + } + + return idx, segments +} + +func (self Prefix) Len() int { + return lenNo +} + +func (self Prefix) Match(s string) bool { + return strings.HasPrefix(s, self.Prefix) +} + +func (self Prefix) String() string { + return fmt.Sprintf("", self.Prefix) +} diff --git a/vendor/github.com/gobwas/glob/match/prefix_any.go b/vendor/github.com/gobwas/glob/match/prefix_any.go new file mode 100644 index 000000000..8ee58fe1b --- /dev/null +++ b/vendor/github.com/gobwas/glob/match/prefix_any.go @@ -0,0 +1,55 @@ +package match + +import ( + "fmt" + "strings" + "unicode/utf8" + + sutil "github.com/gobwas/glob/util/strings" +) + +type PrefixAny struct { + Prefix string + Separators []rune +} + +func NewPrefixAny(s string, sep []rune) PrefixAny { + return PrefixAny{s, sep} +} + +func (self PrefixAny) Index(s string) (int, []int) { + idx := strings.Index(s, self.Prefix) + if idx == -1 { + return -1, nil + } + + n := len(self.Prefix) + sub := s[idx+n:] + i := sutil.IndexAnyRunes(sub, self.Separators) + if i > -1 { + sub = sub[:i] + } + + seg := acquireSegments(len(sub) + 1) + seg = append(seg, n) + for i, r := range sub { + seg = append(seg, n+i+utf8.RuneLen(r)) + } + + return idx, seg +} + +func (self PrefixAny) Len() int { + return lenNo +} + +func (self PrefixAny) Match(s string) bool { + if !strings.HasPrefix(s, self.Prefix) { + return false + } + return sutil.IndexAnyRunes(s[len(self.Prefix):], self.Separators) == -1 +} + +func (self PrefixAny) String() string { + return fmt.Sprintf("", self.Prefix, string(self.Separators)) +} diff --git a/vendor/github.com/gobwas/glob/match/prefix_suffix.go b/vendor/github.com/gobwas/glob/match/prefix_suffix.go new file mode 100644 index 000000000..8208085a1 --- /dev/null +++ b/vendor/github.com/gobwas/glob/match/prefix_suffix.go @@ -0,0 +1,62 @@ +package match + +import ( + "fmt" + "strings" +) + +type PrefixSuffix struct { + Prefix, Suffix string +} + +func NewPrefixSuffix(p, s string) PrefixSuffix { + return PrefixSuffix{p, s} +} + +func (self PrefixSuffix) Index(s string) (int, []int) { + prefixIdx := strings.Index(s, self.Prefix) + if prefixIdx == -1 { + return -1, nil + } + + suffixLen := len(self.Suffix) + if suffixLen <= 0 { + return prefixIdx, []int{len(s) - prefixIdx} + } + + if (len(s) - prefixIdx) <= 0 { + return -1, nil + } + + segments := acquireSegments(len(s) - prefixIdx) + for sub := s[prefixIdx:]; ; { + suffixIdx := strings.LastIndex(sub, self.Suffix) + if suffixIdx == -1 { + break + } + + segments = append(segments, suffixIdx+suffixLen) + sub = sub[:suffixIdx] + } + + if len(segments) == 0 { + releaseSegments(segments) + return -1, nil + } + + reverseSegments(segments) + + return prefixIdx, segments +} + +func (self PrefixSuffix) Len() int { + return lenNo +} + +func (self PrefixSuffix) Match(s string) bool { + return strings.HasPrefix(s, self.Prefix) && strings.HasSuffix(s, self.Suffix) +} + +func (self PrefixSuffix) String() string { + return fmt.Sprintf("", self.Prefix, self.Suffix) +} diff --git a/vendor/github.com/gobwas/glob/match/range.go b/vendor/github.com/gobwas/glob/match/range.go new file mode 100644 index 000000000..ce30245a4 --- /dev/null +++ b/vendor/github.com/gobwas/glob/match/range.go @@ -0,0 +1,48 @@ +package match + +import ( + "fmt" + "unicode/utf8" +) + +type Range struct { + Lo, Hi rune + Not bool +} + +func NewRange(lo, hi rune, not bool) Range { + return Range{lo, hi, not} +} + +func (self Range) Len() int { + return lenOne +} + +func (self Range) Match(s string) bool { + r, w := utf8.DecodeRuneInString(s) + if len(s) > w { + return false + } + + inRange := r >= self.Lo && r <= self.Hi + + return inRange == !self.Not +} + +func (self Range) Index(s string) (int, []int) { + for i, r := range s { + if self.Not != (r >= self.Lo && r <= self.Hi) { + return i, segmentsByRuneLength[utf8.RuneLen(r)] + } + } + + return -1, nil +} + +func (self Range) String() string { + var not string + if self.Not { + not = "!" + } + return fmt.Sprintf("", not, string(self.Lo), string(self.Hi)) +} diff --git a/vendor/github.com/gobwas/glob/match/row.go b/vendor/github.com/gobwas/glob/match/row.go new file mode 100644 index 000000000..4379042e4 --- /dev/null +++ b/vendor/github.com/gobwas/glob/match/row.go @@ -0,0 +1,77 @@ +package match + +import ( + "fmt" +) + +type Row struct { + Matchers Matchers + RunesLength int + Segments []int +} + +func NewRow(len int, m ...Matcher) Row { + return Row{ + Matchers: Matchers(m), + RunesLength: len, + Segments: []int{len}, + } +} + +func (self Row) matchAll(s string) bool { + var idx int + for _, m := range self.Matchers { + length := m.Len() + + var next, i int + for next = range s[idx:] { + i++ + if i == length { + break + } + } + + if i < length || !m.Match(s[idx:idx+next+1]) { + return false + } + + idx += next + 1 + } + + return true +} + +func (self Row) lenOk(s string) bool { + var i int + for range s { + i++ + if i > self.RunesLength { + return false + } + } + return self.RunesLength == i +} + +func (self Row) Match(s string) bool { + return self.lenOk(s) && self.matchAll(s) +} + +func (self Row) Len() (l int) { + return self.RunesLength +} + +func (self Row) Index(s string) (int, []int) { + for i := range s { + if len(s[i:]) < self.RunesLength { + break + } + if self.matchAll(s[i:]) { + return i, self.Segments + } + } + return -1, nil +} + +func (self Row) String() string { + return fmt.Sprintf("", self.RunesLength, self.Matchers) +} diff --git a/vendor/github.com/gobwas/glob/match/segments.go b/vendor/github.com/gobwas/glob/match/segments.go new file mode 100644 index 000000000..9ea6f3094 --- /dev/null +++ b/vendor/github.com/gobwas/glob/match/segments.go @@ -0,0 +1,91 @@ +package match + +import ( + "sync" +) + +type SomePool interface { + Get() []int + Put([]int) +} + +var segmentsPools [1024]sync.Pool + +func toPowerOfTwo(v int) int { + v-- + v |= v >> 1 + v |= v >> 2 + v |= v >> 4 + v |= v >> 8 + v |= v >> 16 + v++ + + return v +} + +const ( + cacheFrom = 16 + cacheToAndHigher = 1024 + cacheFromIndex = 15 + cacheToAndHigherIndex = 1023 +) + +var ( + segments0 = []int{0} + segments1 = []int{1} + segments2 = []int{2} + segments3 = []int{3} + segments4 = []int{4} +) + +var segmentsByRuneLength [5][]int = [5][]int{ + 0: segments0, + 1: segments1, + 2: segments2, + 3: segments3, + 4: segments4, +} + +func init() { + for i := cacheToAndHigher; i >= cacheFrom; i >>= 1 { + func(i int) { + segmentsPools[i-1] = sync.Pool{New: func() interface{} { + return make([]int, 0, i) + }} + }(i) + } +} + +func getTableIndex(c int) int { + p := toPowerOfTwo(c) + switch { + case p >= cacheToAndHigher: + return cacheToAndHigherIndex + case p <= cacheFrom: + return cacheFromIndex + default: + return p - 1 + } +} + +func acquireSegments(c int) []int { + // make []int with less capacity than cacheFrom + // is faster than acquiring it from pool + if c < cacheFrom { + return make([]int, 0, c) + } + + return segmentsPools[getTableIndex(c)].Get().([]int)[:0] +} + +func releaseSegments(s []int) { + c := cap(s) + + // make []int with less capacity than cacheFrom + // is faster than acquiring it from pool + if c < cacheFrom { + return + } + + segmentsPools[getTableIndex(c)].Put(s) +} diff --git a/vendor/github.com/gobwas/glob/match/single.go b/vendor/github.com/gobwas/glob/match/single.go new file mode 100644 index 000000000..ee6e3954c --- /dev/null +++ b/vendor/github.com/gobwas/glob/match/single.go @@ -0,0 +1,43 @@ +package match + +import ( + "fmt" + "github.com/gobwas/glob/util/runes" + "unicode/utf8" +) + +// single represents ? +type Single struct { + Separators []rune +} + +func NewSingle(s []rune) Single { + return Single{s} +} + +func (self Single) Match(s string) bool { + r, w := utf8.DecodeRuneInString(s) + if len(s) > w { + return false + } + + return runes.IndexRune(self.Separators, r) == -1 +} + +func (self Single) Len() int { + return lenOne +} + +func (self Single) Index(s string) (int, []int) { + for i, r := range s { + if runes.IndexRune(self.Separators, r) == -1 { + return i, segmentsByRuneLength[utf8.RuneLen(r)] + } + } + + return -1, nil +} + +func (self Single) String() string { + return fmt.Sprintf("", string(self.Separators)) +} diff --git a/vendor/github.com/gobwas/glob/match/suffix.go b/vendor/github.com/gobwas/glob/match/suffix.go new file mode 100644 index 000000000..85bea8c68 --- /dev/null +++ b/vendor/github.com/gobwas/glob/match/suffix.go @@ -0,0 +1,35 @@ +package match + +import ( + "fmt" + "strings" +) + +type Suffix struct { + Suffix string +} + +func NewSuffix(s string) Suffix { + return Suffix{s} +} + +func (self Suffix) Len() int { + return lenNo +} + +func (self Suffix) Match(s string) bool { + return strings.HasSuffix(s, self.Suffix) +} + +func (self Suffix) Index(s string) (int, []int) { + idx := strings.Index(s, self.Suffix) + if idx == -1 { + return -1, nil + } + + return 0, []int{idx + len(self.Suffix)} +} + +func (self Suffix) String() string { + return fmt.Sprintf("", self.Suffix) +} diff --git a/vendor/github.com/gobwas/glob/match/suffix_any.go b/vendor/github.com/gobwas/glob/match/suffix_any.go new file mode 100644 index 000000000..c5106f819 --- /dev/null +++ b/vendor/github.com/gobwas/glob/match/suffix_any.go @@ -0,0 +1,43 @@ +package match + +import ( + "fmt" + "strings" + + sutil "github.com/gobwas/glob/util/strings" +) + +type SuffixAny struct { + Suffix string + Separators []rune +} + +func NewSuffixAny(s string, sep []rune) SuffixAny { + return SuffixAny{s, sep} +} + +func (self SuffixAny) Index(s string) (int, []int) { + idx := strings.Index(s, self.Suffix) + if idx == -1 { + return -1, nil + } + + i := sutil.LastIndexAnyRunes(s[:idx], self.Separators) + 1 + + return i, []int{idx + len(self.Suffix) - i} +} + +func (self SuffixAny) Len() int { + return lenNo +} + +func (self SuffixAny) Match(s string) bool { + if !strings.HasSuffix(s, self.Suffix) { + return false + } + return sutil.IndexAnyRunes(s[:len(s)-len(self.Suffix)], self.Separators) == -1 +} + +func (self SuffixAny) String() string { + return fmt.Sprintf("", string(self.Separators), self.Suffix) +} diff --git a/vendor/github.com/gobwas/glob/match/super.go b/vendor/github.com/gobwas/glob/match/super.go new file mode 100644 index 000000000..3875950bb --- /dev/null +++ b/vendor/github.com/gobwas/glob/match/super.go @@ -0,0 +1,33 @@ +package match + +import ( + "fmt" +) + +type Super struct{} + +func NewSuper() Super { + return Super{} +} + +func (self Super) Match(s string) bool { + return true +} + +func (self Super) Len() int { + return lenNo +} + +func (self Super) Index(s string) (int, []int) { + segments := acquireSegments(len(s) + 1) + for i := range s { + segments = append(segments, i) + } + segments = append(segments, len(s)) + + return 0, segments +} + +func (self Super) String() string { + return fmt.Sprintf("") +} diff --git a/vendor/github.com/gobwas/glob/match/text.go b/vendor/github.com/gobwas/glob/match/text.go new file mode 100644 index 000000000..0a17616d3 --- /dev/null +++ b/vendor/github.com/gobwas/glob/match/text.go @@ -0,0 +1,45 @@ +package match + +import ( + "fmt" + "strings" + "unicode/utf8" +) + +// raw represents raw string to match +type Text struct { + Str string + RunesLength int + BytesLength int + Segments []int +} + +func NewText(s string) Text { + return Text{ + Str: s, + RunesLength: utf8.RuneCountInString(s), + BytesLength: len(s), + Segments: []int{len(s)}, + } +} + +func (self Text) Match(s string) bool { + return self.Str == s +} + +func (self Text) Len() int { + return self.RunesLength +} + +func (self Text) Index(s string) (int, []int) { + index := strings.Index(s, self.Str) + if index == -1 { + return -1, nil + } + + return index, self.Segments +} + +func (self Text) String() string { + return fmt.Sprintf("", self.Str) +} diff --git a/vendor/github.com/gobwas/glob/readme.md b/vendor/github.com/gobwas/glob/readme.md new file mode 100644 index 000000000..f58144e73 --- /dev/null +++ b/vendor/github.com/gobwas/glob/readme.md @@ -0,0 +1,148 @@ +# glob.[go](https://golang.org) + +[![GoDoc][godoc-image]][godoc-url] [![Build Status][travis-image]][travis-url] + +> Go Globbing Library. + +## Install + +```shell + go get github.com/gobwas/glob +``` + +## Example + +```go + +package main + +import "github.com/gobwas/glob" + +func main() { + var g glob.Glob + + // create simple glob + g = glob.MustCompile("*.github.com") + g.Match("api.github.com") // true + + // quote meta characters and then create simple glob + g = glob.MustCompile(glob.QuoteMeta("*.github.com")) + g.Match("*.github.com") // true + + // create new glob with set of delimiters as ["."] + g = glob.MustCompile("api.*.com", '.') + g.Match("api.github.com") // true + g.Match("api.gi.hub.com") // false + + // create new glob with set of delimiters as ["."] + // but now with super wildcard + g = glob.MustCompile("api.**.com", '.') + g.Match("api.github.com") // true + g.Match("api.gi.hub.com") // true + + // create glob with single symbol wildcard + g = glob.MustCompile("?at") + g.Match("cat") // true + g.Match("fat") // true + g.Match("at") // false + + // create glob with single symbol wildcard and delimiters ['f'] + g = glob.MustCompile("?at", 'f') + g.Match("cat") // true + g.Match("fat") // false + g.Match("at") // false + + // create glob with character-list matchers + g = glob.MustCompile("[abc]at") + g.Match("cat") // true + g.Match("bat") // true + g.Match("fat") // false + g.Match("at") // false + + // create glob with character-list matchers + g = glob.MustCompile("[!abc]at") + g.Match("cat") // false + g.Match("bat") // false + g.Match("fat") // true + g.Match("at") // false + + // create glob with character-range matchers + g = glob.MustCompile("[a-c]at") + g.Match("cat") // true + g.Match("bat") // true + g.Match("fat") // false + g.Match("at") // false + + // create glob with character-range matchers + g = glob.MustCompile("[!a-c]at") + g.Match("cat") // false + g.Match("bat") // false + g.Match("fat") // true + g.Match("at") // false + + // create glob with pattern-alternatives list + g = glob.MustCompile("{cat,bat,[fr]at}") + g.Match("cat") // true + g.Match("bat") // true + g.Match("fat") // true + g.Match("rat") // true + g.Match("at") // false + g.Match("zat") // false +} + +``` + +## Performance + +This library is created for compile-once patterns. This means, that compilation could take time, but +strings matching is done faster, than in case when always parsing template. + +If you will not use compiled `glob.Glob` object, and do `g := glob.MustCompile(pattern); g.Match(...)` every time, then your code will be much more slower. + +Run `go test -bench=.` from source root to see the benchmarks: + +Pattern | Fixture | Match | Speed (ns/op) +--------|---------|-------|-------------- +`[a-z][!a-x]*cat*[h][!b]*eyes*` | `my cat has very bright eyes` | `true` | 432 +`[a-z][!a-x]*cat*[h][!b]*eyes*` | `my dog has very bright eyes` | `false` | 199 +`https://*.google.*` | `https://account.google.com` | `true` | 96 +`https://*.google.*` | `https://google.com` | `false` | 66 +`{https://*.google.*,*yandex.*,*yahoo.*,*mail.ru}` | `http://yahoo.com` | `true` | 163 +`{https://*.google.*,*yandex.*,*yahoo.*,*mail.ru}` | `http://google.com` | `false` | 197 +`{https://*gobwas.com,http://exclude.gobwas.com}` | `https://safe.gobwas.com` | `true` | 22 +`{https://*gobwas.com,http://exclude.gobwas.com}` | `http://safe.gobwas.com` | `false` | 24 +`abc*` | `abcdef` | `true` | 8.15 +`abc*` | `af` | `false` | 5.68 +`*def` | `abcdef` | `true` | 8.84 +`*def` | `af` | `false` | 5.74 +`ab*ef` | `abcdef` | `true` | 15.2 +`ab*ef` | `af` | `false` | 10.4 + +The same things with `regexp` package: + +Pattern | Fixture | Match | Speed (ns/op) +--------|---------|-------|-------------- +`^[a-z][^a-x].*cat.*[h][^b].*eyes.*$` | `my cat has very bright eyes` | `true` | 2553 +`^[a-z][^a-x].*cat.*[h][^b].*eyes.*$` | `my dog has very bright eyes` | `false` | 1383 +`^https:\/\/.*\.google\..*$` | `https://account.google.com` | `true` | 1205 +`^https:\/\/.*\.google\..*$` | `https://google.com` | `false` | 767 +`^(https:\/\/.*\.google\..*|.*yandex\..*|.*yahoo\..*|.*mail\.ru)$` | `http://yahoo.com` | `true` | 1435 +`^(https:\/\/.*\.google\..*|.*yandex\..*|.*yahoo\..*|.*mail\.ru)$` | `http://google.com` | `false` | 1674 +`^(https:\/\/.*gobwas\.com|http://exclude.gobwas.com)$` | `https://safe.gobwas.com` | `true` | 1039 +`^(https:\/\/.*gobwas\.com|http://exclude.gobwas.com)$` | `http://safe.gobwas.com` | `false` | 272 +`^abc.*$` | `abcdef` | `true` | 237 +`^abc.*$` | `af` | `false` | 100 +`^.*def$` | `abcdef` | `true` | 464 +`^.*def$` | `af` | `false` | 265 +`^ab.*ef$` | `abcdef` | `true` | 375 +`^ab.*ef$` | `af` | `false` | 145 + +[godoc-image]: https://godoc.org/github.com/gobwas/glob?status.svg +[godoc-url]: https://godoc.org/github.com/gobwas/glob +[travis-image]: https://travis-ci.org/gobwas/glob.svg?branch=master +[travis-url]: https://travis-ci.org/gobwas/glob + +## Syntax + +Syntax is inspired by [standard wildcards](http://tldp.org/LDP/GNU-Linux-Tools-Summary/html/x11655.htm), +except that `**` is aka super-asterisk, that do not sensitive for separators. \ No newline at end of file diff --git a/vendor/github.com/gobwas/glob/syntax/ast/ast.go b/vendor/github.com/gobwas/glob/syntax/ast/ast.go new file mode 100644 index 000000000..3220a694a --- /dev/null +++ b/vendor/github.com/gobwas/glob/syntax/ast/ast.go @@ -0,0 +1,122 @@ +package ast + +import ( + "bytes" + "fmt" +) + +type Node struct { + Parent *Node + Children []*Node + Value interface{} + Kind Kind +} + +func NewNode(k Kind, v interface{}, ch ...*Node) *Node { + n := &Node{ + Kind: k, + Value: v, + } + for _, c := range ch { + Insert(n, c) + } + return n +} + +func (a *Node) Equal(b *Node) bool { + if a.Kind != b.Kind { + return false + } + if a.Value != b.Value { + return false + } + if len(a.Children) != len(b.Children) { + return false + } + for i, c := range a.Children { + if !c.Equal(b.Children[i]) { + return false + } + } + return true +} + +func (a *Node) String() string { + var buf bytes.Buffer + buf.WriteString(a.Kind.String()) + if a.Value != nil { + buf.WriteString(" =") + buf.WriteString(fmt.Sprintf("%v", a.Value)) + } + if len(a.Children) > 0 { + buf.WriteString(" [") + for i, c := range a.Children { + if i > 0 { + buf.WriteString(", ") + } + buf.WriteString(c.String()) + } + buf.WriteString("]") + } + return buf.String() +} + +func Insert(parent *Node, children ...*Node) { + parent.Children = append(parent.Children, children...) + for _, ch := range children { + ch.Parent = parent + } +} + +type List struct { + Not bool + Chars string +} + +type Range struct { + Not bool + Lo, Hi rune +} + +type Text struct { + Text string +} + +type Kind int + +const ( + KindNothing Kind = iota + KindPattern + KindList + KindRange + KindText + KindAny + KindSuper + KindSingle + KindAnyOf +) + +func (k Kind) String() string { + switch k { + case KindNothing: + return "Nothing" + case KindPattern: + return "Pattern" + case KindList: + return "List" + case KindRange: + return "Range" + case KindText: + return "Text" + case KindAny: + return "Any" + case KindSuper: + return "Super" + case KindSingle: + return "Single" + case KindAnyOf: + return "AnyOf" + default: + return "" + } +} diff --git a/vendor/github.com/gobwas/glob/syntax/ast/parser.go b/vendor/github.com/gobwas/glob/syntax/ast/parser.go new file mode 100644 index 000000000..429b40943 --- /dev/null +++ b/vendor/github.com/gobwas/glob/syntax/ast/parser.go @@ -0,0 +1,157 @@ +package ast + +import ( + "errors" + "fmt" + "github.com/gobwas/glob/syntax/lexer" + "unicode/utf8" +) + +type Lexer interface { + Next() lexer.Token +} + +type parseFn func(*Node, Lexer) (parseFn, *Node, error) + +func Parse(lexer Lexer) (*Node, error) { + var parser parseFn + + root := NewNode(KindPattern, nil) + + var ( + tree *Node + err error + ) + for parser, tree = parserMain, root; parser != nil; { + parser, tree, err = parser(tree, lexer) + if err != nil { + return nil, err + } + } + + return root, nil +} + +func parserMain(tree *Node, lex Lexer) (parseFn, *Node, error) { + for { + token := lex.Next() + switch token.Type { + case lexer.EOF: + return nil, tree, nil + + case lexer.Error: + return nil, tree, errors.New(token.Raw) + + case lexer.Text: + Insert(tree, NewNode(KindText, Text{token.Raw})) + return parserMain, tree, nil + + case lexer.Any: + Insert(tree, NewNode(KindAny, nil)) + return parserMain, tree, nil + + case lexer.Super: + Insert(tree, NewNode(KindSuper, nil)) + return parserMain, tree, nil + + case lexer.Single: + Insert(tree, NewNode(KindSingle, nil)) + return parserMain, tree, nil + + case lexer.RangeOpen: + return parserRange, tree, nil + + case lexer.TermsOpen: + a := NewNode(KindAnyOf, nil) + Insert(tree, a) + + p := NewNode(KindPattern, nil) + Insert(a, p) + + return parserMain, p, nil + + case lexer.Separator: + p := NewNode(KindPattern, nil) + Insert(tree.Parent, p) + + return parserMain, p, nil + + case lexer.TermsClose: + return parserMain, tree.Parent.Parent, nil + + default: + return nil, tree, fmt.Errorf("unexpected token: %s", token) + } + } + return nil, tree, fmt.Errorf("unknown error") +} + +func parserRange(tree *Node, lex Lexer) (parseFn, *Node, error) { + var ( + not bool + lo rune + hi rune + chars string + ) + for { + token := lex.Next() + switch token.Type { + case lexer.EOF: + return nil, tree, errors.New("unexpected end") + + case lexer.Error: + return nil, tree, errors.New(token.Raw) + + case lexer.Not: + not = true + + case lexer.RangeLo: + r, w := utf8.DecodeRuneInString(token.Raw) + if len(token.Raw) > w { + return nil, tree, fmt.Errorf("unexpected length of lo character") + } + lo = r + + case lexer.RangeBetween: + // + + case lexer.RangeHi: + r, w := utf8.DecodeRuneInString(token.Raw) + if len(token.Raw) > w { + return nil, tree, fmt.Errorf("unexpected length of lo character") + } + + hi = r + + if hi < lo { + return nil, tree, fmt.Errorf("hi character '%s' should be greater than lo '%s'", string(hi), string(lo)) + } + + case lexer.Text: + chars = token.Raw + + case lexer.RangeClose: + isRange := lo != 0 && hi != 0 + isChars := chars != "" + + if isChars == isRange { + return nil, tree, fmt.Errorf("could not parse range") + } + + if isRange { + Insert(tree, NewNode(KindRange, Range{ + Lo: lo, + Hi: hi, + Not: not, + })) + } else { + Insert(tree, NewNode(KindList, List{ + Chars: chars, + Not: not, + })) + } + + return parserMain, tree, nil + } + } +} diff --git a/vendor/github.com/gobwas/glob/syntax/lexer/lexer.go b/vendor/github.com/gobwas/glob/syntax/lexer/lexer.go new file mode 100644 index 000000000..a1c8d1962 --- /dev/null +++ b/vendor/github.com/gobwas/glob/syntax/lexer/lexer.go @@ -0,0 +1,273 @@ +package lexer + +import ( + "bytes" + "fmt" + "github.com/gobwas/glob/util/runes" + "unicode/utf8" +) + +const ( + char_any = '*' + char_comma = ',' + char_single = '?' + char_escape = '\\' + char_range_open = '[' + char_range_close = ']' + char_terms_open = '{' + char_terms_close = '}' + char_range_not = '!' + char_range_between = '-' +) + +var specials = []byte{ + char_any, + char_single, + char_escape, + char_range_open, + char_range_close, + char_terms_open, + char_terms_close, +} + +func Special(c byte) bool { + return bytes.IndexByte(specials, c) != -1 +} + +type tokens []Token + +func (i *tokens) shift() (ret Token) { + ret = (*i)[0] + copy(*i, (*i)[1:]) + *i = (*i)[:len(*i)-1] + return +} + +func (i *tokens) push(v Token) { + *i = append(*i, v) +} + +func (i *tokens) empty() bool { + return len(*i) == 0 +} + +var eof rune = 0 + +type lexer struct { + data string + pos int + err error + + tokens tokens + termsLevel int + + lastRune rune + lastRuneSize int + hasRune bool +} + +func NewLexer(source string) *lexer { + l := &lexer{ + data: source, + tokens: tokens(make([]Token, 0, 4)), + } + return l +} + +func (l *lexer) Next() Token { + if l.err != nil { + return Token{Error, l.err.Error()} + } + if !l.tokens.empty() { + return l.tokens.shift() + } + + l.fetchItem() + return l.Next() +} + +func (l *lexer) peek() (r rune, w int) { + if l.pos == len(l.data) { + return eof, 0 + } + + r, w = utf8.DecodeRuneInString(l.data[l.pos:]) + if r == utf8.RuneError { + l.errorf("could not read rune") + r = eof + w = 0 + } + + return +} + +func (l *lexer) read() rune { + if l.hasRune { + l.hasRune = false + l.seek(l.lastRuneSize) + return l.lastRune + } + + r, s := l.peek() + l.seek(s) + + l.lastRune = r + l.lastRuneSize = s + + return r +} + +func (l *lexer) seek(w int) { + l.pos += w +} + +func (l *lexer) unread() { + if l.hasRune { + l.errorf("could not unread rune") + return + } + l.seek(-l.lastRuneSize) + l.hasRune = true +} + +func (l *lexer) errorf(f string, v ...interface{}) { + l.err = fmt.Errorf(f, v...) +} + +func (l *lexer) inTerms() bool { + return l.termsLevel > 0 +} + +func (l *lexer) termsEnter() { + l.termsLevel++ +} + +func (l *lexer) termsLeave() { + l.termsLevel-- +} + +var inTextBreakers = []rune{char_single, char_any, char_range_open, char_terms_open} +var inTermsBreakers = append(inTextBreakers, char_terms_close, char_comma) + +func (l *lexer) fetchItem() { + r := l.read() + switch { + case r == eof: + l.tokens.push(Token{EOF, ""}) + + case r == char_terms_open: + l.termsEnter() + l.tokens.push(Token{TermsOpen, string(r)}) + + case r == char_comma && l.inTerms(): + l.tokens.push(Token{Separator, string(r)}) + + case r == char_terms_close && l.inTerms(): + l.tokens.push(Token{TermsClose, string(r)}) + l.termsLeave() + + case r == char_range_open: + l.tokens.push(Token{RangeOpen, string(r)}) + l.fetchRange() + + case r == char_single: + l.tokens.push(Token{Single, string(r)}) + + case r == char_any: + if l.read() == char_any { + l.tokens.push(Token{Super, string(r) + string(r)}) + } else { + l.unread() + l.tokens.push(Token{Any, string(r)}) + } + + default: + l.unread() + + var breakers []rune + if l.inTerms() { + breakers = inTermsBreakers + } else { + breakers = inTextBreakers + } + l.fetchText(breakers) + } +} + +func (l *lexer) fetchRange() { + var wantHi bool + var wantClose bool + var seenNot bool + for { + r := l.read() + if r == eof { + l.errorf("unexpected end of input") + return + } + + if wantClose { + if r != char_range_close { + l.errorf("expected close range character") + } else { + l.tokens.push(Token{RangeClose, string(r)}) + } + return + } + + if wantHi { + l.tokens.push(Token{RangeHi, string(r)}) + wantClose = true + continue + } + + if !seenNot && r == char_range_not { + l.tokens.push(Token{Not, string(r)}) + seenNot = true + continue + } + + if n, w := l.peek(); n == char_range_between { + l.seek(w) + l.tokens.push(Token{RangeLo, string(r)}) + l.tokens.push(Token{RangeBetween, string(n)}) + wantHi = true + continue + } + + l.unread() // unread first peek and fetch as text + l.fetchText([]rune{char_range_close}) + wantClose = true + } +} + +func (l *lexer) fetchText(breakers []rune) { + var data []rune + var escaped bool + +reading: + for { + r := l.read() + if r == eof { + break + } + + if !escaped { + if r == char_escape { + escaped = true + continue + } + + if runes.IndexRune(breakers, r) != -1 { + l.unread() + break reading + } + } + + escaped = false + data = append(data, r) + } + + if len(data) > 0 { + l.tokens.push(Token{Text, string(data)}) + } +} diff --git a/vendor/github.com/gobwas/glob/syntax/lexer/token.go b/vendor/github.com/gobwas/glob/syntax/lexer/token.go new file mode 100644 index 000000000..2797c4e83 --- /dev/null +++ b/vendor/github.com/gobwas/glob/syntax/lexer/token.go @@ -0,0 +1,88 @@ +package lexer + +import "fmt" + +type TokenType int + +const ( + EOF TokenType = iota + Error + Text + Char + Any + Super + Single + Not + Separator + RangeOpen + RangeClose + RangeLo + RangeHi + RangeBetween + TermsOpen + TermsClose +) + +func (tt TokenType) String() string { + switch tt { + case EOF: + return "eof" + + case Error: + return "error" + + case Text: + return "text" + + case Char: + return "char" + + case Any: + return "any" + + case Super: + return "super" + + case Single: + return "single" + + case Not: + return "not" + + case Separator: + return "separator" + + case RangeOpen: + return "range_open" + + case RangeClose: + return "range_close" + + case RangeLo: + return "range_lo" + + case RangeHi: + return "range_hi" + + case RangeBetween: + return "range_between" + + case TermsOpen: + return "terms_open" + + case TermsClose: + return "terms_close" + + default: + return "undef" + } +} + +type Token struct { + Type TokenType + Raw string +} + +func (t Token) String() string { + return fmt.Sprintf("%v<%q>", t.Type, t.Raw) +} diff --git a/vendor/github.com/gobwas/glob/syntax/syntax.go b/vendor/github.com/gobwas/glob/syntax/syntax.go new file mode 100644 index 000000000..1d168b148 --- /dev/null +++ b/vendor/github.com/gobwas/glob/syntax/syntax.go @@ -0,0 +1,14 @@ +package syntax + +import ( + "github.com/gobwas/glob/syntax/ast" + "github.com/gobwas/glob/syntax/lexer" +) + +func Parse(s string) (*ast.Node, error) { + return ast.Parse(lexer.NewLexer(s)) +} + +func Special(b byte) bool { + return lexer.Special(b) +} diff --git a/vendor/github.com/gobwas/glob/util/runes/runes.go b/vendor/github.com/gobwas/glob/util/runes/runes.go new file mode 100644 index 000000000..a72355641 --- /dev/null +++ b/vendor/github.com/gobwas/glob/util/runes/runes.go @@ -0,0 +1,154 @@ +package runes + +func Index(s, needle []rune) int { + ls, ln := len(s), len(needle) + + switch { + case ln == 0: + return 0 + case ln == 1: + return IndexRune(s, needle[0]) + case ln == ls: + if Equal(s, needle) { + return 0 + } + return -1 + case ln > ls: + return -1 + } + +head: + for i := 0; i < ls && ls-i >= ln; i++ { + for y := 0; y < ln; y++ { + if s[i+y] != needle[y] { + continue head + } + } + + return i + } + + return -1 +} + +func LastIndex(s, needle []rune) int { + ls, ln := len(s), len(needle) + + switch { + case ln == 0: + if ls == 0 { + return 0 + } + return ls + case ln == 1: + return IndexLastRune(s, needle[0]) + case ln == ls: + if Equal(s, needle) { + return 0 + } + return -1 + case ln > ls: + return -1 + } + +head: + for i := ls - 1; i >= 0 && i >= ln; i-- { + for y := ln - 1; y >= 0; y-- { + if s[i-(ln-y-1)] != needle[y] { + continue head + } + } + + return i - ln + 1 + } + + return -1 +} + +// IndexAny returns the index of the first instance of any Unicode code point +// from chars in s, or -1 if no Unicode code point from chars is present in s. +func IndexAny(s, chars []rune) int { + if len(chars) > 0 { + for i, c := range s { + for _, m := range chars { + if c == m { + return i + } + } + } + } + return -1 +} + +func Contains(s, needle []rune) bool { + return Index(s, needle) >= 0 +} + +func Max(s []rune) (max rune) { + for _, r := range s { + if r > max { + max = r + } + } + + return +} + +func Min(s []rune) rune { + min := rune(-1) + for _, r := range s { + if min == -1 { + min = r + continue + } + + if r < min { + min = r + } + } + + return min +} + +func IndexRune(s []rune, r rune) int { + for i, c := range s { + if c == r { + return i + } + } + return -1 +} + +func IndexLastRune(s []rune, r rune) int { + for i := len(s) - 1; i >= 0; i-- { + if s[i] == r { + return i + } + } + + return -1 +} + +func Equal(a, b []rune) bool { + if len(a) == len(b) { + for i := 0; i < len(a); i++ { + if a[i] != b[i] { + return false + } + } + + return true + } + + return false +} + +// HasPrefix tests whether the string s begins with prefix. +func HasPrefix(s, prefix []rune) bool { + return len(s) >= len(prefix) && Equal(s[0:len(prefix)], prefix) +} + +// HasSuffix tests whether the string s ends with suffix. +func HasSuffix(s, suffix []rune) bool { + return len(s) >= len(suffix) && Equal(s[len(s)-len(suffix):], suffix) +} diff --git a/vendor/github.com/gobwas/glob/util/strings/strings.go b/vendor/github.com/gobwas/glob/util/strings/strings.go new file mode 100644 index 000000000..e8ee1920b --- /dev/null +++ b/vendor/github.com/gobwas/glob/util/strings/strings.go @@ -0,0 +1,39 @@ +package strings + +import ( + "strings" + "unicode/utf8" +) + +func IndexAnyRunes(s string, rs []rune) int { + for _, r := range rs { + if i := strings.IndexRune(s, r); i != -1 { + return i + } + } + + return -1 +} + +func LastIndexAnyRunes(s string, rs []rune) int { + for _, r := range rs { + i := -1 + if 0 <= r && r < utf8.RuneSelf { + i = strings.LastIndexByte(s, byte(r)) + } else { + sub := s + for len(sub) > 0 { + j := strings.IndexRune(s, r) + if j == -1 { + break + } + i = j + sub = sub[i+1:] + } + } + if i != -1 { + return i + } + } + return -1 +} diff --git a/vendor/github.com/klauspost/cpuid/private-gen.go b/vendor/github.com/klauspost/cpuid/private-gen.go new file mode 100644 index 000000000..437333d29 --- /dev/null +++ b/vendor/github.com/klauspost/cpuid/private-gen.go @@ -0,0 +1,476 @@ +// +build ignore + +package main + +import ( + "bytes" + "fmt" + "go/ast" + "go/parser" + "go/printer" + "go/token" + "io" + "io/ioutil" + "log" + "os" + "reflect" + "strings" + "unicode" + "unicode/utf8" +) + +var inFiles = []string{"cpuid.go", "cpuid_test.go"} +var copyFiles = []string{"cpuid_amd64.s", "cpuid_386.s", "detect_ref.go", "detect_intel.go"} +var fileSet = token.NewFileSet() +var reWrites = []rewrite{ + initRewrite("CPUInfo -> cpuInfo"), + initRewrite("Vendor -> vendor"), + initRewrite("Flags -> flags"), + initRewrite("Detect -> detect"), + initRewrite("CPU -> cpu"), +} +var excludeNames = map[string]bool{"string": true, "join": true, "trim": true, + // cpuid_test.go + "t": true, "println": true, "logf": true, "log": true, "fatalf": true, "fatal": true, +} + +var excludePrefixes = []string{"test", "benchmark"} + +func main() { + Package := "private" + parserMode := parser.ParseComments + exported := make(map[string]rewrite) + for _, file := range inFiles { + in, err := os.Open(file) + if err != nil { + log.Fatalf("opening input", err) + } + + src, err := ioutil.ReadAll(in) + if err != nil { + log.Fatalf("reading input", err) + } + + astfile, err := parser.ParseFile(fileSet, file, src, parserMode) + if err != nil { + log.Fatalf("parsing input", err) + } + + for _, rw := range reWrites { + astfile = rw(astfile) + } + + // Inspect the AST and print all identifiers and literals. + var startDecl token.Pos + var endDecl token.Pos + ast.Inspect(astfile, func(n ast.Node) bool { + var s string + switch x := n.(type) { + case *ast.Ident: + if x.IsExported() { + t := strings.ToLower(x.Name) + for _, pre := range excludePrefixes { + if strings.HasPrefix(t, pre) { + return true + } + } + if excludeNames[t] != true { + //if x.Pos() > startDecl && x.Pos() < endDecl { + exported[x.Name] = initRewrite(x.Name + " -> " + t) + } + } + + case *ast.GenDecl: + if x.Tok == token.CONST && x.Lparen > 0 { + startDecl = x.Lparen + endDecl = x.Rparen + // fmt.Printf("Decl:%s -> %s\n", fileSet.Position(startDecl), fileSet.Position(endDecl)) + } + } + if s != "" { + fmt.Printf("%s:\t%s\n", fileSet.Position(n.Pos()), s) + } + return true + }) + + for _, rw := range exported { + astfile = rw(astfile) + } + + var buf bytes.Buffer + + printer.Fprint(&buf, fileSet, astfile) + + // Remove package documentation and insert information + s := buf.String() + ind := strings.Index(buf.String(), "\npackage cpuid") + s = s[ind:] + s = "// Generated, DO NOT EDIT,\n" + + "// but copy it to your own project and rename the package.\n" + + "// See more at http://github.com/klauspost/cpuid\n" + + s + + outputName := Package + string(os.PathSeparator) + file + + err = ioutil.WriteFile(outputName, []byte(s), 0644) + if err != nil { + log.Fatalf("writing output: %s", err) + } + log.Println("Generated", outputName) + } + + for _, file := range copyFiles { + dst := "" + if strings.HasPrefix(file, "cpuid") { + dst = Package + string(os.PathSeparator) + file + } else { + dst = Package + string(os.PathSeparator) + "cpuid_" + file + } + err := copyFile(file, dst) + if err != nil { + log.Fatalf("copying file: %s", err) + } + log.Println("Copied", dst) + } +} + +// CopyFile copies a file from src to dst. If src and dst files exist, and are +// the same, then return success. Copy the file contents from src to dst. +func copyFile(src, dst string) (err error) { + sfi, err := os.Stat(src) + if err != nil { + return + } + if !sfi.Mode().IsRegular() { + // cannot copy non-regular files (e.g., directories, + // symlinks, devices, etc.) + return fmt.Errorf("CopyFile: non-regular source file %s (%q)", sfi.Name(), sfi.Mode().String()) + } + dfi, err := os.Stat(dst) + if err != nil { + if !os.IsNotExist(err) { + return + } + } else { + if !(dfi.Mode().IsRegular()) { + return fmt.Errorf("CopyFile: non-regular destination file %s (%q)", dfi.Name(), dfi.Mode().String()) + } + if os.SameFile(sfi, dfi) { + return + } + } + err = copyFileContents(src, dst) + return +} + +// copyFileContents copies the contents of the file named src to the file named +// by dst. The file will be created if it does not already exist. If the +// destination file exists, all it's contents will be replaced by the contents +// of the source file. +func copyFileContents(src, dst string) (err error) { + in, err := os.Open(src) + if err != nil { + return + } + defer in.Close() + out, err := os.Create(dst) + if err != nil { + return + } + defer func() { + cerr := out.Close() + if err == nil { + err = cerr + } + }() + if _, err = io.Copy(out, in); err != nil { + return + } + err = out.Sync() + return +} + +type rewrite func(*ast.File) *ast.File + +// Mostly copied from gofmt +func initRewrite(rewriteRule string) rewrite { + f := strings.Split(rewriteRule, "->") + if len(f) != 2 { + fmt.Fprintf(os.Stderr, "rewrite rule must be of the form 'pattern -> replacement'\n") + os.Exit(2) + } + pattern := parseExpr(f[0], "pattern") + replace := parseExpr(f[1], "replacement") + return func(p *ast.File) *ast.File { return rewriteFile(pattern, replace, p) } +} + +// parseExpr parses s as an expression. +// It might make sense to expand this to allow statement patterns, +// but there are problems with preserving formatting and also +// with what a wildcard for a statement looks like. +func parseExpr(s, what string) ast.Expr { + x, err := parser.ParseExpr(s) + if err != nil { + fmt.Fprintf(os.Stderr, "parsing %s %s at %s\n", what, s, err) + os.Exit(2) + } + return x +} + +// Keep this function for debugging. +/* +func dump(msg string, val reflect.Value) { + fmt.Printf("%s:\n", msg) + ast.Print(fileSet, val.Interface()) + fmt.Println() +} +*/ + +// rewriteFile applies the rewrite rule 'pattern -> replace' to an entire file. +func rewriteFile(pattern, replace ast.Expr, p *ast.File) *ast.File { + cmap := ast.NewCommentMap(fileSet, p, p.Comments) + m := make(map[string]reflect.Value) + pat := reflect.ValueOf(pattern) + repl := reflect.ValueOf(replace) + + var rewriteVal func(val reflect.Value) reflect.Value + rewriteVal = func(val reflect.Value) reflect.Value { + // don't bother if val is invalid to start with + if !val.IsValid() { + return reflect.Value{} + } + for k := range m { + delete(m, k) + } + val = apply(rewriteVal, val) + if match(m, pat, val) { + val = subst(m, repl, reflect.ValueOf(val.Interface().(ast.Node).Pos())) + } + return val + } + + r := apply(rewriteVal, reflect.ValueOf(p)).Interface().(*ast.File) + r.Comments = cmap.Filter(r).Comments() // recreate comments list + return r +} + +// set is a wrapper for x.Set(y); it protects the caller from panics if x cannot be changed to y. +func set(x, y reflect.Value) { + // don't bother if x cannot be set or y is invalid + if !x.CanSet() || !y.IsValid() { + return + } + defer func() { + if x := recover(); x != nil { + if s, ok := x.(string); ok && + (strings.Contains(s, "type mismatch") || strings.Contains(s, "not assignable")) { + // x cannot be set to y - ignore this rewrite + return + } + panic(x) + } + }() + x.Set(y) +} + +// Values/types for special cases. +var ( + objectPtrNil = reflect.ValueOf((*ast.Object)(nil)) + scopePtrNil = reflect.ValueOf((*ast.Scope)(nil)) + + identType = reflect.TypeOf((*ast.Ident)(nil)) + objectPtrType = reflect.TypeOf((*ast.Object)(nil)) + positionType = reflect.TypeOf(token.NoPos) + callExprType = reflect.TypeOf((*ast.CallExpr)(nil)) + scopePtrType = reflect.TypeOf((*ast.Scope)(nil)) +) + +// apply replaces each AST field x in val with f(x), returning val. +// To avoid extra conversions, f operates on the reflect.Value form. +func apply(f func(reflect.Value) reflect.Value, val reflect.Value) reflect.Value { + if !val.IsValid() { + return reflect.Value{} + } + + // *ast.Objects introduce cycles and are likely incorrect after + // rewrite; don't follow them but replace with nil instead + if val.Type() == objectPtrType { + return objectPtrNil + } + + // similarly for scopes: they are likely incorrect after a rewrite; + // replace them with nil + if val.Type() == scopePtrType { + return scopePtrNil + } + + switch v := reflect.Indirect(val); v.Kind() { + case reflect.Slice: + for i := 0; i < v.Len(); i++ { + e := v.Index(i) + set(e, f(e)) + } + case reflect.Struct: + for i := 0; i < v.NumField(); i++ { + e := v.Field(i) + set(e, f(e)) + } + case reflect.Interface: + e := v.Elem() + set(v, f(e)) + } + return val +} + +func isWildcard(s string) bool { + rune, size := utf8.DecodeRuneInString(s) + return size == len(s) && unicode.IsLower(rune) +} + +// match returns true if pattern matches val, +// recording wildcard submatches in m. +// If m == nil, match checks whether pattern == val. +func match(m map[string]reflect.Value, pattern, val reflect.Value) bool { + // Wildcard matches any expression. If it appears multiple + // times in the pattern, it must match the same expression + // each time. + if m != nil && pattern.IsValid() && pattern.Type() == identType { + name := pattern.Interface().(*ast.Ident).Name + if isWildcard(name) && val.IsValid() { + // wildcards only match valid (non-nil) expressions. + if _, ok := val.Interface().(ast.Expr); ok && !val.IsNil() { + if old, ok := m[name]; ok { + return match(nil, old, val) + } + m[name] = val + return true + } + } + } + + // Otherwise, pattern and val must match recursively. + if !pattern.IsValid() || !val.IsValid() { + return !pattern.IsValid() && !val.IsValid() + } + if pattern.Type() != val.Type() { + return false + } + + // Special cases. + switch pattern.Type() { + case identType: + // For identifiers, only the names need to match + // (and none of the other *ast.Object information). + // This is a common case, handle it all here instead + // of recursing down any further via reflection. + p := pattern.Interface().(*ast.Ident) + v := val.Interface().(*ast.Ident) + return p == nil && v == nil || p != nil && v != nil && p.Name == v.Name + case objectPtrType, positionType: + // object pointers and token positions always match + return true + case callExprType: + // For calls, the Ellipsis fields (token.Position) must + // match since that is how f(x) and f(x...) are different. + // Check them here but fall through for the remaining fields. + p := pattern.Interface().(*ast.CallExpr) + v := val.Interface().(*ast.CallExpr) + if p.Ellipsis.IsValid() != v.Ellipsis.IsValid() { + return false + } + } + + p := reflect.Indirect(pattern) + v := reflect.Indirect(val) + if !p.IsValid() || !v.IsValid() { + return !p.IsValid() && !v.IsValid() + } + + switch p.Kind() { + case reflect.Slice: + if p.Len() != v.Len() { + return false + } + for i := 0; i < p.Len(); i++ { + if !match(m, p.Index(i), v.Index(i)) { + return false + } + } + return true + + case reflect.Struct: + for i := 0; i < p.NumField(); i++ { + if !match(m, p.Field(i), v.Field(i)) { + return false + } + } + return true + + case reflect.Interface: + return match(m, p.Elem(), v.Elem()) + } + + // Handle token integers, etc. + return p.Interface() == v.Interface() +} + +// subst returns a copy of pattern with values from m substituted in place +// of wildcards and pos used as the position of tokens from the pattern. +// if m == nil, subst returns a copy of pattern and doesn't change the line +// number information. +func subst(m map[string]reflect.Value, pattern reflect.Value, pos reflect.Value) reflect.Value { + if !pattern.IsValid() { + return reflect.Value{} + } + + // Wildcard gets replaced with map value. + if m != nil && pattern.Type() == identType { + name := pattern.Interface().(*ast.Ident).Name + if isWildcard(name) { + if old, ok := m[name]; ok { + return subst(nil, old, reflect.Value{}) + } + } + } + + if pos.IsValid() && pattern.Type() == positionType { + // use new position only if old position was valid in the first place + if old := pattern.Interface().(token.Pos); !old.IsValid() { + return pattern + } + return pos + } + + // Otherwise copy. + switch p := pattern; p.Kind() { + case reflect.Slice: + v := reflect.MakeSlice(p.Type(), p.Len(), p.Len()) + for i := 0; i < p.Len(); i++ { + v.Index(i).Set(subst(m, p.Index(i), pos)) + } + return v + + case reflect.Struct: + v := reflect.New(p.Type()).Elem() + for i := 0; i < p.NumField(); i++ { + v.Field(i).Set(subst(m, p.Field(i), pos)) + } + return v + + case reflect.Ptr: + v := reflect.New(p.Type()).Elem() + if elem := p.Elem(); elem.IsValid() { + v.Set(subst(m, elem, pos).Addr()) + } + return v + + case reflect.Interface: + v := reflect.New(p.Type()).Elem() + if elem := p.Elem(); elem.IsValid() { + v.Set(subst(m, elem, pos)) + } + return v + } + + return pattern +} diff --git a/vendor/github.com/marten-seemann/qtls/generate_cert.go b/vendor/github.com/marten-seemann/qtls/generate_cert.go new file mode 100644 index 000000000..8d012be75 --- /dev/null +++ b/vendor/github.com/marten-seemann/qtls/generate_cert.go @@ -0,0 +1,169 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +// Generate a self-signed X.509 certificate for a TLS server. Outputs to +// 'cert.pem' and 'key.pem' and will overwrite existing files. + +package main + +import ( + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "flag" + "fmt" + "log" + "math/big" + "net" + "os" + "strings" + "time" +) + +var ( + host = flag.String("host", "", "Comma-separated hostnames and IPs to generate a certificate for") + validFrom = flag.String("start-date", "", "Creation date formatted as Jan 1 15:04:05 2011") + validFor = flag.Duration("duration", 365*24*time.Hour, "Duration that certificate is valid for") + isCA = flag.Bool("ca", false, "whether this cert should be its own Certificate Authority") + rsaBits = flag.Int("rsa-bits", 2048, "Size of RSA key to generate. Ignored if --ecdsa-curve is set") + ecdsaCurve = flag.String("ecdsa-curve", "", "ECDSA curve to use to generate a key. Valid values are P224, P256 (recommended), P384, P521") +) + +func publicKey(priv interface{}) interface{} { + switch k := priv.(type) { + case *rsa.PrivateKey: + return &k.PublicKey + case *ecdsa.PrivateKey: + return &k.PublicKey + default: + return nil + } +} + +func pemBlockForKey(priv interface{}) *pem.Block { + switch k := priv.(type) { + case *rsa.PrivateKey: + return &pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(k)} + case *ecdsa.PrivateKey: + b, err := x509.MarshalECPrivateKey(k) + if err != nil { + fmt.Fprintf(os.Stderr, "Unable to marshal ECDSA private key: %v", err) + os.Exit(2) + } + return &pem.Block{Type: "EC PRIVATE KEY", Bytes: b} + default: + return nil + } +} + +func main() { + flag.Parse() + + if len(*host) == 0 { + log.Fatalf("Missing required --host parameter") + } + + var priv interface{} + var err error + switch *ecdsaCurve { + case "": + priv, err = rsa.GenerateKey(rand.Reader, *rsaBits) + case "P224": + priv, err = ecdsa.GenerateKey(elliptic.P224(), rand.Reader) + case "P256": + priv, err = ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + case "P384": + priv, err = ecdsa.GenerateKey(elliptic.P384(), rand.Reader) + case "P521": + priv, err = ecdsa.GenerateKey(elliptic.P521(), rand.Reader) + default: + fmt.Fprintf(os.Stderr, "Unrecognized elliptic curve: %q", *ecdsaCurve) + os.Exit(1) + } + if err != nil { + log.Fatalf("failed to generate private key: %s", err) + } + + var notBefore time.Time + if len(*validFrom) == 0 { + notBefore = time.Now() + } else { + notBefore, err = time.Parse("Jan 2 15:04:05 2006", *validFrom) + if err != nil { + fmt.Fprintf(os.Stderr, "Failed to parse creation date: %s\n", err) + os.Exit(1) + } + } + + notAfter := notBefore.Add(*validFor) + + serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 128) + serialNumber, err := rand.Int(rand.Reader, serialNumberLimit) + if err != nil { + log.Fatalf("failed to generate serial number: %s", err) + } + + template := x509.Certificate{ + SerialNumber: serialNumber, + Subject: pkix.Name{ + Organization: []string{"Acme Co"}, + }, + NotBefore: notBefore, + NotAfter: notAfter, + + KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + BasicConstraintsValid: true, + } + + hosts := strings.Split(*host, ",") + for _, h := range hosts { + if ip := net.ParseIP(h); ip != nil { + template.IPAddresses = append(template.IPAddresses, ip) + } else { + template.DNSNames = append(template.DNSNames, h) + } + } + + if *isCA { + template.IsCA = true + template.KeyUsage |= x509.KeyUsageCertSign + } + + derBytes, err := x509.CreateCertificate(rand.Reader, &template, &template, publicKey(priv), priv) + if err != nil { + log.Fatalf("Failed to create certificate: %s", err) + } + + certOut, err := os.Create("cert.pem") + if err != nil { + log.Fatalf("failed to open cert.pem for writing: %s", err) + } + if err := pem.Encode(certOut, &pem.Block{Type: "CERTIFICATE", Bytes: derBytes}); err != nil { + log.Fatalf("failed to write data to cert.pem: %s", err) + } + if err := certOut.Close(); err != nil { + log.Fatalf("error closing cert.pem: %s", err) + } + log.Print("wrote cert.pem\n") + + keyOut, err := os.OpenFile("key.pem", os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600) + if err != nil { + log.Print("failed to open key.pem for writing:", err) + return + } + if err := pem.Encode(keyOut, pemBlockForKey(priv)); err != nil { + log.Fatalf("failed to write data to key.pem: %s", err) + } + if err := keyOut.Close(); err != nil { + log.Fatalf("error closing key.pem: %s", err) + } + log.Print("wrote key.pem\n") +} diff --git a/vendor/github.com/miekg/dns/duplicate_generate.go b/vendor/github.com/miekg/dns/duplicate_generate.go new file mode 100644 index 000000000..9b7a71b16 --- /dev/null +++ b/vendor/github.com/miekg/dns/duplicate_generate.go @@ -0,0 +1,144 @@ +//+build ignore + +// types_generate.go is meant to run with go generate. It will use +// go/{importer,types} to track down all the RR struct types. Then for each type +// it will generate conversion tables (TypeToRR and TypeToString) and banal +// methods (len, Header, copy) based on the struct tags. The generated source is +// written to ztypes.go, and is meant to be checked into git. +package main + +import ( + "bytes" + "fmt" + "go/format" + "go/importer" + "go/types" + "log" + "os" +) + +var packageHdr = ` +// Code generated by "go run duplicate_generate.go"; DO NOT EDIT. + +package dns + +` + +func getTypeStruct(t types.Type, scope *types.Scope) (*types.Struct, bool) { + st, ok := t.Underlying().(*types.Struct) + if !ok { + return nil, false + } + if st.Field(0).Type() == scope.Lookup("RR_Header").Type() { + return st, false + } + if st.Field(0).Anonymous() { + st, _ := getTypeStruct(st.Field(0).Type(), scope) + return st, true + } + return nil, false +} + +func main() { + // Import and type-check the package + pkg, err := importer.Default().Import("github.com/miekg/dns") + fatalIfErr(err) + scope := pkg.Scope() + + // Collect actual types (*X) + var namedTypes []string + for _, name := range scope.Names() { + o := scope.Lookup(name) + if o == nil || !o.Exported() { + continue + } + + if st, _ := getTypeStruct(o.Type(), scope); st == nil { + continue + } + + if name == "PrivateRR" || name == "OPT" { + continue + } + + namedTypes = append(namedTypes, o.Name()) + } + + b := &bytes.Buffer{} + b.WriteString(packageHdr) + + // Generate the duplicate check for each type. + fmt.Fprint(b, "// isDuplicate() functions\n\n") + for _, name := range namedTypes { + + o := scope.Lookup(name) + st, isEmbedded := getTypeStruct(o.Type(), scope) + if isEmbedded { + continue + } + fmt.Fprintf(b, "func (r1 *%s) isDuplicate(_r2 RR) bool {\n", name) + fmt.Fprintf(b, "r2, ok := _r2.(*%s)\n", name) + fmt.Fprint(b, "if !ok { return false }\n") + fmt.Fprint(b, "_ = r2\n") + for i := 1; i < st.NumFields(); i++ { + field := st.Field(i).Name() + o2 := func(s string) { fmt.Fprintf(b, s+"\n", field, field) } + o3 := func(s string) { fmt.Fprintf(b, s+"\n", field, field, field) } + + // For some reason, a and aaaa don't pop up as *types.Slice here (mostly like because the are + // *indirectly* defined as a slice in the net package). + if _, ok := st.Field(i).Type().(*types.Slice); ok { + o2("if len(r1.%s) != len(r2.%s) {\nreturn false\n}") + + if st.Tag(i) == `dns:"cdomain-name"` || st.Tag(i) == `dns:"domain-name"` { + o3(`for i := 0; i < len(r1.%s); i++ { + if !isDuplicateName(r1.%s[i], r2.%s[i]) { + return false + } + }`) + + continue + } + + o3(`for i := 0; i < len(r1.%s); i++ { + if r1.%s[i] != r2.%s[i] { + return false + } + }`) + + continue + } + + switch st.Tag(i) { + case `dns:"-"`: + // ignored + case `dns:"a"`, `dns:"aaaa"`: + o2("if !r1.%s.Equal(r2.%s) {\nreturn false\n}") + case `dns:"cdomain-name"`, `dns:"domain-name"`: + o2("if !isDuplicateName(r1.%s, r2.%s) {\nreturn false\n}") + default: + o2("if r1.%s != r2.%s {\nreturn false\n}") + } + } + fmt.Fprintf(b, "return true\n}\n\n") + } + + // gofmt + res, err := format.Source(b.Bytes()) + if err != nil { + b.WriteTo(os.Stderr) + log.Fatal(err) + } + + // write result + f, err := os.Create("zduplicate.go") + fatalIfErr(err) + defer f.Close() + f.Write(res) +} + +func fatalIfErr(err error) { + if err != nil { + log.Fatal(err) + } +} diff --git a/vendor/github.com/miekg/dns/msg_generate.go b/vendor/github.com/miekg/dns/msg_generate.go new file mode 100644 index 000000000..721a0fce3 --- /dev/null +++ b/vendor/github.com/miekg/dns/msg_generate.go @@ -0,0 +1,328 @@ +//+build ignore + +// msg_generate.go is meant to run with go generate. It will use +// go/{importer,types} to track down all the RR struct types. Then for each type +// it will generate pack/unpack methods based on the struct tags. The generated source is +// written to zmsg.go, and is meant to be checked into git. +package main + +import ( + "bytes" + "fmt" + "go/format" + "go/importer" + "go/types" + "log" + "os" + "strings" +) + +var packageHdr = ` +// Code generated by "go run msg_generate.go"; DO NOT EDIT. + +package dns + +` + +// getTypeStruct will take a type and the package scope, and return the +// (innermost) struct if the type is considered a RR type (currently defined as +// those structs beginning with a RR_Header, could be redefined as implementing +// the RR interface). The bool return value indicates if embedded structs were +// resolved. +func getTypeStruct(t types.Type, scope *types.Scope) (*types.Struct, bool) { + st, ok := t.Underlying().(*types.Struct) + if !ok { + return nil, false + } + if st.Field(0).Type() == scope.Lookup("RR_Header").Type() { + return st, false + } + if st.Field(0).Anonymous() { + st, _ := getTypeStruct(st.Field(0).Type(), scope) + return st, true + } + return nil, false +} + +func main() { + // Import and type-check the package + pkg, err := importer.Default().Import("github.com/miekg/dns") + fatalIfErr(err) + scope := pkg.Scope() + + // Collect actual types (*X) + var namedTypes []string + for _, name := range scope.Names() { + o := scope.Lookup(name) + if o == nil || !o.Exported() { + continue + } + if st, _ := getTypeStruct(o.Type(), scope); st == nil { + continue + } + if name == "PrivateRR" { + continue + } + + // Check if corresponding TypeX exists + if scope.Lookup("Type"+o.Name()) == nil && o.Name() != "RFC3597" { + log.Fatalf("Constant Type%s does not exist.", o.Name()) + } + + namedTypes = append(namedTypes, o.Name()) + } + + b := &bytes.Buffer{} + b.WriteString(packageHdr) + + fmt.Fprint(b, "// pack*() functions\n\n") + for _, name := range namedTypes { + o := scope.Lookup(name) + st, _ := getTypeStruct(o.Type(), scope) + + fmt.Fprintf(b, "func (rr *%s) pack(msg []byte, off int, compression compressionMap, compress bool) (off1 int, err error) {\n", name) + for i := 1; i < st.NumFields(); i++ { + o := func(s string) { + fmt.Fprintf(b, s, st.Field(i).Name()) + fmt.Fprint(b, `if err != nil { +return off, err +} +`) + } + + if _, ok := st.Field(i).Type().(*types.Slice); ok { + switch st.Tag(i) { + case `dns:"-"`: // ignored + case `dns:"txt"`: + o("off, err = packStringTxt(rr.%s, msg, off)\n") + case `dns:"opt"`: + o("off, err = packDataOpt(rr.%s, msg, off)\n") + case `dns:"nsec"`: + o("off, err = packDataNsec(rr.%s, msg, off)\n") + case `dns:"domain-name"`: + o("off, err = packDataDomainNames(rr.%s, msg, off, compression, false)\n") + default: + log.Fatalln(name, st.Field(i).Name(), st.Tag(i)) + } + continue + } + + switch { + case st.Tag(i) == `dns:"-"`: // ignored + case st.Tag(i) == `dns:"cdomain-name"`: + o("off, err = packDomainName(rr.%s, msg, off, compression, compress)\n") + case st.Tag(i) == `dns:"domain-name"`: + o("off, err = packDomainName(rr.%s, msg, off, compression, false)\n") + case st.Tag(i) == `dns:"a"`: + o("off, err = packDataA(rr.%s, msg, off)\n") + case st.Tag(i) == `dns:"aaaa"`: + o("off, err = packDataAAAA(rr.%s, msg, off)\n") + case st.Tag(i) == `dns:"uint48"`: + o("off, err = packUint48(rr.%s, msg, off)\n") + case st.Tag(i) == `dns:"txt"`: + o("off, err = packString(rr.%s, msg, off)\n") + + case strings.HasPrefix(st.Tag(i), `dns:"size-base32`): // size-base32 can be packed just like base32 + fallthrough + case st.Tag(i) == `dns:"base32"`: + o("off, err = packStringBase32(rr.%s, msg, off)\n") + + case strings.HasPrefix(st.Tag(i), `dns:"size-base64`): // size-base64 can be packed just like base64 + fallthrough + case st.Tag(i) == `dns:"base64"`: + o("off, err = packStringBase64(rr.%s, msg, off)\n") + + case strings.HasPrefix(st.Tag(i), `dns:"size-hex:SaltLength`): + // directly write instead of using o() so we get the error check in the correct place + field := st.Field(i).Name() + fmt.Fprintf(b, `// Only pack salt if value is not "-", i.e. empty +if rr.%s != "-" { + off, err = packStringHex(rr.%s, msg, off) + if err != nil { + return off, err + } +} +`, field, field) + continue + case strings.HasPrefix(st.Tag(i), `dns:"size-hex`): // size-hex can be packed just like hex + fallthrough + case st.Tag(i) == `dns:"hex"`: + o("off, err = packStringHex(rr.%s, msg, off)\n") + case st.Tag(i) == `dns:"any"`: + o("off, err = packStringAny(rr.%s, msg, off)\n") + case st.Tag(i) == `dns:"octet"`: + o("off, err = packStringOctet(rr.%s, msg, off)\n") + case st.Tag(i) == "": + switch st.Field(i).Type().(*types.Basic).Kind() { + case types.Uint8: + o("off, err = packUint8(rr.%s, msg, off)\n") + case types.Uint16: + o("off, err = packUint16(rr.%s, msg, off)\n") + case types.Uint32: + o("off, err = packUint32(rr.%s, msg, off)\n") + case types.Uint64: + o("off, err = packUint64(rr.%s, msg, off)\n") + case types.String: + o("off, err = packString(rr.%s, msg, off)\n") + default: + log.Fatalln(name, st.Field(i).Name()) + } + default: + log.Fatalln(name, st.Field(i).Name(), st.Tag(i)) + } + } + fmt.Fprintln(b, "return off, nil }\n") + } + + fmt.Fprint(b, "// unpack*() functions\n\n") + for _, name := range namedTypes { + o := scope.Lookup(name) + st, _ := getTypeStruct(o.Type(), scope) + + fmt.Fprintf(b, "func (rr *%s) unpack(msg []byte, off int) (off1 int, err error) {\n", name) + fmt.Fprint(b, `rdStart := off +_ = rdStart + +`) + for i := 1; i < st.NumFields(); i++ { + o := func(s string) { + fmt.Fprintf(b, s, st.Field(i).Name()) + fmt.Fprint(b, `if err != nil { +return off, err +} +`) + } + + // size-* are special, because they reference a struct member we should use for the length. + if strings.HasPrefix(st.Tag(i), `dns:"size-`) { + structMember := structMember(st.Tag(i)) + structTag := structTag(st.Tag(i)) + switch structTag { + case "hex": + fmt.Fprintf(b, "rr.%s, off, err = unpackStringHex(msg, off, off + int(rr.%s))\n", st.Field(i).Name(), structMember) + case "base32": + fmt.Fprintf(b, "rr.%s, off, err = unpackStringBase32(msg, off, off + int(rr.%s))\n", st.Field(i).Name(), structMember) + case "base64": + fmt.Fprintf(b, "rr.%s, off, err = unpackStringBase64(msg, off, off + int(rr.%s))\n", st.Field(i).Name(), structMember) + default: + log.Fatalln(name, st.Field(i).Name(), st.Tag(i)) + } + fmt.Fprint(b, `if err != nil { +return off, err +} +`) + continue + } + + if _, ok := st.Field(i).Type().(*types.Slice); ok { + switch st.Tag(i) { + case `dns:"-"`: // ignored + case `dns:"txt"`: + o("rr.%s, off, err = unpackStringTxt(msg, off)\n") + case `dns:"opt"`: + o("rr.%s, off, err = unpackDataOpt(msg, off)\n") + case `dns:"nsec"`: + o("rr.%s, off, err = unpackDataNsec(msg, off)\n") + case `dns:"domain-name"`: + o("rr.%s, off, err = unpackDataDomainNames(msg, off, rdStart + int(rr.Hdr.Rdlength))\n") + default: + log.Fatalln(name, st.Field(i).Name(), st.Tag(i)) + } + continue + } + + switch st.Tag(i) { + case `dns:"-"`: // ignored + case `dns:"cdomain-name"`: + fallthrough + case `dns:"domain-name"`: + o("rr.%s, off, err = UnpackDomainName(msg, off)\n") + case `dns:"a"`: + o("rr.%s, off, err = unpackDataA(msg, off)\n") + case `dns:"aaaa"`: + o("rr.%s, off, err = unpackDataAAAA(msg, off)\n") + case `dns:"uint48"`: + o("rr.%s, off, err = unpackUint48(msg, off)\n") + case `dns:"txt"`: + o("rr.%s, off, err = unpackString(msg, off)\n") + case `dns:"base32"`: + o("rr.%s, off, err = unpackStringBase32(msg, off, rdStart + int(rr.Hdr.Rdlength))\n") + case `dns:"base64"`: + o("rr.%s, off, err = unpackStringBase64(msg, off, rdStart + int(rr.Hdr.Rdlength))\n") + case `dns:"hex"`: + o("rr.%s, off, err = unpackStringHex(msg, off, rdStart + int(rr.Hdr.Rdlength))\n") + case `dns:"any"`: + o("rr.%s, off, err = unpackStringAny(msg, off, rdStart + int(rr.Hdr.Rdlength))\n") + case `dns:"octet"`: + o("rr.%s, off, err = unpackStringOctet(msg, off)\n") + case "": + switch st.Field(i).Type().(*types.Basic).Kind() { + case types.Uint8: + o("rr.%s, off, err = unpackUint8(msg, off)\n") + case types.Uint16: + o("rr.%s, off, err = unpackUint16(msg, off)\n") + case types.Uint32: + o("rr.%s, off, err = unpackUint32(msg, off)\n") + case types.Uint64: + o("rr.%s, off, err = unpackUint64(msg, off)\n") + case types.String: + o("rr.%s, off, err = unpackString(msg, off)\n") + default: + log.Fatalln(name, st.Field(i).Name()) + } + default: + log.Fatalln(name, st.Field(i).Name(), st.Tag(i)) + } + // If we've hit len(msg) we return without error. + if i < st.NumFields()-1 { + fmt.Fprintf(b, `if off == len(msg) { +return off, nil + } +`) + } + } + fmt.Fprintf(b, "return off, nil }\n\n") + } + + // gofmt + res, err := format.Source(b.Bytes()) + if err != nil { + b.WriteTo(os.Stderr) + log.Fatal(err) + } + + // write result + f, err := os.Create("zmsg.go") + fatalIfErr(err) + defer f.Close() + f.Write(res) +} + +// structMember will take a tag like dns:"size-base32:SaltLength" and return the last part of this string. +func structMember(s string) string { + fields := strings.Split(s, ":") + if len(fields) == 0 { + return "" + } + f := fields[len(fields)-1] + // f should have a closing " + if len(f) > 1 { + return f[:len(f)-1] + } + return f +} + +// structTag will take a tag like dns:"size-base32:SaltLength" and return base32. +func structTag(s string) string { + fields := strings.Split(s, ":") + if len(fields) < 2 { + return "" + } + return fields[1][len("\"size-"):] +} + +func fatalIfErr(err error) { + if err != nil { + log.Fatal(err) + } +} diff --git a/vendor/github.com/miekg/dns/types_generate.go b/vendor/github.com/miekg/dns/types_generate.go new file mode 100644 index 000000000..cbb4a00c1 --- /dev/null +++ b/vendor/github.com/miekg/dns/types_generate.go @@ -0,0 +1,287 @@ +//+build ignore + +// types_generate.go is meant to run with go generate. It will use +// go/{importer,types} to track down all the RR struct types. Then for each type +// it will generate conversion tables (TypeToRR and TypeToString) and banal +// methods (len, Header, copy) based on the struct tags. The generated source is +// written to ztypes.go, and is meant to be checked into git. +package main + +import ( + "bytes" + "fmt" + "go/format" + "go/importer" + "go/types" + "log" + "os" + "strings" + "text/template" +) + +var skipLen = map[string]struct{}{ + "NSEC": {}, + "NSEC3": {}, + "OPT": {}, + "CSYNC": {}, +} + +var packageHdr = ` +// Code generated by "go run types_generate.go"; DO NOT EDIT. + +package dns + +import ( + "encoding/base64" + "net" +) + +` + +var TypeToRR = template.Must(template.New("TypeToRR").Parse(` +// TypeToRR is a map of constructors for each RR type. +var TypeToRR = map[uint16]func() RR{ +{{range .}}{{if ne . "RFC3597"}} Type{{.}}: func() RR { return new({{.}}) }, +{{end}}{{end}} } + +`)) + +var typeToString = template.Must(template.New("typeToString").Parse(` +// TypeToString is a map of strings for each RR type. +var TypeToString = map[uint16]string{ +{{range .}}{{if ne . "NSAPPTR"}} Type{{.}}: "{{.}}", +{{end}}{{end}} TypeNSAPPTR: "NSAP-PTR", +} + +`)) + +var headerFunc = template.Must(template.New("headerFunc").Parse(` +{{range .}} func (rr *{{.}}) Header() *RR_Header { return &rr.Hdr } +{{end}} + +`)) + +// getTypeStruct will take a type and the package scope, and return the +// (innermost) struct if the type is considered a RR type (currently defined as +// those structs beginning with a RR_Header, could be redefined as implementing +// the RR interface). The bool return value indicates if embedded structs were +// resolved. +func getTypeStruct(t types.Type, scope *types.Scope) (*types.Struct, bool) { + st, ok := t.Underlying().(*types.Struct) + if !ok { + return nil, false + } + if st.Field(0).Type() == scope.Lookup("RR_Header").Type() { + return st, false + } + if st.Field(0).Anonymous() { + st, _ := getTypeStruct(st.Field(0).Type(), scope) + return st, true + } + return nil, false +} + +func main() { + // Import and type-check the package + pkg, err := importer.Default().Import("github.com/miekg/dns") + fatalIfErr(err) + scope := pkg.Scope() + + // Collect constants like TypeX + var numberedTypes []string + for _, name := range scope.Names() { + o := scope.Lookup(name) + if o == nil || !o.Exported() { + continue + } + b, ok := o.Type().(*types.Basic) + if !ok || b.Kind() != types.Uint16 { + continue + } + if !strings.HasPrefix(o.Name(), "Type") { + continue + } + name := strings.TrimPrefix(o.Name(), "Type") + if name == "PrivateRR" { + continue + } + numberedTypes = append(numberedTypes, name) + } + + // Collect actual types (*X) + var namedTypes []string + for _, name := range scope.Names() { + o := scope.Lookup(name) + if o == nil || !o.Exported() { + continue + } + if st, _ := getTypeStruct(o.Type(), scope); st == nil { + continue + } + if name == "PrivateRR" { + continue + } + + // Check if corresponding TypeX exists + if scope.Lookup("Type"+o.Name()) == nil && o.Name() != "RFC3597" { + log.Fatalf("Constant Type%s does not exist.", o.Name()) + } + + namedTypes = append(namedTypes, o.Name()) + } + + b := &bytes.Buffer{} + b.WriteString(packageHdr) + + // Generate TypeToRR + fatalIfErr(TypeToRR.Execute(b, namedTypes)) + + // Generate typeToString + fatalIfErr(typeToString.Execute(b, numberedTypes)) + + // Generate headerFunc + fatalIfErr(headerFunc.Execute(b, namedTypes)) + + // Generate len() + fmt.Fprint(b, "// len() functions\n") + for _, name := range namedTypes { + if _, ok := skipLen[name]; ok { + continue + } + o := scope.Lookup(name) + st, isEmbedded := getTypeStruct(o.Type(), scope) + if isEmbedded { + continue + } + fmt.Fprintf(b, "func (rr *%s) len(off int, compression map[string]struct{}) int {\n", name) + fmt.Fprintf(b, "l := rr.Hdr.len(off, compression)\n") + for i := 1; i < st.NumFields(); i++ { + o := func(s string) { fmt.Fprintf(b, s, st.Field(i).Name()) } + + if _, ok := st.Field(i).Type().(*types.Slice); ok { + switch st.Tag(i) { + case `dns:"-"`: + // ignored + case `dns:"cdomain-name"`: + o("for _, x := range rr.%s { l += domainNameLen(x, off+l, compression, true) }\n") + case `dns:"domain-name"`: + o("for _, x := range rr.%s { l += domainNameLen(x, off+l, compression, false) }\n") + case `dns:"txt"`: + o("for _, x := range rr.%s { l += len(x) + 1 }\n") + default: + log.Fatalln(name, st.Field(i).Name(), st.Tag(i)) + } + continue + } + + switch { + case st.Tag(i) == `dns:"-"`: + // ignored + case st.Tag(i) == `dns:"cdomain-name"`: + o("l += domainNameLen(rr.%s, off+l, compression, true)\n") + case st.Tag(i) == `dns:"domain-name"`: + o("l += domainNameLen(rr.%s, off+l, compression, false)\n") + case st.Tag(i) == `dns:"octet"`: + o("l += len(rr.%s)\n") + case strings.HasPrefix(st.Tag(i), `dns:"size-base64`): + fallthrough + case st.Tag(i) == `dns:"base64"`: + o("l += base64.StdEncoding.DecodedLen(len(rr.%s))\n") + case strings.HasPrefix(st.Tag(i), `dns:"size-hex:`): // this has an extra field where the length is stored + o("l += len(rr.%s)/2\n") + case strings.HasPrefix(st.Tag(i), `dns:"size-hex`): + fallthrough + case st.Tag(i) == `dns:"hex"`: + o("l += len(rr.%s)/2 + 1\n") + case st.Tag(i) == `dns:"any"`: + o("l += len(rr.%s)\n") + case st.Tag(i) == `dns:"a"`: + o("if len(rr.%s) != 0 { l += net.IPv4len }\n") + case st.Tag(i) == `dns:"aaaa"`: + o("if len(rr.%s) != 0 { l += net.IPv6len }\n") + case st.Tag(i) == `dns:"txt"`: + o("for _, t := range rr.%s { l += len(t) + 1 }\n") + case st.Tag(i) == `dns:"uint48"`: + o("l += 6 // %s\n") + case st.Tag(i) == "": + switch st.Field(i).Type().(*types.Basic).Kind() { + case types.Uint8: + o("l++ // %s\n") + case types.Uint16: + o("l += 2 // %s\n") + case types.Uint32: + o("l += 4 // %s\n") + case types.Uint64: + o("l += 8 // %s\n") + case types.String: + o("l += len(rr.%s) + 1\n") + default: + log.Fatalln(name, st.Field(i).Name()) + } + default: + log.Fatalln(name, st.Field(i).Name(), st.Tag(i)) + } + } + fmt.Fprintf(b, "return l }\n") + } + + // Generate copy() + fmt.Fprint(b, "// copy() functions\n") + for _, name := range namedTypes { + o := scope.Lookup(name) + st, isEmbedded := getTypeStruct(o.Type(), scope) + if isEmbedded { + continue + } + fmt.Fprintf(b, "func (rr *%s) copy() RR {\n", name) + fields := []string{"rr.Hdr"} + for i := 1; i < st.NumFields(); i++ { + f := st.Field(i).Name() + if sl, ok := st.Field(i).Type().(*types.Slice); ok { + t := sl.Underlying().String() + t = strings.TrimPrefix(t, "[]") + if strings.Contains(t, ".") { + splits := strings.Split(t, ".") + t = splits[len(splits)-1] + } + // For the EDNS0 interface (used in the OPT RR), we need to call the copy method on each element. + if t == "EDNS0" { + fmt.Fprintf(b, "%s := make([]%s, len(rr.%s));\nfor i,e := range rr.%s {\n %s[i] = e.copy()\n}\n", + f, t, f, f, f) + fields = append(fields, f) + continue + } + fmt.Fprintf(b, "%s := make([]%s, len(rr.%s)); copy(%s, rr.%s)\n", + f, t, f, f, f) + fields = append(fields, f) + continue + } + if st.Field(i).Type().String() == "net.IP" { + fields = append(fields, "copyIP(rr."+f+")") + continue + } + fields = append(fields, "rr."+f) + } + fmt.Fprintf(b, "return &%s{%s}\n", name, strings.Join(fields, ",")) + fmt.Fprintf(b, "}\n") + } + + // gofmt + res, err := format.Source(b.Bytes()) + if err != nil { + b.WriteTo(os.Stderr) + log.Fatal(err) + } + + // write result + f, err := os.Create("ztypes.go") + fatalIfErr(err) + defer f.Close() + f.Write(res) +} + +func fatalIfErr(err error) { + if err != nil { + log.Fatal(err) + } +} diff --git a/vendor/github.com/open-policy-agent/opa/LICENSE b/vendor/github.com/open-policy-agent/opa/LICENSE new file mode 100644 index 000000000..8f71f43fe --- /dev/null +++ b/vendor/github.com/open-policy-agent/opa/LICENSE @@ -0,0 +1,202 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + diff --git a/vendor/github.com/open-policy-agent/opa/ast/builtins.go b/vendor/github.com/open-policy-agent/opa/ast/builtins.go new file mode 100644 index 000000000..1ab7dc0a9 --- /dev/null +++ b/vendor/github.com/open-policy-agent/opa/ast/builtins.go @@ -0,0 +1,1902 @@ +// Copyright 2016 The OPA Authors. All rights reserved. +// Use of this source code is governed by an Apache2 +// license that can be found in the LICENSE file. + +package ast + +import ( + "strings" + + "github.com/open-policy-agent/opa/types" +) + +// Builtins is the registry of built-in functions supported by OPA. +// Call RegisterBuiltin to add a new built-in. +var Builtins []*Builtin + +// RegisterBuiltin adds a new built-in function to the registry. +func RegisterBuiltin(b *Builtin) { + Builtins = append(Builtins, b) + BuiltinMap[b.Name] = b + if len(b.Infix) > 0 { + BuiltinMap[b.Infix] = b + } +} + +// DefaultBuiltins is the registry of built-in functions supported in OPA +// by default. When adding a new built-in function to OPA, update this +// list. +var DefaultBuiltins = [...]*Builtin{ + // Unification/equality ("=") + Equality, + + // Assignment (":=") + Assign, + + // Comparisons + GreaterThan, + GreaterThanEq, + LessThan, + LessThanEq, + NotEqual, + Equal, + + // Arithmetic + Plus, + Minus, + Multiply, + Divide, + Round, + Abs, + Rem, + + // Bitwise Arithmetic + BitsOr, + BitsAnd, + BitsNegate, + BitsXOr, + BitsShiftLeft, + BitsShiftRight, + + // Binary + And, + Or, + + // Aggregates + Count, + Sum, + Product, + Max, + Min, + Any, + All, + + // Arrays + ArrayConcat, + ArraySlice, + + // Conversions + ToNumber, + + // Casts (DEPRECATED) + CastObject, + CastNull, + CastBoolean, + CastString, + CastSet, + CastArray, + + // Regular Expressions + RegexMatch, + RegexSplit, + GlobsMatch, + RegexTemplateMatch, + RegexFind, + RegexFindAllStringSubmatch, + + // Sets + SetDiff, + Intersection, + Union, + + // Strings + Concat, + FormatInt, + IndexOf, + Substring, + Lower, + Upper, + Contains, + StartsWith, + EndsWith, + Split, + Replace, + ReplaceN, + Trim, + TrimLeft, + TrimPrefix, + TrimRight, + TrimSuffix, + TrimSpace, + Sprintf, + + // Encoding + JSONMarshal, + JSONUnmarshal, + Base64Encode, + Base64Decode, + Base64UrlEncode, + Base64UrlDecode, + URLQueryDecode, + URLQueryEncode, + URLQueryEncodeObject, + YAMLMarshal, + YAMLUnmarshal, + + // Object Manipulation + ObjectUnion, + ObjectRemove, + ObjectFilter, + ObjectGet, + + // JSON Object Manipulation + JSONFilter, + JSONRemove, + + // Tokens + JWTDecode, + JWTVerifyRS256, + JWTVerifyPS256, + JWTVerifyES256, + JWTVerifyHS256, + JWTDecodeVerify, + JWTEncodeSignRaw, + JWTEncodeSign, + + // Time + NowNanos, + ParseNanos, + ParseRFC3339Nanos, + ParseDurationNanos, + Date, + Clock, + Weekday, + + // Crypto + CryptoX509ParseCertificates, + CryptoMd5, + CryptoSha1, + CryptoSha256, + + // Graphs + WalkBuiltin, + + // Sort + Sort, + + // Types + IsNumber, + IsString, + IsBoolean, + IsArray, + IsSet, + IsObject, + IsNull, + TypeNameBuiltin, + + // HTTP + HTTPSend, + + // Rego + RegoParseModule, + + // OPA + OPARuntime, + + // Tracing + Trace, + + // CIDR + NetCIDROverlap, + NetCIDRIntersects, + NetCIDRContains, + NetCIDRExpand, + + // Glob + GlobMatch, + GlobQuoteMeta, + + // Units + UnitsParseBytes, +} + +// BuiltinMap provides a convenient mapping of built-in names to +// built-in definitions. +var BuiltinMap map[string]*Builtin + +// IgnoreDuringPartialEval is a set of built-in functions that should not be +// evaluated during partial evaluation. These functions are not partially +// evaluated because they are not pure. +var IgnoreDuringPartialEval = []*Builtin{ + NowNanos, + HTTPSend, +} + +/** + * Unification + */ + +// Equality represents the "=" operator. +var Equality = &Builtin{ + Name: "eq", + Infix: "=", + Decl: types.NewFunction( + types.Args(types.A, types.A), + types.B, + ), +} + +/** + * Assignment + */ + +// Assign represents the assignment (":=") operator. +var Assign = &Builtin{ + Name: "assign", + Infix: ":=", + Decl: types.NewFunction( + types.Args(types.A, types.A), + types.B, + ), +} + +/** + * Comparisons + */ + +// GreaterThan represents the ">" comparison operator. +var GreaterThan = &Builtin{ + Name: "gt", + Infix: ">", + Decl: types.NewFunction( + types.Args(types.A, types.A), + types.B, + ), +} + +// GreaterThanEq represents the ">=" comparison operator. +var GreaterThanEq = &Builtin{ + Name: "gte", + Infix: ">=", + Decl: types.NewFunction( + types.Args(types.A, types.A), + types.B, + ), +} + +// LessThan represents the "<" comparison operator. +var LessThan = &Builtin{ + Name: "lt", + Infix: "<", + Decl: types.NewFunction( + types.Args(types.A, types.A), + types.B, + ), +} + +// LessThanEq represents the "<=" comparison operator. +var LessThanEq = &Builtin{ + Name: "lte", + Infix: "<=", + Decl: types.NewFunction( + types.Args(types.A, types.A), + types.B, + ), +} + +// NotEqual represents the "!=" comparison operator. +var NotEqual = &Builtin{ + Name: "neq", + Infix: "!=", + Decl: types.NewFunction( + types.Args(types.A, types.A), + types.B, + ), +} + +// Equal represents the "==" comparison operator. +var Equal = &Builtin{ + Name: "equal", + Infix: "==", + Decl: types.NewFunction( + types.Args(types.A, types.A), + types.B, + ), +} + +/** + * Arithmetic + */ + +// Plus adds two numbers together. +var Plus = &Builtin{ + Name: "plus", + Infix: "+", + Decl: types.NewFunction( + types.Args(types.N, types.N), + types.N, + ), +} + +// Minus subtracts the second number from the first number or computes the diff +// between two sets. +var Minus = &Builtin{ + Name: "minus", + Infix: "-", + Decl: types.NewFunction( + types.Args( + types.NewAny(types.N, types.NewSet(types.A)), + types.NewAny(types.N, types.NewSet(types.A)), + ), + types.NewAny(types.N, types.NewSet(types.A)), + ), +} + +// Multiply multiplies two numbers together. +var Multiply = &Builtin{ + Name: "mul", + Infix: "*", + Decl: types.NewFunction( + types.Args(types.N, types.N), + types.N, + ), +} + +// Divide divides the first number by the second number. +var Divide = &Builtin{ + Name: "div", + Infix: "/", + Decl: types.NewFunction( + types.Args(types.N, types.N), + types.N, + ), +} + +// Round rounds the number up to the nearest integer. +var Round = &Builtin{ + Name: "round", + Decl: types.NewFunction( + types.Args(types.N), + types.N, + ), +} + +// Abs returns the number without its sign. +var Abs = &Builtin{ + Name: "abs", + Decl: types.NewFunction( + types.Args(types.N), + types.N, + ), +} + +// Rem returns the remainder for x%y for y != 0. +var Rem = &Builtin{ + Name: "rem", + Infix: "%", + Decl: types.NewFunction( + types.Args(types.N, types.N), + types.N, + ), +} + +/** + * Bitwise + */ + +// BitsOr returns the bitwise "or" of two integers. +var BitsOr = &Builtin{ + Name: "bits.or", + Decl: types.NewFunction( + types.Args(types.N, types.N), + types.N, + ), +} + +// BitsAnd returns the bitwise "and" of two integers. +var BitsAnd = &Builtin{ + Name: "bits.and", + Decl: types.NewFunction( + types.Args(types.N, types.N), + types.N, + ), +} + +// BitsNegate returns the bitwise "negation" of an integer (i.e. flips each +// bit). +var BitsNegate = &Builtin{ + Name: "bits.negate", + Decl: types.NewFunction( + types.Args(types.N), + types.N, + ), +} + +// BitsXOr returns the bitwise "exclusive-or" of two integers. +var BitsXOr = &Builtin{ + Name: "bits.xor", + Decl: types.NewFunction( + types.Args(types.N, types.N), + types.N, + ), +} + +// BitsShiftLeft returns a new integer with its bits shifted some value to the +// left. +var BitsShiftLeft = &Builtin{ + Name: "bits.lsh", + Decl: types.NewFunction( + types.Args(types.N, types.N), + types.N, + ), +} + +// BitsShiftRight returns a new integer with its bits shifted some value to the +// right. +var BitsShiftRight = &Builtin{ + Name: "bits.rsh", + Decl: types.NewFunction( + types.Args(types.N, types.N), + types.N, + ), +} + +/** + * Sets + */ + +// And performs an intersection operation on sets. +var And = &Builtin{ + Name: "and", + Infix: "&", + Decl: types.NewFunction( + types.Args( + types.NewSet(types.A), + types.NewSet(types.A), + ), + types.NewSet(types.A), + ), +} + +// Or performs a union operation on sets. +var Or = &Builtin{ + Name: "or", + Infix: "|", + Decl: types.NewFunction( + types.Args( + types.NewSet(types.A), + types.NewSet(types.A), + ), + types.NewSet(types.A), + ), +} + +/** + * Aggregates + */ + +// Count takes a collection or string and counts the number of elements in it. +var Count = &Builtin{ + Name: "count", + Decl: types.NewFunction( + types.Args( + types.NewAny( + types.NewSet(types.A), + types.NewArray(nil, types.A), + types.NewObject(nil, types.NewDynamicProperty(types.A, types.A)), + types.S, + ), + ), + types.N, + ), +} + +// Sum takes an array or set of numbers and sums them. +var Sum = &Builtin{ + Name: "sum", + Decl: types.NewFunction( + types.Args( + types.NewAny( + types.NewSet(types.N), + types.NewArray(nil, types.N), + ), + ), + types.N, + ), +} + +// Product takes an array or set of numbers and multiplies them. +var Product = &Builtin{ + Name: "product", + Decl: types.NewFunction( + types.Args( + types.NewAny( + types.NewSet(types.N), + types.NewArray(nil, types.N), + ), + ), + types.N, + ), +} + +// Max returns the maximum value in a collection. +var Max = &Builtin{ + Name: "max", + Decl: types.NewFunction( + types.Args( + types.NewAny( + types.NewSet(types.A), + types.NewArray(nil, types.A), + ), + ), + types.A, + ), +} + +// Min returns the minimum value in a collection. +var Min = &Builtin{ + Name: "min", + Decl: types.NewFunction( + types.Args( + types.NewAny( + types.NewSet(types.A), + types.NewArray(nil, types.A), + ), + ), + types.A, + ), +} + +// All takes a list and returns true if all of the items +// are true. A collection of length 0 returns true. +var All = &Builtin{ + Name: "all", + Decl: types.NewFunction( + types.Args( + types.NewAny( + types.NewSet(types.A), + types.NewArray(nil, types.A), + ), + ), + types.B, + ), +} + +// Any takes a collection and returns true if any of the items +// is true. A collection of length 0 returns false. +var Any = &Builtin{ + Name: "any", + Decl: types.NewFunction( + types.Args( + types.NewAny( + types.NewSet(types.A), + types.NewArray(nil, types.A), + ), + ), + types.B, + ), +} + +/** + * Arrays + */ + +// ArrayConcat returns the result of concatenating two arrays together. +var ArrayConcat = &Builtin{ + Name: "array.concat", + Decl: types.NewFunction( + types.Args( + types.NewArray(nil, types.A), + types.NewArray(nil, types.A), + ), + types.NewArray(nil, types.A), + ), +} + +// ArraySlice returns a slice of a given array +var ArraySlice = &Builtin{ + Name: "array.slice", + Decl: types.NewFunction( + types.Args( + types.NewArray(nil, types.A), + types.NewNumber(), + types.NewNumber(), + ), + types.NewArray(nil, types.A), + ), +} + +/** + * Conversions + */ + +// ToNumber takes a string, bool, or number value and converts it to a number. +// Strings are converted to numbers using strconv.Atoi. +// Boolean false is converted to 0 and boolean true is converted to 1. +var ToNumber = &Builtin{ + Name: "to_number", + Decl: types.NewFunction( + types.Args( + types.NewAny( + types.N, + types.S, + types.B, + types.NewNull(), + ), + ), + types.N, + ), +} + +/** + * Regular Expressions + */ + +// RegexMatch takes two strings and evaluates to true if the string in the second +// position matches the pattern in the first position. +var RegexMatch = &Builtin{ + Name: "re_match", + Decl: types.NewFunction( + types.Args( + types.S, + types.S, + ), + types.B, + ), +} + +// RegexFindAllStringSubmatch returns an array of all successive matches of the expression. +// It takes two strings and a number, the pattern, the value and number of matches to +// return, -1 means all matches. +var RegexFindAllStringSubmatch = &Builtin{ + Name: "regex.find_all_string_submatch_n", + Decl: types.NewFunction( + types.Args( + types.S, + types.S, + types.N, + ), + types.NewArray(nil, types.NewArray(nil, types.S)), + ), +} + +// RegexTemplateMatch takes two strings and evaluates to true if the string in the second +// position matches the pattern in the first position. +var RegexTemplateMatch = &Builtin{ + Name: "regex.template_match", + Decl: types.NewFunction( + types.Args( + types.S, + types.S, + types.S, + types.S, + ), + types.B, + ), +} + +// RegexSplit splits the input string by the occurrences of the given pattern. +var RegexSplit = &Builtin{ + Name: "regex.split", + Decl: types.NewFunction( + types.Args( + types.S, + types.S, + ), + types.NewArray(nil, types.S), + ), +} + +// RegexFind takes two strings and a number, the pattern, the value and number of match values to +// return, -1 means all match values. +var RegexFind = &Builtin{ + Name: "regex.find_n", + Decl: types.NewFunction( + types.Args( + types.S, + types.S, + types.N, + ), + types.NewArray(nil, types.S), + ), +} + +// GlobsMatch takes two strings regexp-style strings and evaluates to true if their +// intersection matches a non-empty set of non-empty strings. +// Examples: +// - "a.a." and ".b.b" -> true. +// - "[a-z]*" and [0-9]+" -> not true. +var GlobsMatch = &Builtin{ + Name: "regex.globs_match", + Decl: types.NewFunction( + types.Args( + types.S, + types.S, + ), + types.B, + ), +} + +/** + * Strings + */ + +// Concat joins an array of strings with an input string. +var Concat = &Builtin{ + Name: "concat", + Decl: types.NewFunction( + types.Args( + types.S, + types.NewAny( + types.NewSet(types.S), + types.NewArray(nil, types.S), + ), + ), + types.S, + ), +} + +// FormatInt returns the string representation of the number in the given base after converting it to an integer value. +var FormatInt = &Builtin{ + Name: "format_int", + Decl: types.NewFunction( + types.Args( + types.N, + types.N, + ), + types.S, + ), +} + +// IndexOf returns the index of a substring contained inside a string +var IndexOf = &Builtin{ + Name: "indexof", + Decl: types.NewFunction( + types.Args( + types.S, + types.S, + ), + types.N, + ), +} + +// Substring returns the portion of a string for a given start index and a length. +// If the length is less than zero, then substring returns the remainder of the string. +var Substring = &Builtin{ + Name: "substring", + Decl: types.NewFunction( + types.Args( + types.S, + types.N, + types.N, + ), + types.S, + ), +} + +// Contains returns true if the search string is included in the base string +var Contains = &Builtin{ + Name: "contains", + Decl: types.NewFunction( + types.Args( + types.S, + types.S, + ), + types.B, + ), +} + +// StartsWith returns true if the search string begins with the base string +var StartsWith = &Builtin{ + Name: "startswith", + Decl: types.NewFunction( + types.Args( + types.S, + types.S, + ), + types.B, + ), +} + +// EndsWith returns true if the search string begins with the base string +var EndsWith = &Builtin{ + Name: "endswith", + Decl: types.NewFunction( + types.Args( + types.S, + types.S, + ), + types.B, + ), +} + +// Lower returns the input string but with all characters in lower-case +var Lower = &Builtin{ + Name: "lower", + Decl: types.NewFunction( + types.Args(types.S), + types.S, + ), +} + +// Upper returns the input string but with all characters in upper-case +var Upper = &Builtin{ + Name: "upper", + Decl: types.NewFunction( + types.Args(types.S), + types.S, + ), +} + +// Split returns an array containing elements of the input string split on a delimiter. +var Split = &Builtin{ + Name: "split", + Decl: types.NewFunction( + types.Args( + types.S, + types.S, + ), + types.NewArray(nil, types.S), + ), +} + +// Replace returns the given string with all instances of the second argument replaced +// by the third. +var Replace = &Builtin{ + Name: "replace", + Decl: types.NewFunction( + types.Args( + types.S, + types.S, + types.S, + ), + types.S, + ), +} + +// ReplaceN replaces a string from a list of old, new string pairs. +// Replacements are performed in the order they appear in the target string, without overlapping matches. +// The old string comparisons are done in argument order. +var ReplaceN = &Builtin{ + Name: "strings.replace_n", + Decl: types.NewFunction( + types.Args( + types.NewObject( + nil, + types.NewDynamicProperty( + types.S, + types.S)), + types.S, + ), + types.S, + ), +} + +// Trim returns the given string with all leading or trailing instances of the second +// argument removed. +var Trim = &Builtin{ + Name: "trim", + Decl: types.NewFunction( + types.Args( + types.S, + types.S, + ), + types.S, + ), +} + +// TrimLeft returns the given string with all leading instances of second argument removed. +var TrimLeft = &Builtin{ + Name: "trim_left", + Decl: types.NewFunction( + types.Args( + types.S, + types.S, + ), + types.S, + ), +} + +// TrimPrefix returns the given string without the second argument prefix string. +// If the given string doesn't start with prefix, it is returned unchanged. +var TrimPrefix = &Builtin{ + Name: "trim_prefix", + Decl: types.NewFunction( + types.Args( + types.S, + types.S, + ), + types.S, + ), +} + +// TrimRight returns the given string with all trailing instances of second argument removed. +var TrimRight = &Builtin{ + Name: "trim_right", + Decl: types.NewFunction( + types.Args( + types.S, + types.S, + ), + types.S, + ), +} + +// TrimSuffix returns the given string without the second argument suffix string. +// If the given string doesn't end with suffix, it is returned unchanged. +var TrimSuffix = &Builtin{ + Name: "trim_suffix", + Decl: types.NewFunction( + types.Args( + types.S, + types.S, + ), + types.S, + ), +} + +// TrimSpace return the given string with all leading and trailing white space removed. +var TrimSpace = &Builtin{ + Name: "trim_space", + Decl: types.NewFunction( + types.Args( + types.S, + ), + types.S, + ), +} + +// Sprintf returns the given string, formatted. +var Sprintf = &Builtin{ + Name: "sprintf", + Decl: types.NewFunction( + types.Args( + types.S, + types.NewArray(nil, types.A), + ), + types.S, + ), +} + +// UnitsParseBytes converts strings like 10GB, 5K, 4mb, and the like into an +// integer number of bytes. +var UnitsParseBytes = &Builtin{ + Name: "units.parse_bytes", + Decl: types.NewFunction( + types.Args( + types.S, + ), + types.N, + ), +} + +/** + * JSON + */ + +// JSONMarshal serializes the input term. +var JSONMarshal = &Builtin{ + Name: "json.marshal", + Decl: types.NewFunction( + types.Args(types.A), + types.S, + ), +} + +// JSONUnmarshal deserializes the input string. +var JSONUnmarshal = &Builtin{ + Name: "json.unmarshal", + Decl: types.NewFunction( + types.Args(types.S), + types.A, + ), +} + +// JSONFilter filters the JSON object +var JSONFilter = &Builtin{ + Name: "json.filter", + Decl: types.NewFunction( + types.Args( + types.NewObject( + nil, + types.NewDynamicProperty(types.A, types.A), + ), + types.NewAny( + types.NewArray( + nil, + types.NewAny( + types.S, + types.NewArray( + nil, + types.A, + ), + ), + ), + types.NewSet( + types.NewAny( + types.S, + types.NewArray( + nil, + types.A, + ), + ), + ), + ), + ), + types.A, + ), +} + +// JSONRemove removes paths in the JSON object +var JSONRemove = &Builtin{ + Name: "json.remove", + Decl: types.NewFunction( + types.Args( + types.NewObject( + nil, + types.NewDynamicProperty(types.A, types.A), + ), + types.NewAny( + types.NewArray( + nil, + types.NewAny( + types.S, + types.NewArray( + nil, + types.A, + ), + ), + ), + types.NewSet( + types.NewAny( + types.S, + types.NewArray( + nil, + types.A, + ), + ), + ), + ), + ), + types.A, + ), +} + +// ObjectUnion creates a new object that is the asymmetric union of two objects +var ObjectUnion = &Builtin{ + Name: "object.union", + Decl: types.NewFunction( + types.Args( + types.NewObject( + nil, + types.NewDynamicProperty(types.A, types.A), + ), + types.NewObject( + nil, + types.NewDynamicProperty(types.A, types.A), + ), + ), + types.A, + ), +} + +// ObjectRemove Removes specified keys from an object +var ObjectRemove = &Builtin{ + Name: "object.remove", + Decl: types.NewFunction( + types.Args( + types.NewObject( + nil, + types.NewDynamicProperty(types.A, types.A), + ), + types.NewAny( + types.NewArray(nil, types.A), + types.NewSet(types.A), + types.NewObject(nil, types.NewDynamicProperty(types.A, types.A)), + ), + ), + types.A, + ), +} + +// ObjectFilter filters the object by keeping only specified keys +var ObjectFilter = &Builtin{ + Name: "object.filter", + Decl: types.NewFunction( + types.Args( + types.NewObject( + nil, + types.NewDynamicProperty(types.A, types.A), + ), + types.NewAny( + types.NewArray(nil, types.A), + types.NewSet(types.A), + types.NewObject(nil, types.NewDynamicProperty(types.A, types.A)), + ), + ), + types.A, + ), +} + +// Base64Encode serializes the input string into base64 encoding. +var Base64Encode = &Builtin{ + Name: "base64.encode", + Decl: types.NewFunction( + types.Args(types.S), + types.S, + ), +} + +// Base64Decode deserializes the base64 encoded input string. +var Base64Decode = &Builtin{ + Name: "base64.decode", + Decl: types.NewFunction( + types.Args(types.S), + types.S, + ), +} + +// Base64UrlEncode serializes the input string into base64url encoding. +var Base64UrlEncode = &Builtin{ + Name: "base64url.encode", + Decl: types.NewFunction( + types.Args(types.S), + types.S, + ), +} + +// Base64UrlDecode deserializes the base64url encoded input string. +var Base64UrlDecode = &Builtin{ + Name: "base64url.decode", + Decl: types.NewFunction( + types.Args(types.S), + types.S, + ), +} + +// URLQueryDecode decodes a URL encoded input string. +var URLQueryDecode = &Builtin{ + Name: "urlquery.decode", + Decl: types.NewFunction( + types.Args(types.S), + types.S, + ), +} + +// URLQueryEncode encodes the input string into a URL encoded string. +var URLQueryEncode = &Builtin{ + Name: "urlquery.encode", + Decl: types.NewFunction( + types.Args(types.S), + types.S, + ), +} + +// URLQueryEncodeObject encodes the given JSON into a URL encoded query string. +var URLQueryEncodeObject = &Builtin{ + Name: "urlquery.encode_object", + Decl: types.NewFunction( + types.Args( + types.NewObject( + nil, + types.NewDynamicProperty( + types.S, + types.NewAny( + types.S, + types.NewArray(nil, types.S), + types.NewSet(types.S))))), + types.S, + ), +} + +// YAMLMarshal serializes the input term. +var YAMLMarshal = &Builtin{ + Name: "yaml.marshal", + Decl: types.NewFunction( + types.Args(types.A), + types.S, + ), +} + +// YAMLUnmarshal deserializes the input string. +var YAMLUnmarshal = &Builtin{ + Name: "yaml.unmarshal", + Decl: types.NewFunction( + types.Args(types.S), + types.A, + ), +} + +/** + * Tokens + */ + +// JWTDecode decodes a JSON Web Token and outputs it as an Object. +var JWTDecode = &Builtin{ + Name: "io.jwt.decode", + Decl: types.NewFunction( + types.Args(types.S), + types.NewArray([]types.Type{ + types.NewObject(nil, types.NewDynamicProperty(types.A, types.A)), + types.NewObject(nil, types.NewDynamicProperty(types.A, types.A)), + types.S, + }, nil), + ), +} + +// JWTVerifyRS256 verifies if a RS256 JWT signature is valid or not. +var JWTVerifyRS256 = &Builtin{ + Name: "io.jwt.verify_rs256", + Decl: types.NewFunction( + types.Args( + types.S, + types.S, + ), + types.B, + ), +} + +// JWTVerifyPS256 verifies if a PS256 JWT signature is valid or not. +var JWTVerifyPS256 = &Builtin{ + Name: "io.jwt.verify_ps256", + Decl: types.NewFunction( + types.Args( + types.S, + types.S, + ), + types.B, + ), +} + +// JWTVerifyES256 verifies if a ES256 JWT signature is valid or not. +var JWTVerifyES256 = &Builtin{ + Name: "io.jwt.verify_es256", + Decl: types.NewFunction( + types.Args( + types.S, + types.S, + ), + types.B, + ), +} + +// JWTVerifyHS256 verifies if a HS256 (secret) JWT signature is valid or not. +var JWTVerifyHS256 = &Builtin{ + Name: "io.jwt.verify_hs256", + Decl: types.NewFunction( + types.Args( + types.S, + types.S, + ), + types.B, + ), +} + +// JWTDecodeVerify verifies a JWT signature under parameterized constraints and decodes the claims if it is valid. +var JWTDecodeVerify = &Builtin{ + Name: "io.jwt.decode_verify", + Decl: types.NewFunction( + types.Args( + types.S, + types.NewObject(nil, types.NewDynamicProperty(types.S, types.A)), + ), + types.NewArray([]types.Type{ + types.B, + types.NewObject(nil, types.NewDynamicProperty(types.A, types.A)), + types.NewObject(nil, types.NewDynamicProperty(types.A, types.A)), + }, nil), + ), +} + +// JWTEncodeSignRaw encodes and optionally sign a JSON Web Token. +// Inputs are protected headers, payload, secret +var JWTEncodeSignRaw = &Builtin{ + Name: "io.jwt.encode_sign_raw", + Decl: types.NewFunction( + types.Args( + types.S, + types.S, + types.S, + ), + types.S, + ), +} + +// JWTEncodeSign encodes and optionally sign a JSON Web Token. +// Inputs are protected headers, payload, secret +var JWTEncodeSign = &Builtin{ + Name: "io.jwt.encode_sign", + Decl: types.NewFunction( + types.Args( + types.NewObject(nil, types.NewDynamicProperty(types.S, types.A)), + types.NewObject(nil, types.NewDynamicProperty(types.S, types.A)), + types.NewObject(nil, types.NewDynamicProperty(types.S, types.A)), + ), + types.S, + ), +} + +/** + * Time + */ + +// NowNanos returns the current time since epoch in nanoseconds. +var NowNanos = &Builtin{ + Name: "time.now_ns", + Decl: types.NewFunction( + nil, + types.N, + ), +} + +// ParseNanos returns the time in nanoseconds parsed from the string in the given format. +var ParseNanos = &Builtin{ + Name: "time.parse_ns", + Decl: types.NewFunction( + types.Args( + types.S, + types.S, + ), + types.N, + ), +} + +// ParseRFC3339Nanos returns the time in nanoseconds parsed from the string in RFC3339 format. +var ParseRFC3339Nanos = &Builtin{ + Name: "time.parse_rfc3339_ns", + Decl: types.NewFunction( + types.Args(types.S), + types.N, + ), +} + +// ParseDurationNanos returns the duration in nanoseconds represented by a duration string. +// Duration string is similar to the Go time.ParseDuration string +var ParseDurationNanos = &Builtin{ + Name: "time.parse_duration_ns", + Decl: types.NewFunction( + types.Args(types.S), + types.N, + ), +} + +// Date returns the [year, month, day] for the nanoseconds since epoch. +var Date = &Builtin{ + Name: "time.date", + Decl: types.NewFunction( + types.Args( + types.NewAny( + types.N, + types.NewArray([]types.Type{types.N, types.S}, nil), + ), + ), + types.NewArray([]types.Type{types.N, types.N, types.N}, nil), + ), +} + +// Clock returns the [hour, minute, second] of the day for the nanoseconds since epoch. +var Clock = &Builtin{ + Name: "time.clock", + Decl: types.NewFunction( + types.Args( + types.NewAny( + types.N, + types.NewArray([]types.Type{types.N, types.S}, nil), + ), + ), + types.NewArray([]types.Type{types.N, types.N, types.N}, nil), + ), +} + +// Weekday returns the day of the week (Monday, Tuesday, ...) for the nanoseconds since epoch. +var Weekday = &Builtin{ + Name: "time.weekday", + Decl: types.NewFunction( + types.Args( + types.NewAny( + types.N, + types.NewArray([]types.Type{types.N, types.S}, nil), + ), + ), + types.S, + ), +} + +/** + * Crypto. + */ + +// CryptoX509ParseCertificates returns one or more certificates from the given +// base64 encoded string containing DER encoded certificates that have been +// concatenated. +var CryptoX509ParseCertificates = &Builtin{ + Name: "crypto.x509.parse_certificates", + Decl: types.NewFunction( + types.Args(types.S), + types.NewArray(nil, types.NewObject(nil, types.NewDynamicProperty(types.S, types.A))), + ), +} + +// CryptoMd5 returns a string representing the input string hashed with the md5 function +var CryptoMd5 = &Builtin{ + Name: "crypto.md5", + Decl: types.NewFunction( + types.Args(types.S), + types.S, + ), +} + +// CryptoSha1 returns a string representing the input string hashed with the sha1 function +var CryptoSha1 = &Builtin{ + Name: "crypto.sha1", + Decl: types.NewFunction( + types.Args(types.S), + types.S, + ), +} + +// CryptoSha256 returns a string representing the input string hashed with the sha256 function +var CryptoSha256 = &Builtin{ + Name: "crypto.sha256", + Decl: types.NewFunction( + types.Args(types.S), + types.S, + ), +} + +/** + * Graphs. + */ + +// WalkBuiltin generates [path, value] tuples for all nested documents +// (recursively). +var WalkBuiltin = &Builtin{ + Name: "walk", + Relation: true, + Decl: types.NewFunction( + types.Args(types.A), + types.NewArray( + []types.Type{ + types.NewArray(nil, types.A), + types.A, + }, + nil, + ), + ), +} + +/** + * Sorting + */ + +// Sort returns a sorted array. +var Sort = &Builtin{ + Name: "sort", + Decl: types.NewFunction( + types.Args( + types.NewAny( + types.NewArray(nil, types.A), + types.NewSet(types.A), + ), + ), + types.NewArray(nil, types.A), + ), +} + +/** + * Type + */ + +// IsNumber returns true if the input value is a number +var IsNumber = &Builtin{ + Name: "is_number", + Decl: types.NewFunction( + types.Args( + types.A, + ), + types.B, + ), +} + +// IsString returns true if the input value is a string. +var IsString = &Builtin{ + Name: "is_string", + Decl: types.NewFunction( + types.Args( + types.A, + ), + types.B, + ), +} + +// IsBoolean returns true if the input value is a boolean. +var IsBoolean = &Builtin{ + Name: "is_boolean", + Decl: types.NewFunction( + types.Args( + types.A, + ), + types.B, + ), +} + +// IsArray returns true if the input value is an array. +var IsArray = &Builtin{ + Name: "is_array", + Decl: types.NewFunction( + types.Args( + types.A, + ), + types.B, + ), +} + +// IsSet returns true if the input value is a set. +var IsSet = &Builtin{ + Name: "is_set", + Decl: types.NewFunction( + types.Args( + types.A, + ), + types.B, + ), +} + +// IsObject returns true if the input value is an object. +var IsObject = &Builtin{ + Name: "is_object", + Decl: types.NewFunction( + types.Args( + types.A, + ), + types.B, + ), +} + +// IsNull returns true if the input value is null. +var IsNull = &Builtin{ + Name: "is_null", + Decl: types.NewFunction( + types.Args( + types.A, + ), + types.B, + ), +} + +/** + * Type Name + */ + +// TypeNameBuiltin returns the type of the input. +var TypeNameBuiltin = &Builtin{ + Name: "type_name", + Decl: types.NewFunction( + types.Args( + types.NewAny( + types.A, + ), + ), + types.S, + ), +} + +/** + * HTTP Request + */ + +// HTTPSend returns a HTTP response to the given HTTP request. +var HTTPSend = &Builtin{ + Name: "http.send", + Decl: types.NewFunction( + types.Args( + types.NewObject(nil, types.NewDynamicProperty(types.S, types.A)), + ), + types.NewObject(nil, types.NewDynamicProperty(types.A, types.A)), + ), +} + +/** + * Rego + */ + +// RegoParseModule parses the input Rego file and returns a JSON representation +// of the AST. +var RegoParseModule = &Builtin{ + Name: "rego.parse_module", + Decl: types.NewFunction( + types.Args( + types.S, + types.S, + ), + types.NewObject(nil, types.NewDynamicProperty(types.S, types.A)), // TODO(tsandall): import AST schema + ), +} + +/** + * OPA + */ + +// OPARuntime returns an object containing OPA runtime information such as the +// configuration that OPA was booted with. +var OPARuntime = &Builtin{ + Name: "opa.runtime", + Decl: types.NewFunction( + nil, + types.NewObject(nil, types.NewDynamicProperty(types.S, types.A)), + ), +} + +/** + * Trace + */ + +// Trace prints a note that is included in the query explanation. +var Trace = &Builtin{ + Name: "trace", + Decl: types.NewFunction( + types.Args( + types.S, + ), + types.B, + ), +} + +/** + * Set + */ + +// Intersection returns the intersection of the given input sets +var Intersection = &Builtin{ + Name: "intersection", + Decl: types.NewFunction( + types.Args( + types.NewSet(types.NewSet(types.A)), + ), + types.NewSet(types.A), + ), +} + +// Union returns the union of the given input sets +var Union = &Builtin{ + Name: "union", + Decl: types.NewFunction( + types.Args( + types.NewSet(types.NewSet(types.A)), + ), + types.NewSet(types.A), + ), +} + +/** + * Glob + */ + +// GlobMatch - not to be confused with regex.globs_match - parses and matches strings against the glob notation. +var GlobMatch = &Builtin{ + Name: "glob.match", + Decl: types.NewFunction( + types.Args( + types.S, + types.NewArray(nil, types.S), + types.S, + ), + types.B, + ), +} + +// GlobQuoteMeta returns a string which represents a version of the pattern where all asterisks have been escaped. +var GlobQuoteMeta = &Builtin{ + Name: "glob.quote_meta", + Decl: types.NewFunction( + types.Args( + types.S, + ), + types.S, + ), +} + +/** + * Net CIDR + */ + +// NetCIDRIntersects checks if a cidr intersects with another cidr and returns true or false +var NetCIDRIntersects = &Builtin{ + Name: "net.cidr_intersects", + Decl: types.NewFunction( + types.Args( + types.S, + types.S, + ), + types.B, + ), +} + +// NetCIDRExpand returns a set of hosts inside the specified cidr. +var NetCIDRExpand = &Builtin{ + Name: "net.cidr_expand", + Decl: types.NewFunction( + types.Args( + types.S, + ), + types.NewSet(types.S), + ), +} + +// NetCIDRContains checks if a cidr or ip is contained within another cidr and returns true or false +var NetCIDRContains = &Builtin{ + Name: "net.cidr_contains", + Decl: types.NewFunction( + types.Args( + types.S, + types.S, + ), + types.B, + ), +} + +/** + * Deprecated built-ins. + */ + +// SetDiff has been replaced by the minus built-in. +var SetDiff = &Builtin{ + Name: "set_diff", + Decl: types.NewFunction( + types.Args( + types.NewSet(types.A), + types.NewSet(types.A), + ), + types.NewSet(types.A), + ), +} + +// NetCIDROverlap has been replaced by the `net.cidr_contains` built-in. +var NetCIDROverlap = &Builtin{ + Name: "net.cidr_overlap", + Decl: types.NewFunction( + types.Args( + types.S, + types.S, + ), + types.B, + ), +} + +// CastArray checks the underlying type of the input. If it is array or set, an array +// containing the values is returned. If it is not an array, an error is thrown. +var CastArray = &Builtin{ + Name: "cast_array", + Decl: types.NewFunction( + types.Args(types.A), + types.NewArray(nil, types.A), + ), +} + +// CastSet checks the underlying type of the input. +// If it is a set, the set is returned. +// If it is an array, the array is returned in set form (all duplicates removed) +// If neither, an error is thrown +var CastSet = &Builtin{ + Name: "cast_set", + Decl: types.NewFunction( + types.Args(types.A), + types.NewSet(types.A), + ), +} + +// CastString returns input if it is a string; if not returns error. +// For formatting variables, see sprintf +var CastString = &Builtin{ + Name: "cast_string", + Decl: types.NewFunction( + types.Args(types.A), + types.S, + ), +} + +// CastBoolean returns input if it is a boolean; if not returns error. +var CastBoolean = &Builtin{ + Name: "cast_boolean", + Decl: types.NewFunction( + types.Args(types.A), + types.B, + ), +} + +// CastNull returns null if input is null; if not returns error. +var CastNull = &Builtin{ + Name: "cast_null", + Decl: types.NewFunction( + types.Args(types.A), + types.NewNull(), + ), +} + +// CastObject returns the given object if it is null; throws an error otherwise +var CastObject = &Builtin{ + Name: "cast_object", + Decl: types.NewFunction( + types.Args(types.A), + types.NewObject(nil, types.NewDynamicProperty(types.A, types.A)), + ), +} + +// ObjectGet returns takes an object and returns a value under its key if +// present, otherwise it returns the default. +var ObjectGet = &Builtin{ + Name: "object.get", + Decl: types.NewFunction( + types.Args( + types.NewObject(nil, types.NewDynamicProperty(types.A, types.A)), + types.A, + types.A, + ), + types.A, + ), +} + +// Builtin represents a built-in function supported by OPA. Every built-in +// function is uniquely identified by a name. +type Builtin struct { + Name string // Unique name of built-in function, e.g., (arg1,arg2,...,argN) + Infix string // Unique name of infix operator. Default should be unset. + Decl *types.Function // Built-in function type declaration. + Relation bool // Indicates if the built-in acts as a relation. +} + +// Expr creates a new expression for the built-in with the given operands. +func (b *Builtin) Expr(operands ...*Term) *Expr { + ts := make([]*Term, len(operands)+1) + ts[0] = NewTerm(b.Ref()) + for i := range operands { + ts[i+1] = operands[i] + } + return &Expr{ + Terms: ts, + } +} + +// Call creates a new term for the built-in with the given operands. +func (b *Builtin) Call(operands ...*Term) *Term { + call := make(Call, len(operands)+1) + call[0] = NewTerm(b.Ref()) + for i := range operands { + call[i+1] = operands[i] + } + return NewTerm(call) +} + +// Ref returns a Ref that refers to the built-in function. +func (b *Builtin) Ref() Ref { + parts := strings.Split(b.Name, ".") + ref := make(Ref, len(parts)) + ref[0] = VarTerm(parts[0]) + for i := 1; i < len(parts); i++ { + ref[i] = StringTerm(parts[i]) + } + return ref +} + +// IsTargetPos returns true if a variable in the i-th position will be bound by +// evaluating the call expression. +func (b *Builtin) IsTargetPos(i int) bool { + return len(b.Decl.Args()) == i +} + +func init() { + BuiltinMap = map[string]*Builtin{} + for _, b := range DefaultBuiltins { + RegisterBuiltin(b) + } +} diff --git a/vendor/github.com/open-policy-agent/opa/ast/check.go b/vendor/github.com/open-policy-agent/opa/ast/check.go new file mode 100644 index 000000000..cff3be288 --- /dev/null +++ b/vendor/github.com/open-policy-agent/opa/ast/check.go @@ -0,0 +1,983 @@ +// Copyright 2017 The OPA Authors. All rights reserved. +// Use of this source code is governed by an Apache2 +// license that can be found in the LICENSE file. + +package ast + +import ( + "fmt" + "sort" + "strings" + + "github.com/open-policy-agent/opa/types" + "github.com/open-policy-agent/opa/util" +) + +type rewriteVars func(x Ref) Ref + +// exprChecker defines the interface for executing type checking on a single +// expression. The exprChecker must update the provided TypeEnv with inferred +// types of vars. +type exprChecker func(*TypeEnv, *Expr) *Error + +// typeChecker implements type checking on queries and rules. Errors are +// accumulated on the typeChecker so that a single run can report multiple +// issues. +type typeChecker struct { + errs Errors + exprCheckers map[string]exprChecker + varRewriter rewriteVars +} + +// newTypeChecker returns a new typeChecker object that has no errors. +func newTypeChecker() *typeChecker { + tc := &typeChecker{} + tc.exprCheckers = map[string]exprChecker{ + "eq": tc.checkExprEq, + } + return tc +} + +func (tc *typeChecker) WithVarRewriter(f rewriteVars) *typeChecker { + tc.varRewriter = f + return tc +} + +// CheckBody runs type checking on the body and returns a TypeEnv if no errors +// are found. The resulting TypeEnv wraps the provided one. The resulting +// TypeEnv will be able to resolve types of vars contained in the body. +func (tc *typeChecker) CheckBody(env *TypeEnv, body Body) (*TypeEnv, Errors) { + + if env == nil { + env = NewTypeEnv() + } else { + env = env.wrap() + } + + WalkExprs(body, func(expr *Expr) bool { + + closureErrs := tc.checkClosures(env, expr) + for _, err := range closureErrs { + tc.err(err) + } + + hasClosureErrors := len(closureErrs) > 0 + + vis := newRefChecker(env, tc.varRewriter) + NewGenericVisitor(vis.Visit).Walk(expr) + for _, err := range vis.errs { + tc.err(err) + } + + hasRefErrors := len(vis.errs) > 0 + + if err := tc.checkExpr(env, expr); err != nil { + // Suppress this error if a more actionable one has occurred. In + // this case, if an error occurred in a ref or closure contained in + // this expression, and the error is due to a nil type, then it's + // likely to be the result of the more specific error. + skip := (hasClosureErrors || hasRefErrors) && causedByNilType(err) + if !skip { + tc.err(err) + } + } + + return true + }) + + return env, tc.errs +} + +// CheckTypes runs type checking on the rules returns a TypeEnv if no errors +// are found. The resulting TypeEnv wraps the provided one. The resulting +// TypeEnv will be able to resolve types of refs that refer to rules. +func (tc *typeChecker) CheckTypes(env *TypeEnv, sorted []util.T) (*TypeEnv, Errors) { + if env == nil { + env = NewTypeEnv() + } else { + env = env.wrap() + } + for _, s := range sorted { + tc.checkRule(env, s.(*Rule)) + } + tc.errs.Sort() + return env, tc.errs +} + +func (tc *typeChecker) checkClosures(env *TypeEnv, expr *Expr) Errors { + var result Errors + WalkClosures(expr, func(x interface{}) bool { + switch x := x.(type) { + case *ArrayComprehension: + _, errs := newTypeChecker().WithVarRewriter(tc.varRewriter).CheckBody(env, x.Body) + if len(errs) > 0 { + result = errs + return true + } + case *SetComprehension: + _, errs := newTypeChecker().WithVarRewriter(tc.varRewriter).CheckBody(env, x.Body) + if len(errs) > 0 { + result = errs + return true + } + case *ObjectComprehension: + _, errs := newTypeChecker().WithVarRewriter(tc.varRewriter).CheckBody(env, x.Body) + if len(errs) > 0 { + result = errs + return true + } + } + return false + }) + return result +} + +func (tc *typeChecker) checkLanguageBuiltins(env *TypeEnv, builtins map[string]*Builtin) *TypeEnv { + if env == nil { + env = NewTypeEnv() + } else { + env = env.wrap() + } + for _, bi := range builtins { + env.tree.Put(bi.Ref(), bi.Decl) + } + return env +} + +func (tc *typeChecker) checkRule(env *TypeEnv, rule *Rule) { + + cpy, err := tc.CheckBody(env, rule.Body) + + if len(err) == 0 { + + path := rule.Path() + var tpe types.Type + + if len(rule.Head.Args) > 0 { + + // If args are not referred to in body, infer as any. + WalkVars(rule.Head.Args, func(v Var) bool { + if cpy.Get(v) == nil { + cpy.tree.PutOne(v, types.A) + } + return false + }) + + // Construct function type. + args := make([]types.Type, len(rule.Head.Args)) + for i := 0; i < len(rule.Head.Args); i++ { + args[i] = cpy.Get(rule.Head.Args[i]) + } + + f := types.NewFunction(args, cpy.Get(rule.Head.Value)) + + // Union with existing. + exist := env.tree.Get(path) + tpe = types.Or(exist, f) + + } else { + switch rule.Head.DocKind() { + case CompleteDoc: + typeV := cpy.Get(rule.Head.Value) + if typeV != nil { + exist := env.tree.Get(path) + tpe = types.Or(typeV, exist) + } + case PartialObjectDoc: + typeK := cpy.Get(rule.Head.Key) + typeV := cpy.Get(rule.Head.Value) + if typeK != nil && typeV != nil { + exist := env.tree.Get(path) + typeV = types.Or(types.Values(exist), typeV) + typeK = types.Or(types.Keys(exist), typeK) + tpe = types.NewObject(nil, types.NewDynamicProperty(typeK, typeV)) + } + case PartialSetDoc: + typeK := cpy.Get(rule.Head.Key) + if typeK != nil { + exist := env.tree.Get(path) + typeK = types.Or(types.Keys(exist), typeK) + tpe = types.NewSet(typeK) + } + } + } + + if tpe != nil { + env.tree.Put(path, tpe) + } + } +} + +func (tc *typeChecker) checkExpr(env *TypeEnv, expr *Expr) *Error { + if !expr.IsCall() { + return nil + } + + checker := tc.exprCheckers[expr.Operator().String()] + if checker != nil { + return checker(env, expr) + } + + return tc.checkExprBuiltin(env, expr) +} + +func (tc *typeChecker) checkExprBuiltin(env *TypeEnv, expr *Expr) *Error { + + args := expr.Operands() + pre := getArgTypes(env, args) + + // NOTE(tsandall): undefined functions will have been caught earlier in the + // compiler. We check for undefined functions before the safety check so + // that references to non-existent functions result in undefined function + // errors as opposed to unsafe var errors. + // + // We cannot run type checking before the safety check because part of the + // type checker relies on reordering (in particular for references to local + // vars). + name := expr.Operator() + tpe := env.Get(name) + + if tpe == nil { + return NewError(TypeErr, expr.Location, "undefined function %v", name) + } + + ftpe, ok := tpe.(*types.Function) + if !ok { + return NewError(TypeErr, expr.Location, "undefined function %v", name) + } + + maxArgs := len(ftpe.Args()) + expArgs := ftpe.Args() + + if ftpe.Result() != nil { + maxArgs++ + expArgs = append(expArgs, ftpe.Result()) + } + + if len(args) > maxArgs { + return newArgError(expr.Location, name, "too many arguments", pre, expArgs) + } else if len(args) < len(ftpe.Args()) { + return newArgError(expr.Location, name, "too few arguments", pre, expArgs) + } + + for i := range args { + if !unify1(env, args[i], expArgs[i], false) { + post := make([]types.Type, len(args)) + for i := range args { + post[i] = env.Get(args[i]) + } + return newArgError(expr.Location, name, "invalid argument(s)", post, expArgs) + } + } + + return nil +} + +func (tc *typeChecker) checkExprEq(env *TypeEnv, expr *Expr) *Error { + + pre := getArgTypes(env, expr.Operands()) + exp := Equality.Decl.Args() + + if len(pre) < len(exp) { + return newArgError(expr.Location, expr.Operator(), "too few arguments", pre, exp) + } else if len(exp) < len(pre) { + return newArgError(expr.Location, expr.Operator(), "too many arguments", pre, exp) + } + + a, b := expr.Operand(0), expr.Operand(1) + typeA, typeB := env.Get(a), env.Get(b) + + if !unify2(env, a, typeA, b, typeB) { + err := NewError(TypeErr, expr.Location, "match error") + err.Details = &UnificationErrDetail{ + Left: typeA, + Right: typeB, + } + return err + } + + return nil +} + +func unify2(env *TypeEnv, a *Term, typeA types.Type, b *Term, typeB types.Type) bool { + + nilA := types.Nil(typeA) + nilB := types.Nil(typeB) + + if nilA && !nilB { + return unify1(env, a, typeB, false) + } else if nilB && !nilA { + return unify1(env, b, typeA, false) + } else if !nilA && !nilB { + return unifies(typeA, typeB) + } + + switch a.Value.(type) { + case Array: + return unify2Array(env, a, typeA, b, typeB) + case Object: + return unify2Object(env, a, typeA, b, typeB) + case Var: + switch b.Value.(type) { + case Var: + return unify1(env, a, types.A, false) && unify1(env, b, env.Get(a), false) + case Array: + return unify2Array(env, b, typeB, a, typeA) + case Object: + return unify2Object(env, b, typeB, a, typeA) + } + } + + return false +} + +func unify2Array(env *TypeEnv, a *Term, typeA types.Type, b *Term, typeB types.Type) bool { + arr := a.Value.(Array) + switch bv := b.Value.(type) { + case Array: + if len(arr) == len(bv) { + for i := range arr { + if !unify2(env, arr[i], env.Get(arr[i]), bv[i], env.Get(bv[i])) { + return false + } + } + return true + } + case Var: + return unify1(env, a, types.A, false) && unify1(env, b, env.Get(a), false) + } + return false +} + +func unify2Object(env *TypeEnv, a *Term, typeA types.Type, b *Term, typeB types.Type) bool { + obj := a.Value.(Object) + switch bv := b.Value.(type) { + case Object: + cv := obj.Intersect(bv) + if obj.Len() == bv.Len() && bv.Len() == len(cv) { + for i := range cv { + if !unify2(env, cv[i][1], env.Get(cv[i][1]), cv[i][2], env.Get(cv[i][2])) { + return false + } + } + return true + } + case Var: + return unify1(env, a, types.A, false) && unify1(env, b, env.Get(a), false) + } + return false +} + +func unify1(env *TypeEnv, term *Term, tpe types.Type, union bool) bool { + switch v := term.Value.(type) { + case Array: + switch tpe := tpe.(type) { + case *types.Array: + return unify1Array(env, v, tpe, union) + case types.Any: + if types.Compare(tpe, types.A) == 0 { + for i := range v { + unify1(env, v[i], types.A, true) + } + return true + } + unifies := false + for i := range tpe { + unifies = unify1(env, term, tpe[i], true) || unifies + } + return unifies + } + return false + case Object: + switch tpe := tpe.(type) { + case *types.Object: + return unify1Object(env, v, tpe, union) + case types.Any: + if types.Compare(tpe, types.A) == 0 { + v.Foreach(func(key, value *Term) { + unify1(env, key, types.A, true) + unify1(env, value, types.A, true) + }) + return true + } + unifies := false + for i := range tpe { + unifies = unify1(env, term, tpe[i], true) || unifies + } + return unifies + } + return false + case Set: + switch tpe := tpe.(type) { + case *types.Set: + return unify1Set(env, v, tpe, union) + case types.Any: + if types.Compare(tpe, types.A) == 0 { + v.Foreach(func(elem *Term) { + unify1(env, elem, types.A, true) + }) + return true + } + unifies := false + for i := range tpe { + unifies = unify1(env, term, tpe[i], true) || unifies + } + return unifies + } + return false + case Ref, *ArrayComprehension, *ObjectComprehension, *SetComprehension: + return unifies(env.Get(v), tpe) + case Var: + if !union { + if exist := env.Get(v); exist != nil { + return unifies(exist, tpe) + } + env.tree.PutOne(term.Value, tpe) + } else { + env.tree.PutOne(term.Value, types.Or(env.Get(v), tpe)) + } + return true + default: + if !IsConstant(v) { + panic("unreachable") + } + return unifies(env.Get(term), tpe) + } +} + +func unify1Array(env *TypeEnv, val Array, tpe *types.Array, union bool) bool { + if len(val) != tpe.Len() && tpe.Dynamic() == nil { + return false + } + for i := range val { + if !unify1(env, val[i], tpe.Select(i), union) { + return false + } + } + return true +} + +func unify1Object(env *TypeEnv, val Object, tpe *types.Object, union bool) bool { + if val.Len() != len(tpe.Keys()) && tpe.DynamicValue() == nil { + return false + } + stop := val.Until(func(k, v *Term) bool { + if IsConstant(k.Value) { + if child := selectConstant(tpe, k); child != nil { + if !unify1(env, v, child, union) { + return true + } + } else { + return true + } + } else { + // Inferring type of value under dynamic key would involve unioning + // with all property values of tpe whose keys unify. For now, type + // these values as Any. We can investigate stricter inference in + // the future. + unify1(env, v, types.A, union) + } + return false + }) + return !stop +} + +func unify1Set(env *TypeEnv, val Set, tpe *types.Set, union bool) bool { + of := types.Values(tpe) + return !val.Until(func(elem *Term) bool { + return !unify1(env, elem, of, union) + }) +} + +func (tc *typeChecker) err(err *Error) { + tc.errs = append(tc.errs, err) +} + +type refChecker struct { + env *TypeEnv + errs Errors + varRewriter rewriteVars +} + +func newRefChecker(env *TypeEnv, f rewriteVars) *refChecker { + + if f == nil { + f = rewriteVarsNop + } + + return &refChecker{ + env: env, + errs: nil, + varRewriter: f, + } +} + +func (rc *refChecker) Visit(x interface{}) bool { + switch x := x.(type) { + case *ArrayComprehension, *ObjectComprehension, *SetComprehension: + return true + case *Expr: + switch terms := x.Terms.(type) { + case []*Term: + for i := 1; i < len(terms); i++ { + NewGenericVisitor(rc.Visit).Walk(terms[i]) + } + return true + case *Term: + NewGenericVisitor(rc.Visit).Walk(terms) + return true + } + case Ref: + if err := rc.checkApply(rc.env, x); err != nil { + rc.errs = append(rc.errs, err) + return true + } + if err := rc.checkRef(rc.env, rc.env.tree, x, 0); err != nil { + rc.errs = append(rc.errs, err) + } + } + return false +} + +func (rc *refChecker) checkApply(curr *TypeEnv, ref Ref) *Error { + if tpe := curr.Get(ref); tpe != nil { + if _, ok := tpe.(*types.Function); ok { + return newRefErrUnsupported(ref[0].Location, rc.varRewriter(ref), len(ref)-1, tpe) + } + } + return nil +} + +func (rc *refChecker) checkRef(curr *TypeEnv, node *typeTreeNode, ref Ref, idx int) *Error { + + if idx == len(ref) { + return nil + } + + head := ref[idx] + + // Handle constant ref operands, i.e., strings or the ref head. + if _, ok := head.Value.(String); ok || idx == 0 { + + child := node.Child(head.Value) + if child == nil { + + if curr.next != nil { + next := curr.next + return rc.checkRef(next, next.tree, ref, 0) + } + + if RootDocumentNames.Contains(ref[0]) { + return rc.checkRefLeaf(types.A, ref, 1) + } + + return rc.checkRefLeaf(types.A, ref, 0) + } + + if child.Leaf() { + return rc.checkRefLeaf(child.Value(), ref, idx+1) + } + + return rc.checkRef(curr, child, ref, idx+1) + } + + // Handle dynamic ref operands. + switch value := head.Value.(type) { + + case Var: + + if exist := rc.env.Get(value); exist != nil { + if !unifies(types.S, exist) { + return newRefErrInvalid(ref[0].Location, rc.varRewriter(ref), idx, exist, types.S, getOneOfForNode(node)) + } + } else { + rc.env.tree.PutOne(value, types.S) + } + + case Ref: + + exist := rc.env.Get(value) + if exist == nil { + // If ref type is unknown, an error will already be reported so + // stop here. + return nil + } + + if !unifies(types.S, exist) { + return newRefErrInvalid(ref[0].Location, rc.varRewriter(ref), idx, exist, types.S, getOneOfForNode(node)) + } + + // Catch other ref operand types here. Non-leaf nodes must be referred to + // with string values. + default: + return newRefErrInvalid(ref[0].Location, rc.varRewriter(ref), idx, nil, types.S, getOneOfForNode(node)) + } + + // Run checking on remaining portion of the ref. Note, since the ref + // potentially refers to data for which no type information exists, + // checking should never fail. + node.Children().Iter(func(_, child util.T) bool { + rc.checkRef(curr, child.(*typeTreeNode), ref, idx+1) + return false + }) + + return nil +} + +func (rc *refChecker) checkRefLeaf(tpe types.Type, ref Ref, idx int) *Error { + + if idx == len(ref) { + return nil + } + + head := ref[idx] + + keys := types.Keys(tpe) + if keys == nil { + return newRefErrUnsupported(ref[0].Location, rc.varRewriter(ref), idx-1, tpe) + } + + switch value := head.Value.(type) { + + case Var: + if exist := rc.env.Get(value); exist != nil { + if !unifies(exist, keys) { + return newRefErrInvalid(ref[0].Location, rc.varRewriter(ref), idx, exist, keys, getOneOfForType(tpe)) + } + } else { + rc.env.tree.PutOne(value, types.Keys(tpe)) + } + + case Ref: + if exist := rc.env.Get(value); exist != nil { + if !unifies(exist, keys) { + return newRefErrInvalid(ref[0].Location, rc.varRewriter(ref), idx, exist, keys, getOneOfForType(tpe)) + } + } + + case Array, Object, Set: + // Composite references operands may only be used with a set. + if !unifies(tpe, types.NewSet(types.A)) { + return newRefErrInvalid(ref[0].Location, rc.varRewriter(ref), idx, tpe, types.NewSet(types.A), nil) + } + if !unify1(rc.env, head, keys, false) { + return newRefErrInvalid(ref[0].Location, rc.varRewriter(ref), idx, rc.env.Get(head), keys, nil) + } + + default: + child := selectConstant(tpe, head) + if child == nil { + return newRefErrInvalid(ref[0].Location, rc.varRewriter(ref), idx, nil, types.Keys(tpe), getOneOfForType(tpe)) + } + return rc.checkRefLeaf(child, ref, idx+1) + } + + return rc.checkRefLeaf(types.Values(tpe), ref, idx+1) +} + +func unifies(a, b types.Type) bool { + + if a == nil || b == nil { + return false + } + + anyA, ok1 := a.(types.Any) + if ok1 { + if unifiesAny(anyA, b) { + return true + } + } + + anyB, ok2 := b.(types.Any) + if ok2 { + if unifiesAny(anyB, a) { + return true + } + } + + if ok1 || ok2 { + return false + } + + switch a := a.(type) { + case types.Null: + _, ok := b.(types.Null) + return ok + case types.Boolean: + _, ok := b.(types.Boolean) + return ok + case types.Number: + _, ok := b.(types.Number) + return ok + case types.String: + _, ok := b.(types.String) + return ok + case *types.Array: + b, ok := b.(*types.Array) + if !ok { + return false + } + return unifiesArrays(a, b) + case *types.Object: + b, ok := b.(*types.Object) + if !ok { + return false + } + return unifiesObjects(a, b) + case *types.Set: + b, ok := b.(*types.Set) + if !ok { + return false + } + return unifies(types.Values(a), types.Values(b)) + case *types.Function: + // TODO(tsandall): revisit once functions become first-class values. + return false + default: + panic("unreachable") + } +} + +func unifiesAny(a types.Any, b types.Type) bool { + if _, ok := b.(*types.Function); ok { + return false + } + for i := range a { + if unifies(a[i], b) { + return true + } + } + return len(a) == 0 +} + +func unifiesArrays(a, b *types.Array) bool { + + if !unifiesArraysStatic(a, b) { + return false + } + + if !unifiesArraysStatic(b, a) { + return false + } + + return a.Dynamic() == nil || b.Dynamic() == nil || unifies(a.Dynamic(), b.Dynamic()) +} + +func unifiesArraysStatic(a, b *types.Array) bool { + if a.Len() != 0 { + for i := 0; i < a.Len(); i++ { + if !unifies(a.Select(i), b.Select(i)) { + return false + } + } + } + return true +} + +func unifiesObjects(a, b *types.Object) bool { + if !unifiesObjectsStatic(a, b) { + return false + } + + if !unifiesObjectsStatic(b, a) { + return false + } + + return a.DynamicValue() == nil || b.DynamicValue() == nil || unifies(a.DynamicValue(), b.DynamicValue()) +} + +func unifiesObjectsStatic(a, b *types.Object) bool { + for _, k := range a.Keys() { + if !unifies(a.Select(k), b.Select(k)) { + return false + } + } + return true +} + +// typeErrorCause defines an interface to determine the reason for a type +// error. The type error details implement this interface so that type checking +// can report more actionable errors. +type typeErrorCause interface { + nilType() bool +} + +func causedByNilType(err *Error) bool { + cause, ok := err.Details.(typeErrorCause) + if !ok { + return false + } + return cause.nilType() +} + +// ArgErrDetail represents a generic argument error. +type ArgErrDetail struct { + Have []types.Type `json:"have"` + Want []types.Type `json:"want"` +} + +// Lines returns the string representation of the detail. +func (d *ArgErrDetail) Lines() []string { + lines := make([]string, 2) + lines[0] = fmt.Sprint("have: ", formatArgs(d.Have)) + lines[1] = fmt.Sprint("want: ", formatArgs(d.Want)) + return lines +} + +func (d *ArgErrDetail) nilType() bool { + for i := range d.Have { + if types.Nil(d.Have[i]) { + return true + } + } + return false +} + +// UnificationErrDetail describes a type mismatch error when two values are +// unified (e.g., x = [1,2,y]). +type UnificationErrDetail struct { + Left types.Type `json:"a"` + Right types.Type `json:"b"` +} + +func (a *UnificationErrDetail) nilType() bool { + return types.Nil(a.Left) || types.Nil(a.Right) +} + +// Lines returns the string representation of the detail. +func (a *UnificationErrDetail) Lines() []string { + lines := make([]string, 2) + lines[0] = fmt.Sprint("left : ", types.Sprint(a.Left)) + lines[1] = fmt.Sprint("right : ", types.Sprint(a.Right)) + return lines +} + +// RefErrUnsupportedDetail describes an undefined reference error where the +// referenced value does not support dereferencing (e.g., scalars). +type RefErrUnsupportedDetail struct { + Ref Ref `json:"ref"` // invalid ref + Pos int `json:"pos"` // invalid element + Have types.Type `json:"have"` // referenced type +} + +// Lines returns the string representation of the detail. +func (r *RefErrUnsupportedDetail) Lines() []string { + lines := []string{ + r.Ref.String(), + strings.Repeat("^", len(r.Ref[:r.Pos+1].String())), + fmt.Sprintf("have: %v", r.Have), + } + return lines +} + +// RefErrInvalidDetail describes an undefined reference error where the referenced +// value does not support the reference operand (e.g., missing object key, +// invalid key type, etc.) +type RefErrInvalidDetail struct { + Ref Ref `json:"ref"` // invalid ref + Pos int `json:"pos"` // invalid element + Have types.Type `json:"have,omitempty"` // type of invalid element (for var/ref elements) + Want types.Type `json:"want"` // allowed type (for non-object values) + OneOf []Value `json:"oneOf"` // allowed values (e.g., for object keys) +} + +// Lines returns the string representation of the detail. +func (r *RefErrInvalidDetail) Lines() []string { + lines := []string{r.Ref.String()} + offset := len(r.Ref[:r.Pos].String()) + 1 + pad := strings.Repeat(" ", offset) + lines = append(lines, fmt.Sprintf("%s^", pad)) + if r.Have != nil { + lines = append(lines, fmt.Sprintf("%shave (type): %v", pad, r.Have)) + } else { + lines = append(lines, fmt.Sprintf("%shave: %v", pad, r.Ref[r.Pos])) + } + if len(r.OneOf) > 0 { + lines = append(lines, fmt.Sprintf("%swant (one of): %v", pad, r.OneOf)) + } else { + lines = append(lines, fmt.Sprintf("%swant (type): %v", pad, r.Want)) + } + return lines +} + +func formatArgs(args []types.Type) string { + buf := make([]string, len(args)) + for i := range args { + buf[i] = types.Sprint(args[i]) + } + return "(" + strings.Join(buf, ", ") + ")" +} + +func newRefErrInvalid(loc *Location, ref Ref, idx int, have, want types.Type, oneOf []Value) *Error { + err := newRefError(loc, ref) + err.Details = &RefErrInvalidDetail{ + Ref: ref, + Pos: idx, + Have: have, + Want: want, + OneOf: oneOf, + } + return err +} + +func newRefErrUnsupported(loc *Location, ref Ref, idx int, have types.Type) *Error { + err := newRefError(loc, ref) + err.Details = &RefErrUnsupportedDetail{ + Ref: ref, + Pos: idx, + Have: have, + } + return err +} + +func newRefError(loc *Location, ref Ref) *Error { + return NewError(TypeErr, loc, "undefined ref: %v", ref) +} + +func newArgError(loc *Location, builtinName Ref, msg string, have []types.Type, want []types.Type) *Error { + err := NewError(TypeErr, loc, "%v: %v", builtinName, msg) + err.Details = &ArgErrDetail{ + Have: have, + Want: want, + } + return err +} + +func getOneOfForNode(node *typeTreeNode) (result []Value) { + node.Children().Iter(func(k, _ util.T) bool { + result = append(result, k.(Value)) + return false + }) + + sortValueSlice(result) + return result +} + +func getOneOfForType(tpe types.Type) (result []Value) { + switch tpe := tpe.(type) { + case *types.Object: + for _, k := range tpe.Keys() { + v, err := InterfaceToValue(k) + if err != nil { + panic(err) + } + result = append(result, v) + } + } + sortValueSlice(result) + return result +} + +func sortValueSlice(sl []Value) { + sort.Slice(sl, func(i, j int) bool { + return sl[i].Compare(sl[j]) < 0 + }) +} + +func getArgTypes(env *TypeEnv, args []*Term) []types.Type { + pre := make([]types.Type, len(args)) + for i := range args { + pre[i] = env.Get(args[i]) + } + return pre +} diff --git a/vendor/github.com/open-policy-agent/opa/ast/compare.go b/vendor/github.com/open-policy-agent/opa/ast/compare.go new file mode 100644 index 000000000..46bf71c41 --- /dev/null +++ b/vendor/github.com/open-policy-agent/opa/ast/compare.go @@ -0,0 +1,327 @@ +// Copyright 2016 The OPA Authors. All rights reserved. +// Use of this source code is governed by an Apache2 +// license that can be found in the LICENSE file. + +package ast + +import ( + "encoding/json" + "fmt" + "math/big" +) + +// Compare returns an integer indicating whether two AST values are less than, +// equal to, or greater than each other. +// +// If a is less than b, the return value is negative. If a is greater than b, +// the return value is positive. If a is equal to b, the return value is zero. +// +// Different types are never equal to each other. For comparison purposes, types +// are sorted as follows: +// +// nil < Null < Boolean < Number < String < Var < Ref < Array < Object < Set < +// ArrayComprehension < ObjectComprehension < SetComprehension < Expr < SomeDecl +// < With < Body < Rule < Import < Package < Module. +// +// Arrays and Refs are equal iff both a and b have the same length and all +// corresponding elements are equal. If one element is not equal, the return +// value is the same as for the first differing element. If all elements are +// equal but a and b have different lengths, the shorter is considered less than +// the other. +// +// Objects are considered equal iff both a and b have the same sorted (key, +// value) pairs and are of the same length. Other comparisons are consistent but +// not defined. +// +// Sets are considered equal iff the symmetric difference of a and b is empty. +// Other comparisons are consistent but not defined. +func Compare(a, b interface{}) int { + + if t, ok := a.(*Term); ok { + if t == nil { + a = nil + } else { + a = t.Value + } + } + + if t, ok := b.(*Term); ok { + if t == nil { + b = nil + } else { + b = t.Value + } + } + + if a == nil { + if b == nil { + return 0 + } + return -1 + } + if b == nil { + return 1 + } + + sortA := sortOrder(a) + sortB := sortOrder(b) + + if sortA < sortB { + return -1 + } else if sortB < sortA { + return 1 + } + + switch a := a.(type) { + case Null: + return 0 + case Boolean: + b := b.(Boolean) + if a.Equal(b) { + return 0 + } + if !a { + return -1 + } + return 1 + case Number: + if ai, err := json.Number(a).Int64(); err == nil { + if bi, err := json.Number(b.(Number)).Int64(); err == nil { + if ai == bi { + return 0 + } + if ai < bi { + return -1 + } + return 1 + } + } + + bigA, ok := new(big.Float).SetString(string(a)) + if !ok { + panic("illegal value") + } + bigB, ok := new(big.Float).SetString(string(b.(Number))) + if !ok { + panic("illegal value") + } + return bigA.Cmp(bigB) + case String: + b := b.(String) + if a.Equal(b) { + return 0 + } + if a < b { + return -1 + } + return 1 + case Var: + b := b.(Var) + if a.Equal(b) { + return 0 + } + if a < b { + return -1 + } + return 1 + case Ref: + b := b.(Ref) + return termSliceCompare(a, b) + case Array: + b := b.(Array) + return termSliceCompare(a, b) + case Object: + b := b.(Object) + return a.Compare(b) + case Set: + b := b.(Set) + return a.Compare(b) + case *ArrayComprehension: + b := b.(*ArrayComprehension) + if cmp := Compare(a.Term, b.Term); cmp != 0 { + return cmp + } + return Compare(a.Body, b.Body) + case *ObjectComprehension: + b := b.(*ObjectComprehension) + if cmp := Compare(a.Key, b.Key); cmp != 0 { + return cmp + } + if cmp := Compare(a.Value, b.Value); cmp != 0 { + return cmp + } + return Compare(a.Body, b.Body) + case *SetComprehension: + b := b.(*SetComprehension) + if cmp := Compare(a.Term, b.Term); cmp != 0 { + return cmp + } + return Compare(a.Body, b.Body) + case Call: + b := b.(Call) + return termSliceCompare(a, b) + case *Expr: + b := b.(*Expr) + return a.Compare(b) + case *SomeDecl: + b := b.(*SomeDecl) + return a.Compare(b) + case *With: + b := b.(*With) + return a.Compare(b) + case Body: + b := b.(Body) + return a.Compare(b) + case *Head: + b := b.(*Head) + return a.Compare(b) + case *Rule: + b := b.(*Rule) + return a.Compare(b) + case Args: + b := b.(Args) + return termSliceCompare(a, b) + case *Import: + b := b.(*Import) + return a.Compare(b) + case *Package: + b := b.(*Package) + return a.Compare(b) + case *Module: + b := b.(*Module) + return a.Compare(b) + } + panic(fmt.Sprintf("illegal value: %T", a)) +} + +type termSlice []*Term + +func (s termSlice) Less(i, j int) bool { return Compare(s[i].Value, s[j].Value) < 0 } +func (s termSlice) Swap(i, j int) { x := s[i]; s[i] = s[j]; s[j] = x } +func (s termSlice) Len() int { return len(s) } + +func sortOrder(x interface{}) int { + switch x.(type) { + case Null: + return 0 + case Boolean: + return 1 + case Number: + return 2 + case String: + return 3 + case Var: + return 4 + case Ref: + return 5 + case Array: + return 6 + case Object: + return 7 + case Set: + return 8 + case *ArrayComprehension: + return 9 + case *ObjectComprehension: + return 10 + case *SetComprehension: + return 11 + case Call: + return 12 + case Args: + return 13 + case *Expr: + return 100 + case *SomeDecl: + return 101 + case *With: + return 110 + case *Head: + return 120 + case Body: + return 200 + case *Rule: + return 1000 + case *Import: + return 1001 + case *Package: + return 1002 + case *Module: + return 10000 + } + panic(fmt.Sprintf("illegal value: %T", x)) +} + +func importsCompare(a, b []*Import) int { + minLen := len(a) + if len(b) < minLen { + minLen = len(b) + } + for i := 0; i < minLen; i++ { + if cmp := a[i].Compare(b[i]); cmp != 0 { + return cmp + } + } + if len(a) < len(b) { + return -1 + } + if len(b) < len(a) { + return 1 + } + return 0 +} + +func rulesCompare(a, b []*Rule) int { + minLen := len(a) + if len(b) < minLen { + minLen = len(b) + } + for i := 0; i < minLen; i++ { + if cmp := a[i].Compare(b[i]); cmp != 0 { + return cmp + } + } + if len(a) < len(b) { + return -1 + } + if len(b) < len(a) { + return 1 + } + return 0 +} + +func termSliceCompare(a, b []*Term) int { + minLen := len(a) + if len(b) < minLen { + minLen = len(b) + } + for i := 0; i < minLen; i++ { + if cmp := Compare(a[i], b[i]); cmp != 0 { + return cmp + } + } + if len(a) < len(b) { + return -1 + } else if len(b) < len(a) { + return 1 + } + return 0 +} + +func withSliceCompare(a, b []*With) int { + minLen := len(a) + if len(b) < minLen { + minLen = len(b) + } + for i := 0; i < minLen; i++ { + if cmp := Compare(a[i], b[i]); cmp != 0 { + return cmp + } + } + if len(a) < len(b) { + return -1 + } else if len(b) < len(a) { + return 1 + } + return 0 +} diff --git a/vendor/github.com/open-policy-agent/opa/ast/compile.go b/vendor/github.com/open-policy-agent/opa/ast/compile.go new file mode 100644 index 000000000..56edbe52c --- /dev/null +++ b/vendor/github.com/open-policy-agent/opa/ast/compile.go @@ -0,0 +1,3419 @@ +// Copyright 2016 The OPA Authors. All rights reserved. +// Use of this source code is governed by an Apache2 +// license that can be found in the LICENSE file. + +package ast + +import ( + "fmt" + "sort" + "strconv" + "strings" + + "github.com/open-policy-agent/opa/metrics" + "github.com/open-policy-agent/opa/util" +) + +// CompileErrorLimitDefault is the default number errors a compiler will allow before +// exiting. +const CompileErrorLimitDefault = 10 + +var errLimitReached = NewError(CompileErr, nil, "error limit reached") + +// Compiler contains the state of a compilation process. +type Compiler struct { + + // Errors contains errors that occurred during the compilation process. + // If there are one or more errors, the compilation process is considered + // "failed". + Errors Errors + + // Modules contains the compiled modules. The compiled modules are the + // output of the compilation process. If the compilation process failed, + // there is no guarantee about the state of the modules. + Modules map[string]*Module + + // ModuleTree organizes the modules into a tree where each node is keyed by + // an element in the module's package path. E.g., given modules containing + // the following package directives: "a", "a.b", "a.c", and "a.b", the + // resulting module tree would be: + // + // root + // | + // +--- data (no modules) + // | + // +--- a (1 module) + // | + // +--- b (2 modules) + // | + // +--- c (1 module) + // + ModuleTree *ModuleTreeNode + + // RuleTree organizes rules into a tree where each node is keyed by an + // element in the rule's path. The rule path is the concatenation of the + // containing package and the stringified rule name. E.g., given the + // following module: + // + // package ex + // p[1] { true } + // p[2] { true } + // q = true + // + // root + // | + // +--- data (no rules) + // | + // +--- ex (no rules) + // | + // +--- p (2 rules) + // | + // +--- q (1 rule) + RuleTree *TreeNode + + // Graph contains dependencies between rules. An edge (u,v) is added to the + // graph if rule 'u' refers to the virtual document defined by 'v'. + Graph *Graph + + // TypeEnv holds type information for values inferred by the compiler. + TypeEnv *TypeEnv + + // RewrittenVars is a mapping of variables that have been rewritten + // with the key being the generated name and value being the original. + RewrittenVars map[Var]Var + + localvargen *localVarGenerator + moduleLoader ModuleLoader + ruleIndices *util.HashMap + stages []struct { + name string + metricName string + f func() + } + maxErrs int + sorted []string // list of sorted module names + pathExists func([]string) (bool, error) + after map[string][]CompilerStageDefinition + metrics metrics.Metrics + builtins map[string]*Builtin + unsafeBuiltinsMap map[string]struct{} +} + +// CompilerStage defines the interface for stages in the compiler. +type CompilerStage func(*Compiler) *Error + +// CompilerStageDefinition defines a compiler stage +type CompilerStageDefinition struct { + Name string + MetricName string + Stage CompilerStage +} + +// QueryContext contains contextual information for running an ad-hoc query. +// +// Ad-hoc queries can be run in the context of a package and imports may be +// included to provide concise access to data. +type QueryContext struct { + Package *Package + Imports []*Import +} + +// NewQueryContext returns a new QueryContext object. +func NewQueryContext() *QueryContext { + return &QueryContext{} +} + +// WithPackage sets the pkg on qc. +func (qc *QueryContext) WithPackage(pkg *Package) *QueryContext { + if qc == nil { + qc = NewQueryContext() + } + qc.Package = pkg + return qc +} + +// WithImports sets the imports on qc. +func (qc *QueryContext) WithImports(imports []*Import) *QueryContext { + if qc == nil { + qc = NewQueryContext() + } + qc.Imports = imports + return qc +} + +// Copy returns a deep copy of qc. +func (qc *QueryContext) Copy() *QueryContext { + if qc == nil { + return nil + } + cpy := *qc + if cpy.Package != nil { + cpy.Package = qc.Package.Copy() + } + cpy.Imports = make([]*Import, len(qc.Imports)) + for i := range qc.Imports { + cpy.Imports[i] = qc.Imports[i].Copy() + } + return &cpy +} + +// QueryCompiler defines the interface for compiling ad-hoc queries. +type QueryCompiler interface { + + // Compile should be called to compile ad-hoc queries. The return value is + // the compiled version of the query. + Compile(q Body) (Body, error) + + // TypeEnv returns the type environment built after running type checking + // on the query. + TypeEnv() *TypeEnv + + // WithContext sets the QueryContext on the QueryCompiler. Subsequent calls + // to Compile will take the QueryContext into account. + WithContext(qctx *QueryContext) QueryCompiler + + // WithUnsafeBuiltins sets the built-in functions to treat as unsafe and not + // allow inside of queries. By default the query compiler inherits the + // compiler's unsafe built-in functions. This function allows callers to + // override that set. If an empty (non-nil) map is provided, all built-ins + // are allowed. + WithUnsafeBuiltins(unsafe map[string]struct{}) QueryCompiler + + // WithStageAfter registers a stage to run during query compilation after + // the named stage. + WithStageAfter(after string, stage QueryCompilerStageDefinition) QueryCompiler + + // RewrittenVars maps generated vars in the compiled query to vars from the + // parsed query. For example, given the query "input := 1" the rewritten + // query would be "__local0__ = 1". The mapping would then be {__local0__: input}. + RewrittenVars() map[Var]Var +} + +// QueryCompilerStage defines the interface for stages in the query compiler. +type QueryCompilerStage func(QueryCompiler, Body) (Body, error) + +// QueryCompilerStageDefinition defines a QueryCompiler stage +type QueryCompilerStageDefinition struct { + Name string + MetricName string + Stage QueryCompilerStage +} + +const compileStageMetricPrefex = "ast_compile_stage_" + +// NewCompiler returns a new empty compiler. +func NewCompiler() *Compiler { + + c := &Compiler{ + Modules: map[string]*Module{}, + TypeEnv: NewTypeEnv(), + RewrittenVars: map[Var]Var{}, + ruleIndices: util.NewHashMap(func(a, b util.T) bool { + r1, r2 := a.(Ref), b.(Ref) + return r1.Equal(r2) + }, func(x util.T) int { + return x.(Ref).Hash() + }), + maxErrs: CompileErrorLimitDefault, + after: map[string][]CompilerStageDefinition{}, + unsafeBuiltinsMap: map[string]struct{}{}, + } + + c.ModuleTree = NewModuleTree(nil) + c.RuleTree = NewRuleTree(c.ModuleTree) + + // Initialize the compiler with the statically compiled built-in functions. + // If the caller customizes the compiler, a copy will be made. + c.builtins = BuiltinMap + checker := newTypeChecker() + c.TypeEnv = checker.checkLanguageBuiltins(nil, c.builtins) + + c.stages = []struct { + name string + metricName string + f func() + }{ + // Reference resolution should run first as it may be used to lazily + // load additional modules. If any stages run before resolution, they + // need to be re-run after resolution. + {"ResolveRefs", "compile_stage_resolve_refs", c.resolveAllRefs}, + + // The local variable generator must be initialized after references are + // resolved and the dynamic module loader has run but before subsequent + // stages that need to generate variables. + {"InitLocalVarGen", "compile_stage_init_local_var_gen", c.initLocalVarGen}, + + {"RewriteLocalVars", "compile_stage_rewrite_local_vars", c.rewriteLocalVars}, + {"RewriteExprTerms", "compile_stage_rewrite_expr_terms", c.rewriteExprTerms}, + {"SetModuleTree", "compile_stage_set_module_tree", c.setModuleTree}, + {"SetRuleTree", "compile_stage_set_rule_tree", c.setRuleTree}, + {"SetGraph", "compile_stage_set_graph", c.setGraph}, + {"RewriteComprehensionTerms", "compile_stage_rewrite_comprehension_terms", c.rewriteComprehensionTerms}, + {"RewriteRefsInHead", "compile_stage_rewrite_refs_in_head", c.rewriteRefsInHead}, + {"RewriteWithValues", "compile_stage_rewrite_with_values", c.rewriteWithModifiers}, + {"CheckRuleConflicts", "compile_stage_check_rule_conflicts", c.checkRuleConflicts}, + {"CheckUndefinedFuncs", "compile_stage_check_undefined_funcs", c.checkUndefinedFuncs}, + {"CheckSafetyRuleHeads", "compile_stage_check_safety_rule_heads", c.checkSafetyRuleHeads}, + {"CheckSafetyRuleBodies", "compile_stage_check_safety_rule_bodies", c.checkSafetyRuleBodies}, + {"RewriteEquals", "compile_stage_rewrite_equals", c.rewriteEquals}, + {"RewriteDynamicTerms", "compile_stage_rewrite_dynamic_terms", c.rewriteDynamicTerms}, + {"CheckRecursion", "compile_stage_check_recursion", c.checkRecursion}, + {"CheckTypes", "compile_stage_check_types", c.checkTypes}, + {"CheckUnsafeBuiltins", "compile_state_check_unsafe_builtins", c.checkUnsafeBuiltins}, + {"BuildRuleIndices", "compile_stage_rebuild_indices", c.buildRuleIndices}, + } + + return c +} + +// SetErrorLimit sets the number of errors the compiler can encounter before it +// quits. Zero or a negative number indicates no limit. +func (c *Compiler) SetErrorLimit(limit int) *Compiler { + c.maxErrs = limit + return c +} + +// WithPathConflictsCheck enables base-virtual document conflict +// detection. The compiler will check that rules don't overlap with +// paths that exist as determined by the provided callable. +func (c *Compiler) WithPathConflictsCheck(fn func([]string) (bool, error)) *Compiler { + c.pathExists = fn + return c +} + +// WithStageAfter registers a stage to run during compilation after +// the named stage. +func (c *Compiler) WithStageAfter(after string, stage CompilerStageDefinition) *Compiler { + c.after[after] = append(c.after[after], stage) + return c +} + +// WithMetrics will set a metrics.Metrics and be used for profiling +// the Compiler instance. +func (c *Compiler) WithMetrics(metrics metrics.Metrics) *Compiler { + c.metrics = metrics + return c +} + +// WithBuiltins adds a set of custom built-in functions to the compiler. +func (c *Compiler) WithBuiltins(builtins map[string]*Builtin) *Compiler { + if len(builtins) == 0 { + return c + } + cpy := make(map[string]*Builtin, len(c.builtins)+len(builtins)) + for k, v := range c.builtins { + cpy[k] = v + } + for k, v := range builtins { + cpy[k] = v + } + c.builtins = cpy + // Build type env for custom functions and wrap existing one. + checker := newTypeChecker() + c.TypeEnv = checker.checkLanguageBuiltins(c.TypeEnv, builtins) + return c +} + +// WithUnsafeBuiltins will add all built-ins in the map to the "blacklist". +func (c *Compiler) WithUnsafeBuiltins(unsafeBuiltins map[string]struct{}) *Compiler { + for name := range unsafeBuiltins { + c.unsafeBuiltinsMap[name] = struct{}{} + } + return c +} + +// QueryCompiler returns a new QueryCompiler object. +func (c *Compiler) QueryCompiler() QueryCompiler { + return newQueryCompiler(c) +} + +// Compile runs the compilation process on the input modules. The compiled +// version of the modules and associated data structures are stored on the +// compiler. If the compilation process fails for any reason, the compiler will +// contain a slice of errors. +func (c *Compiler) Compile(modules map[string]*Module) { + + c.Modules = make(map[string]*Module, len(modules)) + + for k, v := range modules { + c.Modules[k] = v.Copy() + c.sorted = append(c.sorted, k) + } + + sort.Strings(c.sorted) + + c.compile() +} + +// Failed returns true if a compilation error has been encountered. +func (c *Compiler) Failed() bool { + return len(c.Errors) > 0 +} + +// GetArity returns the number of args a function referred to by ref takes. If +// ref refers to built-in function, the built-in declaration is consulted, +// otherwise, the ref is used to perform a ruleset lookup. +func (c *Compiler) GetArity(ref Ref) int { + if bi := c.builtins[ref.String()]; bi != nil { + return len(bi.Decl.Args()) + } + rules := c.GetRulesExact(ref) + if len(rules) == 0 { + return -1 + } + return len(rules[0].Head.Args) +} + +// GetRulesExact returns a slice of rules referred to by the reference. +// +// E.g., given the following module: +// +// package a.b.c +// +// p[k] = v { ... } # rule1 +// p[k1] = v1 { ... } # rule2 +// +// The following calls yield the rules on the right. +// +// GetRulesExact("data.a.b.c.p") => [rule1, rule2] +// GetRulesExact("data.a.b.c.p.x") => nil +// GetRulesExact("data.a.b.c") => nil +func (c *Compiler) GetRulesExact(ref Ref) (rules []*Rule) { + node := c.RuleTree + + for _, x := range ref { + if node = node.Child(x.Value); node == nil { + return nil + } + } + + return extractRules(node.Values) +} + +// GetRulesForVirtualDocument returns a slice of rules that produce the virtual +// document referred to by the reference. +// +// E.g., given the following module: +// +// package a.b.c +// +// p[k] = v { ... } # rule1 +// p[k1] = v1 { ... } # rule2 +// +// The following calls yield the rules on the right. +// +// GetRulesForVirtualDocument("data.a.b.c.p") => [rule1, rule2] +// GetRulesForVirtualDocument("data.a.b.c.p.x") => [rule1, rule2] +// GetRulesForVirtualDocument("data.a.b.c") => nil +func (c *Compiler) GetRulesForVirtualDocument(ref Ref) (rules []*Rule) { + + node := c.RuleTree + + for _, x := range ref { + if node = node.Child(x.Value); node == nil { + return nil + } + if len(node.Values) > 0 { + return extractRules(node.Values) + } + } + + return extractRules(node.Values) +} + +// GetRulesWithPrefix returns a slice of rules that share the prefix ref. +// +// E.g., given the following module: +// +// package a.b.c +// +// p[x] = y { ... } # rule1 +// p[k] = v { ... } # rule2 +// q { ... } # rule3 +// +// The following calls yield the rules on the right. +// +// GetRulesWithPrefix("data.a.b.c.p") => [rule1, rule2] +// GetRulesWithPrefix("data.a.b.c.p.a") => nil +// GetRulesWithPrefix("data.a.b.c") => [rule1, rule2, rule3] +func (c *Compiler) GetRulesWithPrefix(ref Ref) (rules []*Rule) { + + node := c.RuleTree + + for _, x := range ref { + if node = node.Child(x.Value); node == nil { + return nil + } + } + + var acc func(node *TreeNode) + + acc = func(node *TreeNode) { + rules = append(rules, extractRules(node.Values)...) + for _, child := range node.Children { + if child.Hide { + continue + } + acc(child) + } + } + + acc(node) + + return rules +} + +func extractRules(s []util.T) (rules []*Rule) { + for _, r := range s { + rules = append(rules, r.(*Rule)) + } + return rules +} + +// GetRules returns a slice of rules that are referred to by ref. +// +// E.g., given the following module: +// +// package a.b.c +// +// p[x] = y { q[x] = y; ... } # rule1 +// q[x] = y { ... } # rule2 +// +// The following calls yield the rules on the right. +// +// GetRules("data.a.b.c.p") => [rule1] +// GetRules("data.a.b.c.p.x") => [rule1] +// GetRules("data.a.b.c.q") => [rule2] +// GetRules("data.a.b.c") => [rule1, rule2] +// GetRules("data.a.b.d") => nil +func (c *Compiler) GetRules(ref Ref) (rules []*Rule) { + + set := map[*Rule]struct{}{} + + for _, rule := range c.GetRulesForVirtualDocument(ref) { + set[rule] = struct{}{} + } + + for _, rule := range c.GetRulesWithPrefix(ref) { + set[rule] = struct{}{} + } + + for rule := range set { + rules = append(rules, rule) + } + + return rules +} + +// GetRulesDynamic returns a slice of rules that could be referred to by a ref. +// When parts of the ref are statically known, we use that information to narrow +// down which rules the ref could refer to, but in the most general case this +// will be an over-approximation. +// +// E.g., given the following modules: +// +// package a.b.c +// +// r1 = 1 # rule1 +// +// and: +// +// package a.d.c +// +// r2 = 2 # rule2 +// +// The following calls yield the rules on the right. +// +// GetRulesDynamic("data.a[x].c[y]") => [rule1, rule2] +// GetRulesDynamic("data.a[x].c.r2") => [rule2] +// GetRulesDynamic("data.a.b[x][y]") => [rule1] +func (c *Compiler) GetRulesDynamic(ref Ref) (rules []*Rule) { + node := c.RuleTree + + set := map[*Rule]struct{}{} + var walk func(node *TreeNode, i int) + walk = func(node *TreeNode, i int) { + if i >= len(ref) { + // We've reached the end of the reference and want to collect everything + // under this "prefix". + node.DepthFirst(func(descendant *TreeNode) bool { + insertRules(set, descendant.Values) + return descendant.Hide + }) + } else if i == 0 || IsConstant(ref[i].Value) { + // The head of the ref is always grounded. In case another part of the + // ref is also grounded, we can lookup the exact child. If it's not found + // we can immediately return... + if child := node.Child(ref[i].Value); child == nil { + return + } else if len(child.Values) > 0 { + // If there are any rules at this position, it's what the ref would + // refer to. We can just append those and stop here. + insertRules(set, child.Values) + } else { + // Otherwise, we continue using the child node. + walk(child, i+1) + } + } else { + // This part of the ref is a dynamic term. We can't know what it refers + // to and will just need to try all of the children. + for _, child := range node.Children { + if child.Hide { + continue + } + insertRules(set, child.Values) + walk(child, i+1) + } + } + } + + walk(node, 0) + for rule := range set { + rules = append(rules, rule) + } + return rules +} + +// Utility: add all rule values to the set. +func insertRules(set map[*Rule]struct{}, rules []util.T) { + for _, rule := range rules { + set[rule.(*Rule)] = struct{}{} + } +} + +// RuleIndex returns a RuleIndex built for the rule set referred to by path. +// The path must refer to the rule set exactly, i.e., given a rule set at path +// data.a.b.c.p, refs data.a.b.c.p.x and data.a.b.c would not return a +// RuleIndex built for the rule. +func (c *Compiler) RuleIndex(path Ref) RuleIndex { + r, ok := c.ruleIndices.Get(path) + if !ok { + return nil + } + return r.(RuleIndex) +} + +// ModuleLoader defines the interface that callers can implement to enable lazy +// loading of modules during compilation. +type ModuleLoader func(resolved map[string]*Module) (parsed map[string]*Module, err error) + +// WithModuleLoader sets f as the ModuleLoader on the compiler. +// +// The compiler will invoke the ModuleLoader after resolving all references in +// the current set of input modules. The ModuleLoader can return a new +// collection of parsed modules that are to be included in the compilation +// process. This process will repeat until the ModuleLoader returns an empty +// collection or an error. If an error is returned, compilation will stop +// immediately. +func (c *Compiler) WithModuleLoader(f ModuleLoader) *Compiler { + c.moduleLoader = f + return c +} + +// buildRuleIndices constructs indices for rules. +func (c *Compiler) buildRuleIndices() { + + c.RuleTree.DepthFirst(func(node *TreeNode) bool { + if len(node.Values) == 0 { + return false + } + index := newBaseDocEqIndex(func(ref Ref) bool { + return isVirtual(c.RuleTree, ref.GroundPrefix()) + }) + if rules := extractRules(node.Values); index.Build(rules) { + c.ruleIndices.Put(rules[0].Path(), index) + } + return false + }) + +} + +// checkRecursion ensures that there are no recursive definitions, i.e., there are +// no cycles in the Graph. +func (c *Compiler) checkRecursion() { + eq := func(a, b util.T) bool { + return a.(*Rule) == b.(*Rule) + } + + c.RuleTree.DepthFirst(func(node *TreeNode) bool { + for _, rule := range node.Values { + for node := rule.(*Rule); node != nil; node = node.Else { + c.checkSelfPath(node.Loc(), eq, node, node) + } + } + return false + }) +} + +func (c *Compiler) checkSelfPath(loc *Location, eq func(a, b util.T) bool, a, b util.T) { + tr := NewGraphTraversal(c.Graph) + if p := util.DFSPath(tr, eq, a, b); len(p) > 0 { + n := []string{} + for _, x := range p { + n = append(n, astNodeToString(x)) + } + c.err(NewError(RecursionErr, loc, "rule %v is recursive: %v", astNodeToString(a), strings.Join(n, " -> "))) + } +} + +func astNodeToString(x interface{}) string { + switch x := x.(type) { + case *Rule: + return string(x.Head.Name) + default: + panic("not reached") + } +} + +// checkRuleConflicts ensures that rules definitions are not in conflict. +func (c *Compiler) checkRuleConflicts() { + c.RuleTree.DepthFirst(func(node *TreeNode) bool { + if len(node.Values) == 0 { + return false + } + + kinds := map[DocKind]struct{}{} + defaultRules := 0 + arities := map[int]struct{}{} + declared := false + + for _, rule := range node.Values { + r := rule.(*Rule) + kinds[r.Head.DocKind()] = struct{}{} + arities[len(r.Head.Args)] = struct{}{} + if r.Head.Assign { + declared = true + } + if r.Default { + defaultRules++ + } + } + + name := Var(node.Key.(String)) + + if declared && len(node.Values) > 1 { + c.err(NewError(TypeErr, node.Values[0].(*Rule).Loc(), "rule named %v redeclared at %v", name, node.Values[1].(*Rule).Loc())) + } else if len(kinds) > 1 || len(arities) > 1 { + c.err(NewError(TypeErr, node.Values[0].(*Rule).Loc(), "conflicting rules named %v found", name)) + } else if defaultRules > 1 { + c.err(NewError(TypeErr, node.Values[0].(*Rule).Loc(), "multiple default rules named %s found", name)) + } + + return false + }) + + if c.pathExists != nil { + for _, err := range CheckPathConflicts(c, c.pathExists) { + c.err(err) + } + } + + c.ModuleTree.DepthFirst(func(node *ModuleTreeNode) bool { + for _, mod := range node.Modules { + for _, rule := range mod.Rules { + if childNode, ok := node.Children[String(rule.Head.Name)]; ok { + for _, childMod := range childNode.Modules { + msg := fmt.Sprintf("%v conflicts with rule defined at %v", childMod.Package, rule.Loc()) + c.err(NewError(TypeErr, mod.Package.Loc(), msg)) + } + } + } + } + return false + }) +} + +func (c *Compiler) checkUndefinedFuncs() { + for _, name := range c.sorted { + m := c.Modules[name] + for _, err := range checkUndefinedFuncs(m, c.GetArity) { + c.err(err) + } + } +} + +func checkUndefinedFuncs(x interface{}, arity func(Ref) int) Errors { + + var errs Errors + + WalkExprs(x, func(expr *Expr) bool { + if !expr.IsCall() { + return false + } + ref := expr.Operator() + if arity(ref) >= 0 { + return false + } + errs = append(errs, NewError(TypeErr, expr.Loc(), "undefined function %v", ref)) + return true + }) + + return errs +} + +// checkSafetyRuleBodies ensures that variables appearing in negated expressions or non-target +// positions of built-in expressions will be bound when evaluating the rule from left +// to right, re-ordering as necessary. +func (c *Compiler) checkSafetyRuleBodies() { + for _, name := range c.sorted { + m := c.Modules[name] + WalkRules(m, func(r *Rule) bool { + safe := ReservedVars.Copy() + safe.Update(r.Head.Args.Vars()) + r.Body = c.checkBodySafety(safe, m, r.Body) + return false + }) + } +} + +func (c *Compiler) checkBodySafety(safe VarSet, m *Module, b Body) Body { + reordered, unsafe := reorderBodyForSafety(c.builtins, c.GetArity, safe, b) + if errs := safetyErrorSlice(unsafe); len(errs) > 0 { + for _, err := range errs { + c.err(err) + } + return b + } + return reordered +} + +var safetyCheckVarVisitorParams = VarVisitorParams{ + SkipRefCallHead: true, + SkipClosures: true, +} + +// checkSafetyRuleHeads ensures that variables appearing in the head of a +// rule also appear in the body. +func (c *Compiler) checkSafetyRuleHeads() { + + for _, name := range c.sorted { + m := c.Modules[name] + WalkRules(m, func(r *Rule) bool { + safe := r.Body.Vars(safetyCheckVarVisitorParams) + safe.Update(r.Head.Args.Vars()) + unsafe := r.Head.Vars().Diff(safe) + for v := range unsafe { + if !v.IsGenerated() { + c.err(NewError(UnsafeVarErr, r.Loc(), "var %v is unsafe", v)) + } + } + return false + }) + } +} + +// checkTypes runs the type checker on all rules. The type checker builds a +// TypeEnv that is stored on the compiler. +func (c *Compiler) checkTypes() { + // Recursion is caught in earlier step, so this cannot fail. + sorted, _ := c.Graph.Sort() + checker := newTypeChecker().WithVarRewriter(rewriteVarsInRef(c.RewrittenVars)) + env, errs := checker.CheckTypes(c.TypeEnv, sorted) + for _, err := range errs { + c.err(err) + } + c.TypeEnv = env +} + +func (c *Compiler) checkUnsafeBuiltins() { + for _, name := range c.sorted { + errs := checkUnsafeBuiltins(c.unsafeBuiltinsMap, c.Modules[name]) + for _, err := range errs { + c.err(err) + } + } +} + +func (c *Compiler) runStage(metricName string, f func()) { + if c.metrics != nil { + c.metrics.Timer(metricName).Start() + defer c.metrics.Timer(metricName).Stop() + } + f() +} + +func (c *Compiler) runStageAfter(metricName string, s CompilerStage) *Error { + if c.metrics != nil { + c.metrics.Timer(metricName).Start() + defer c.metrics.Timer(metricName).Stop() + } + return s(c) +} + +func (c *Compiler) compile() { + defer func() { + if r := recover(); r != nil && r != errLimitReached { + panic(r) + } + }() + + for _, s := range c.stages { + c.runStage(s.metricName, s.f) + if c.Failed() { + return + } + for _, s := range c.after[s.name] { + err := c.runStageAfter(s.MetricName, s.Stage) + if err != nil { + c.err(err) + } + } + } +} + +func (c *Compiler) err(err *Error) { + if c.maxErrs > 0 && len(c.Errors) >= c.maxErrs { + c.Errors = append(c.Errors, errLimitReached) + panic(errLimitReached) + } + c.Errors = append(c.Errors, err) +} + +func (c *Compiler) getExports() *util.HashMap { + + rules := util.NewHashMap(func(a, b util.T) bool { + r1 := a.(Ref) + r2 := a.(Ref) + return r1.Equal(r2) + }, func(v util.T) int { + return v.(Ref).Hash() + }) + + for _, name := range c.sorted { + mod := c.Modules[name] + rv, ok := rules.Get(mod.Package.Path) + if !ok { + rv = []Var{} + } + rvs := rv.([]Var) + + for _, rule := range mod.Rules { + rvs = append(rvs, rule.Head.Name) + } + rules.Put(mod.Package.Path, rvs) + } + + return rules +} + +// resolveAllRefs resolves references in expressions to their fully qualified values. +// +// For instance, given the following module: +// +// package a.b +// import data.foo.bar +// p[x] { bar[_] = x } +// +// The reference "bar[_]" would be resolved to "data.foo.bar[_]". +func (c *Compiler) resolveAllRefs() { + + rules := c.getExports() + + for _, name := range c.sorted { + mod := c.Modules[name] + + var ruleExports []Var + if x, ok := rules.Get(mod.Package.Path); ok { + ruleExports = x.([]Var) + } + + globals := getGlobals(mod.Package, ruleExports, mod.Imports) + + WalkRules(mod, func(rule *Rule) bool { + err := resolveRefsInRule(globals, rule) + if err != nil { + c.err(NewError(CompileErr, rule.Location, err.Error())) + } + return false + }) + + // Once imports have been resolved, they are no longer needed. + mod.Imports = nil + } + + if c.moduleLoader != nil { + + parsed, err := c.moduleLoader(c.Modules) + if err != nil { + c.err(NewError(CompileErr, nil, err.Error())) + return + } + + if len(parsed) == 0 { + return + } + + for id, module := range parsed { + c.Modules[id] = module.Copy() + c.sorted = append(c.sorted, id) + } + + sort.Strings(c.sorted) + c.resolveAllRefs() + } +} + +func (c *Compiler) initLocalVarGen() { + c.localvargen = newLocalVarGeneratorForModuleSet(c.sorted, c.Modules) +} + +func (c *Compiler) rewriteComprehensionTerms() { + f := newEqualityFactory(c.localvargen) + for _, name := range c.sorted { + mod := c.Modules[name] + rewriteComprehensionTerms(f, mod) + } +} + +func (c *Compiler) rewriteExprTerms() { + for _, name := range c.sorted { + mod := c.Modules[name] + WalkRules(mod, func(rule *Rule) bool { + rewriteExprTermsInHead(c.localvargen, rule) + rule.Body = rewriteExprTermsInBody(c.localvargen, rule.Body) + return false + }) + } +} + +// rewriteTermsInHead will rewrite rules so that the head does not contain any +// terms that require evaluation (e.g., refs or comprehensions). If the key or +// value contains or more of these terms, the key or value will be moved into +// the body and assigned to a new variable. The new variable will replace the +// key or value in the head. +// +// For instance, given the following rule: +// +// p[{"foo": data.foo[i]}] { i < 100 } +// +// The rule would be re-written as: +// +// p[__local0__] { i < 100; __local0__ = {"foo": data.foo[i]} } +func (c *Compiler) rewriteRefsInHead() { + f := newEqualityFactory(c.localvargen) + for _, name := range c.sorted { + mod := c.Modules[name] + WalkRules(mod, func(rule *Rule) bool { + if requiresEval(rule.Head.Key) { + expr := f.Generate(rule.Head.Key) + rule.Head.Key = expr.Operand(0) + rule.Body.Append(expr) + } + if requiresEval(rule.Head.Value) { + expr := f.Generate(rule.Head.Value) + rule.Head.Value = expr.Operand(0) + rule.Body.Append(expr) + } + for i := 0; i < len(rule.Head.Args); i++ { + if requiresEval(rule.Head.Args[i]) { + expr := f.Generate(rule.Head.Args[i]) + rule.Head.Args[i] = expr.Operand(0) + rule.Body.Append(expr) + } + } + return false + }) + } +} + +func (c *Compiler) rewriteEquals() { + for _, name := range c.sorted { + mod := c.Modules[name] + rewriteEquals(mod) + } +} + +func (c *Compiler) rewriteDynamicTerms() { + f := newEqualityFactory(c.localvargen) + for _, name := range c.sorted { + mod := c.Modules[name] + WalkRules(mod, func(rule *Rule) bool { + rule.Body = rewriteDynamics(f, rule.Body) + return false + }) + } +} + +func (c *Compiler) rewriteLocalVars() { + + for _, name := range c.sorted { + mod := c.Modules[name] + gen := c.localvargen + + WalkRules(mod, func(rule *Rule) bool { + + var errs Errors + + // Rewrite assignments contained in head of rule. Assignments can + // occur in rule head if they're inside a comprehension. Note, + // assigned vars in comprehensions in the head will be rewritten + // first to preserve scoping rules. For example: + // + // p = [x | x := 1] { x := 2 } becomes p = [__local0__ | __local0__ = 1] { __local1__ = 2 } + // + // This behaviour is consistent scoping inside the body. For example: + // + // p = xs { x := 2; xs = [x | x := 1] } becomes p = xs { __local0__ = 2; xs = [__local1__ | __local1__ = 1] } + WalkTerms(rule.Head, func(term *Term) bool { + stop := false + stack := newLocalDeclaredVars() + switch v := term.Value.(type) { + case *ArrayComprehension: + errs = rewriteDeclaredVarsInArrayComprehension(gen, stack, v, errs) + stop = true + case *SetComprehension: + errs = rewriteDeclaredVarsInSetComprehension(gen, stack, v, errs) + stop = true + case *ObjectComprehension: + errs = rewriteDeclaredVarsInObjectComprehension(gen, stack, v, errs) + stop = true + } + + for k, v := range stack.rewritten { + c.RewrittenVars[k] = v + } + + return stop + }) + + for _, err := range errs { + c.err(err) + } + + // Rewrite assignments in body. + used := NewVarSet() + + if rule.Head.Key != nil { + used.Update(rule.Head.Key.Vars()) + } + + if rule.Head.Value != nil { + used.Update(rule.Head.Value.Vars()) + } + + stack := newLocalDeclaredVars() + + c.rewriteLocalArgVars(gen, stack, rule) + + body, declared, errs := rewriteLocalVars(gen, stack, used, rule.Body) + for _, err := range errs { + c.err(err) + } + + // For rewritten vars use the collection of all variables that + // were in the stack at some point in time. + for k, v := range stack.rewritten { + c.RewrittenVars[k] = v + } + + rule.Body = body + + // Rewrite vars in head that refer to locally declared vars in the body. + vis := NewGenericVisitor(func(x interface{}) bool { + + term, ok := x.(*Term) + if !ok { + return false + } + + switch v := term.Value.(type) { + case Object: + // Make a copy of the object because the keys may be mutated. + cpy, _ := v.Map(func(k, v *Term) (*Term, *Term, error) { + if vark, ok := k.Value.(Var); ok { + if gv, ok := declared[vark]; ok { + k = k.Copy() + k.Value = gv + } + } + return k, v, nil + }) + term.Value = cpy + case Var: + if gv, ok := declared[v]; ok { + term.Value = gv + return true + } + } + + return false + }) + + vis.Walk(rule.Head.Args) + + if rule.Head.Key != nil { + vis.Walk(rule.Head.Key) + } + + if rule.Head.Value != nil { + vis.Walk(rule.Head.Value) + } + + return false + }) + } +} + +func (c *Compiler) rewriteLocalArgVars(gen *localVarGenerator, stack *localDeclaredVars, rule *Rule) { + + vis := &ruleArgLocalRewriter{ + stack: stack, + gen: gen, + } + + for i := range rule.Head.Args { + Walk(vis, rule.Head.Args[i]) + } + + for i := range vis.errs { + c.err(vis.errs[i]) + } +} + +type ruleArgLocalRewriter struct { + stack *localDeclaredVars + gen *localVarGenerator + errs []*Error +} + +func (vis *ruleArgLocalRewriter) Visit(x interface{}) Visitor { + + t, ok := x.(*Term) + if !ok { + return vis + } + + switch v := t.Value.(type) { + case Var: + gv, ok := vis.stack.Declared(v) + if !ok { + gv = vis.gen.Generate() + vis.stack.Insert(v, gv, argVar) + } + t.Value = gv + return nil + case Object: + if cpy, err := v.Map(func(k, v *Term) (*Term, *Term, error) { + vcpy := v.Copy() + Walk(vis, vcpy) + return k, vcpy, nil + }); err != nil { + vis.errs = append(vis.errs, NewError(CompileErr, t.Location, err.Error())) + } else { + t.Value = cpy + } + return nil + case Null, Boolean, Number, String, *ArrayComprehension, *SetComprehension, *ObjectComprehension, Set: + // Scalars are no-ops. Comprehensions are handled above. Sets must not + // contain variables. + return nil + default: + // Recurse on refs, arrays, and calls. Any embedded + // variables can be rewritten. + return vis + } +} + +func (c *Compiler) rewriteWithModifiers() { + f := newEqualityFactory(c.localvargen) + for _, name := range c.sorted { + mod := c.Modules[name] + t := NewGenericTransformer(func(x interface{}) (interface{}, error) { + body, ok := x.(Body) + if !ok { + return x, nil + } + body, err := rewriteWithModifiersInBody(c, f, body) + if err != nil { + c.err(err) + } + + return body, nil + }) + Transform(t, mod) + } +} + +func (c *Compiler) setModuleTree() { + c.ModuleTree = NewModuleTree(c.Modules) +} + +func (c *Compiler) setRuleTree() { + c.RuleTree = NewRuleTree(c.ModuleTree) +} + +func (c *Compiler) setGraph() { + c.Graph = NewGraph(c.Modules, c.GetRulesDynamic) +} + +type queryCompiler struct { + compiler *Compiler + qctx *QueryContext + typeEnv *TypeEnv + rewritten map[Var]Var + after map[string][]QueryCompilerStageDefinition + unsafeBuiltins map[string]struct{} +} + +func newQueryCompiler(compiler *Compiler) QueryCompiler { + qc := &queryCompiler{ + compiler: compiler, + qctx: nil, + after: map[string][]QueryCompilerStageDefinition{}, + } + return qc +} + +func (qc *queryCompiler) WithContext(qctx *QueryContext) QueryCompiler { + qc.qctx = qctx + return qc +} + +func (qc *queryCompiler) WithStageAfter(after string, stage QueryCompilerStageDefinition) QueryCompiler { + qc.after[after] = append(qc.after[after], stage) + return qc +} + +func (qc *queryCompiler) WithUnsafeBuiltins(unsafe map[string]struct{}) QueryCompiler { + qc.unsafeBuiltins = unsafe + return qc +} + +func (qc *queryCompiler) RewrittenVars() map[Var]Var { + return qc.rewritten +} + +func (qc *queryCompiler) runStage(metricName string, qctx *QueryContext, query Body, s func(*QueryContext, Body) (Body, error)) (Body, error) { + if qc.compiler.metrics != nil { + qc.compiler.metrics.Timer(metricName).Start() + defer qc.compiler.metrics.Timer(metricName).Stop() + } + return s(qctx, query) +} + +func (qc *queryCompiler) runStageAfter(metricName string, query Body, s QueryCompilerStage) (Body, error) { + if qc.compiler.metrics != nil { + qc.compiler.metrics.Timer(metricName).Start() + defer qc.compiler.metrics.Timer(metricName).Stop() + } + return s(qc, query) +} + +func (qc *queryCompiler) Compile(query Body) (Body, error) { + + query = query.Copy() + + stages := []struct { + name string + metricName string + f func(*QueryContext, Body) (Body, error) + }{ + {"ResolveRefs", "query_compile_stage_resolve_refs", qc.resolveRefs}, + {"RewriteLocalVars", "query_compile_stage_rewrite_local_vars", qc.rewriteLocalVars}, + {"RewriteExprTerms", "query_compile_stage_rewrite_expr_terms", qc.rewriteExprTerms}, + {"RewriteComprehensionTerms", "query_compile_stage_rewrite_comprehension_terms", qc.rewriteComprehensionTerms}, + {"RewriteWithValues", "query_compile_stage_rewrite_with_values", qc.rewriteWithModifiers}, + {"CheckUndefinedFuncs", "query_compile_stage_check_undefined_funcs", qc.checkUndefinedFuncs}, + {"CheckSafety", "query_compile_stage_check_safety", qc.checkSafety}, + {"RewriteDynamicTerms", "query_compile_stage_rewrite_dynamic_terms", qc.rewriteDynamicTerms}, + {"CheckTypes", "query_compile_stage_check_types", qc.checkTypes}, + {"CheckUnsafeBuiltins", "query_compile_stage_check_unsafe_builtins", qc.checkUnsafeBuiltins}, + } + + qctx := qc.qctx.Copy() + + for _, s := range stages { + var err error + query, err = qc.runStage(s.metricName, qctx, query, s.f) + if err != nil { + return nil, qc.applyErrorLimit(err) + } + for _, s := range qc.after[s.name] { + query, err = qc.runStageAfter(s.MetricName, query, s.Stage) + if err != nil { + return nil, qc.applyErrorLimit(err) + } + } + } + + return query, nil +} + +func (qc *queryCompiler) TypeEnv() *TypeEnv { + return qc.typeEnv +} + +func (qc *queryCompiler) applyErrorLimit(err error) error { + if errs, ok := err.(Errors); ok { + if qc.compiler.maxErrs > 0 && len(errs) > qc.compiler.maxErrs { + err = append(errs[:qc.compiler.maxErrs], errLimitReached) + } + } + return err +} + +func (qc *queryCompiler) resolveRefs(qctx *QueryContext, body Body) (Body, error) { + + var globals map[Var]Ref + + if qctx != nil && qctx.Package != nil { + var ruleExports []Var + rules := qc.compiler.getExports() + if exist, ok := rules.Get(qctx.Package.Path); ok { + ruleExports = exist.([]Var) + } + + globals = getGlobals(qctx.Package, ruleExports, qc.qctx.Imports) + qctx.Imports = nil + } + + ignore := &declaredVarStack{declaredVars(body)} + + return resolveRefsInBody(globals, ignore, body), nil +} + +func (qc *queryCompiler) rewriteComprehensionTerms(_ *QueryContext, body Body) (Body, error) { + gen := newLocalVarGenerator("q", body) + f := newEqualityFactory(gen) + node, err := rewriteComprehensionTerms(f, body) + if err != nil { + return nil, err + } + return node.(Body), nil +} + +func (qc *queryCompiler) rewriteDynamicTerms(_ *QueryContext, body Body) (Body, error) { + gen := newLocalVarGenerator("q", body) + f := newEqualityFactory(gen) + return rewriteDynamics(f, body), nil +} + +func (qc *queryCompiler) rewriteExprTerms(_ *QueryContext, body Body) (Body, error) { + gen := newLocalVarGenerator("q", body) + return rewriteExprTermsInBody(gen, body), nil +} + +func (qc *queryCompiler) rewriteLocalVars(_ *QueryContext, body Body) (Body, error) { + gen := newLocalVarGenerator("q", body) + stack := newLocalDeclaredVars() + body, _, err := rewriteLocalVars(gen, stack, nil, body) + if len(err) != 0 { + return nil, err + } + qc.rewritten = make(map[Var]Var, len(stack.rewritten)) + for k, v := range stack.rewritten { + // The vars returned during the rewrite will include all seen vars, + // even if they're not declared with an assignment operation. We don't + // want to include these inside the rewritten set though. + qc.rewritten[k] = v + } + return body, nil +} + +func (qc *queryCompiler) checkUndefinedFuncs(_ *QueryContext, body Body) (Body, error) { + if errs := checkUndefinedFuncs(body, qc.compiler.GetArity); len(errs) > 0 { + return nil, errs + } + return body, nil +} + +func (qc *queryCompiler) checkSafety(_ *QueryContext, body Body) (Body, error) { + safe := ReservedVars.Copy() + reordered, unsafe := reorderBodyForSafety(qc.compiler.builtins, qc.compiler.GetArity, safe, body) + if errs := safetyErrorSlice(unsafe); len(errs) > 0 { + return nil, errs + } + return reordered, nil +} + +func (qc *queryCompiler) checkTypes(qctx *QueryContext, body Body) (Body, error) { + var errs Errors + checker := newTypeChecker().WithVarRewriter(rewriteVarsInRef(qc.rewritten, qc.compiler.RewrittenVars)) + qc.typeEnv, errs = checker.CheckBody(qc.compiler.TypeEnv, body) + if len(errs) > 0 { + return nil, errs + } + return body, nil +} + +func (qc *queryCompiler) checkUnsafeBuiltins(qctx *QueryContext, body Body) (Body, error) { + var unsafe map[string]struct{} + if qc.unsafeBuiltins != nil { + unsafe = qc.unsafeBuiltins + } else { + unsafe = qc.compiler.unsafeBuiltinsMap + } + errs := checkUnsafeBuiltins(unsafe, body) + if len(errs) > 0 { + return nil, errs + } + return body, nil +} + +func (qc *queryCompiler) rewriteWithModifiers(qctx *QueryContext, body Body) (Body, error) { + f := newEqualityFactory(newLocalVarGenerator("q", body)) + body, err := rewriteWithModifiersInBody(qc.compiler, f, body) + if err != nil { + return nil, Errors{err} + } + return body, nil +} + +// ModuleTreeNode represents a node in the module tree. The module +// tree is keyed by the package path. +type ModuleTreeNode struct { + Key Value + Modules []*Module + Children map[Value]*ModuleTreeNode + Hide bool +} + +// NewModuleTree returns a new ModuleTreeNode that represents the root +// of the module tree populated with the given modules. +func NewModuleTree(mods map[string]*Module) *ModuleTreeNode { + root := &ModuleTreeNode{ + Children: map[Value]*ModuleTreeNode{}, + } + for _, m := range mods { + node := root + for i, x := range m.Package.Path { + c, ok := node.Children[x.Value] + if !ok { + var hide bool + if i == 1 && x.Value.Compare(SystemDocumentKey) == 0 { + hide = true + } + c = &ModuleTreeNode{ + Key: x.Value, + Children: map[Value]*ModuleTreeNode{}, + Hide: hide, + } + node.Children[x.Value] = c + } + node = c + } + node.Modules = append(node.Modules, m) + } + return root +} + +// Size returns the number of modules in the tree. +func (n *ModuleTreeNode) Size() int { + s := len(n.Modules) + for _, c := range n.Children { + s += c.Size() + } + return s +} + +// DepthFirst performs a depth-first traversal of the module tree rooted at n. +// If f returns true, traversal will not continue to the children of n. +func (n *ModuleTreeNode) DepthFirst(f func(node *ModuleTreeNode) bool) { + if !f(n) { + for _, node := range n.Children { + node.DepthFirst(f) + } + } +} + +// TreeNode represents a node in the rule tree. The rule tree is keyed by +// rule path. +type TreeNode struct { + Key Value + Values []util.T + Children map[Value]*TreeNode + Hide bool +} + +// NewRuleTree returns a new TreeNode that represents the root +// of the rule tree populated with the given rules. +func NewRuleTree(mtree *ModuleTreeNode) *TreeNode { + + ruleSets := map[String][]util.T{} + + // Build rule sets for this package. + for _, mod := range mtree.Modules { + for _, rule := range mod.Rules { + key := String(rule.Head.Name) + ruleSets[key] = append(ruleSets[key], rule) + } + } + + // Each rule set becomes a leaf node. + children := map[Value]*TreeNode{} + + for key, rules := range ruleSets { + children[key] = &TreeNode{ + Key: key, + Children: nil, + Values: rules, + } + } + + // Each module in subpackage becomes child node. + for _, child := range mtree.Children { + children[child.Key] = NewRuleTree(child) + } + + return &TreeNode{ + Key: mtree.Key, + Values: nil, + Children: children, + Hide: mtree.Hide, + } +} + +// Size returns the number of rules in the tree. +func (n *TreeNode) Size() int { + s := len(n.Values) + for _, c := range n.Children { + s += c.Size() + } + return s +} + +// Child returns n's child with key k. +func (n *TreeNode) Child(k Value) *TreeNode { + switch k.(type) { + case String, Var: + return n.Children[k] + } + return nil +} + +// DepthFirst performs a depth-first traversal of the rule tree rooted at n. If +// f returns true, traversal will not continue to the children of n. +func (n *TreeNode) DepthFirst(f func(node *TreeNode) bool) { + if !f(n) { + for _, node := range n.Children { + node.DepthFirst(f) + } + } +} + +// Graph represents the graph of dependencies between rules. +type Graph struct { + adj map[util.T]map[util.T]struct{} + nodes map[util.T]struct{} + sorted []util.T +} + +// NewGraph returns a new Graph based on modules. The list function must return +// the rules referred to directly by the ref. +func NewGraph(modules map[string]*Module, list func(Ref) []*Rule) *Graph { + + graph := &Graph{ + adj: map[util.T]map[util.T]struct{}{}, + nodes: map[util.T]struct{}{}, + sorted: nil, + } + + // Create visitor to walk a rule AST and add edges to the rule graph for + // each dependency. + vis := func(a *Rule) *GenericVisitor { + stop := false + return NewGenericVisitor(func(x interface{}) bool { + switch x := x.(type) { + case Ref: + for _, b := range list(x) { + for node := b; node != nil; node = node.Else { + graph.addDependency(a, node) + } + } + case *Rule: + if stop { + // Do not recurse into else clauses (which will be handled + // by the outer visitor.) + return true + } + stop = true + } + return false + }) + } + + // Walk over all rules, add them to graph, and build adjencency lists. + for _, module := range modules { + WalkRules(module, func(a *Rule) bool { + graph.addNode(a) + vis(a).Walk(a) + return false + }) + } + + return graph +} + +// Dependencies returns the set of rules that x depends on. +func (g *Graph) Dependencies(x util.T) map[util.T]struct{} { + return g.adj[x] +} + +// Sort returns a slice of rules sorted by dependencies. If a cycle is found, +// ok is set to false. +func (g *Graph) Sort() (sorted []util.T, ok bool) { + if g.sorted != nil { + return g.sorted, true + } + + sort := &graphSort{ + sorted: make([]util.T, 0, len(g.nodes)), + deps: g.Dependencies, + marked: map[util.T]struct{}{}, + temp: map[util.T]struct{}{}, + } + + for node := range g.nodes { + if !sort.Visit(node) { + return nil, false + } + } + + g.sorted = sort.sorted + return g.sorted, true +} + +func (g *Graph) addDependency(u util.T, v util.T) { + + if _, ok := g.nodes[u]; !ok { + g.addNode(u) + } + + if _, ok := g.nodes[v]; !ok { + g.addNode(v) + } + + edges, ok := g.adj[u] + if !ok { + edges = map[util.T]struct{}{} + g.adj[u] = edges + } + + edges[v] = struct{}{} +} + +func (g *Graph) addNode(n util.T) { + g.nodes[n] = struct{}{} +} + +type graphSort struct { + sorted []util.T + deps func(util.T) map[util.T]struct{} + marked map[util.T]struct{} + temp map[util.T]struct{} +} + +func (sort *graphSort) Marked(node util.T) bool { + _, marked := sort.marked[node] + return marked +} + +func (sort *graphSort) Visit(node util.T) (ok bool) { + if _, ok := sort.temp[node]; ok { + return false + } + if sort.Marked(node) { + return true + } + sort.temp[node] = struct{}{} + for other := range sort.deps(node) { + if !sort.Visit(other) { + return false + } + } + sort.marked[node] = struct{}{} + delete(sort.temp, node) + sort.sorted = append(sort.sorted, node) + return true +} + +// GraphTraversal is a Traversal that understands the dependency graph +type GraphTraversal struct { + graph *Graph + visited map[util.T]struct{} +} + +// NewGraphTraversal returns a Traversal for the dependency graph +func NewGraphTraversal(graph *Graph) *GraphTraversal { + return &GraphTraversal{ + graph: graph, + visited: map[util.T]struct{}{}, + } +} + +// Edges lists all dependency connections for a given node +func (g *GraphTraversal) Edges(x util.T) []util.T { + r := []util.T{} + for v := range g.graph.Dependencies(x) { + r = append(r, v) + } + return r +} + +// Visited returns whether a node has been visited, setting a node to visited if not +func (g *GraphTraversal) Visited(u util.T) bool { + _, ok := g.visited[u] + g.visited[u] = struct{}{} + return ok +} + +type unsafePair struct { + Expr *Expr + Vars VarSet +} + +type unsafeVarLoc struct { + Var Var + Loc *Location +} + +type unsafeVars map[*Expr]VarSet + +func (vs unsafeVars) Add(e *Expr, v Var) { + if u, ok := vs[e]; ok { + u[v] = struct{}{} + } else { + vs[e] = VarSet{v: struct{}{}} + } +} + +func (vs unsafeVars) Set(e *Expr, s VarSet) { + vs[e] = s +} + +func (vs unsafeVars) Update(o unsafeVars) { + for k, v := range o { + if _, ok := vs[k]; !ok { + vs[k] = VarSet{} + } + vs[k].Update(v) + } +} + +func (vs unsafeVars) Vars() (result []unsafeVarLoc) { + + locs := map[Var]*Location{} + + // If var appears in multiple sets then pick first by location. + for expr, vars := range vs { + for v := range vars { + if locs[v].Compare(expr.Location) > 0 { + locs[v] = expr.Location + } + } + } + + for v, loc := range locs { + result = append(result, unsafeVarLoc{ + Var: v, + Loc: loc, + }) + } + + sort.Slice(result, func(i, j int) bool { + return result[i].Loc.Compare(result[j].Loc) < 0 + }) + + return result +} + +func (vs unsafeVars) Slice() (result []unsafePair) { + for expr, vs := range vs { + result = append(result, unsafePair{ + Expr: expr, + Vars: vs, + }) + } + return +} + +// reorderBodyForSafety returns a copy of the body ordered such that +// left to right evaluation of the body will not encounter unbound variables +// in input positions or negated expressions. +// +// Expressions are added to the re-ordered body as soon as they are considered +// safe. If multiple expressions become safe in the same pass, they are added +// in their original order. This results in minimal re-ordering of the body. +// +// If the body cannot be reordered to ensure safety, the second return value +// contains a mapping of expressions to unsafe variables in those expressions. +func reorderBodyForSafety(builtins map[string]*Builtin, arity func(Ref) int, globals VarSet, body Body) (Body, unsafeVars) { + + body, unsafe := reorderBodyForClosures(builtins, arity, globals, body) + if len(unsafe) != 0 { + return nil, unsafe + } + + reordered := Body{} + safe := VarSet{} + + for _, e := range body { + for v := range e.Vars(safetyCheckVarVisitorParams) { + if globals.Contains(v) { + safe.Add(v) + } else { + unsafe.Add(e, v) + } + } + } + + for { + n := len(reordered) + + for _, e := range body { + if reordered.Contains(e) { + continue + } + + safe.Update(outputVarsForExpr(e, builtins, arity, safe)) + + for v := range unsafe[e] { + if safe.Contains(v) { + delete(unsafe[e], v) + } + } + + if len(unsafe[e]) == 0 { + delete(unsafe, e) + reordered = append(reordered, e) + } + } + + if len(reordered) == n { + break + } + } + + // Recursively visit closures and perform the safety checks on them. + // Update the globals at each expression to include the variables that could + // be closed over. + g := globals.Copy() + for i, e := range reordered { + if i > 0 { + g.Update(reordered[i-1].Vars(safetyCheckVarVisitorParams)) + } + vis := &bodySafetyVisitor{ + builtins: builtins, + arity: arity, + current: e, + globals: g, + unsafe: unsafe, + } + NewGenericVisitor(vis.Visit).Walk(e) + } + + // Need to reset expression indices as re-ordering may have + // changed them. + setExprIndices(reordered) + + return reordered, unsafe +} + +type bodySafetyVisitor struct { + builtins map[string]*Builtin + arity func(Ref) int + current *Expr + globals VarSet + unsafe unsafeVars +} + +func (vis *bodySafetyVisitor) Visit(x interface{}) bool { + switch x := x.(type) { + case *Expr: + cpy := *vis + cpy.current = x + + switch ts := x.Terms.(type) { + case *SomeDecl: + NewGenericVisitor(cpy.Visit).Walk(ts) + case []*Term: + for _, t := range ts { + NewGenericVisitor(cpy.Visit).Walk(t) + } + case *Term: + NewGenericVisitor(cpy.Visit).Walk(ts) + } + for i := range x.With { + NewGenericVisitor(cpy.Visit).Walk(x.With[i]) + } + return true + case *ArrayComprehension: + vis.checkArrayComprehensionSafety(x) + return true + case *ObjectComprehension: + vis.checkObjectComprehensionSafety(x) + return true + case *SetComprehension: + vis.checkSetComprehensionSafety(x) + return true + } + return false +} + +// Check term for safety. This is analogous to the rule head safety check. +func (vis *bodySafetyVisitor) checkComprehensionSafety(tv VarSet, body Body) Body { + bv := body.Vars(safetyCheckVarVisitorParams) + bv.Update(vis.globals) + uv := tv.Diff(bv) + for v := range uv { + vis.unsafe.Add(vis.current, v) + } + + // Check body for safety, reordering as necessary. + r, u := reorderBodyForSafety(vis.builtins, vis.arity, vis.globals, body) + if len(u) == 0 { + return r + } + + vis.unsafe.Update(u) + return body +} + +func (vis *bodySafetyVisitor) checkArrayComprehensionSafety(ac *ArrayComprehension) { + ac.Body = vis.checkComprehensionSafety(ac.Term.Vars(), ac.Body) +} + +func (vis *bodySafetyVisitor) checkObjectComprehensionSafety(oc *ObjectComprehension) { + tv := oc.Key.Vars() + tv.Update(oc.Value.Vars()) + oc.Body = vis.checkComprehensionSafety(tv, oc.Body) +} + +func (vis *bodySafetyVisitor) checkSetComprehensionSafety(sc *SetComprehension) { + sc.Body = vis.checkComprehensionSafety(sc.Term.Vars(), sc.Body) +} + +// reorderBodyForClosures returns a copy of the body ordered such that +// expressions (such as array comprehensions) that close over variables are ordered +// after other expressions that contain the same variable in an output position. +func reorderBodyForClosures(builtins map[string]*Builtin, arity func(Ref) int, globals VarSet, body Body) (Body, unsafeVars) { + + reordered := Body{} + unsafe := unsafeVars{} + + for { + n := len(reordered) + + for _, e := range body { + if reordered.Contains(e) { + continue + } + + // Collect vars that are contained in closures within this + // expression. + vs := VarSet{} + WalkClosures(e, func(x interface{}) bool { + vis := &VarVisitor{vars: vs} + vis.Walk(x) + return true + }) + + // Compute vars that are closed over from the body but not yet + // contained in the output position of an expression in the reordered + // body. These vars are considered unsafe. + cv := vs.Intersect(body.Vars(safetyCheckVarVisitorParams)).Diff(globals) + uv := cv.Diff(outputVarsForBody(reordered, builtins, arity, globals)) + + if len(uv) == 0 { + reordered = append(reordered, e) + delete(unsafe, e) + } else { + unsafe.Set(e, uv) + } + } + + if len(reordered) == n { + break + } + } + + return reordered, unsafe +} + +func outputVarsForBody(body Body, builtins map[string]*Builtin, arity func(Ref) int, safe VarSet) VarSet { + o := safe.Copy() + for _, e := range body { + o.Update(outputVarsForExpr(e, builtins, arity, o)) + } + return o.Diff(safe) +} + +func outputVarsForExpr(expr *Expr, builtins map[string]*Builtin, arity func(Ref) int, safe VarSet) VarSet { + + // Negated expressions must be safe. + if expr.Negated { + return VarSet{} + } + + // With modifier inputs must be safe. + for _, with := range expr.With { + unsafe := false + WalkVars(with, func(v Var) bool { + if !safe.Contains(v) { + unsafe = true + return true + } + return false + }) + if unsafe { + return VarSet{} + } + } + + if !expr.IsCall() { + return outputVarsForExprRefs(expr, safe) + } + + terms := expr.Terms.([]*Term) + name := terms[0].String() + + if b := builtins[name]; b != nil { + if b.Name == Equality.Name { + return outputVarsForExprEq(expr, safe) + } + return outputVarsForExprBuiltin(expr, b, safe) + } + + return outputVarsForExprCall(expr, builtins, arity, safe, terms) +} + +func outputVarsForExprBuiltin(expr *Expr, b *Builtin, safe VarSet) VarSet { + + output := outputVarsForExprRefs(expr, safe) + terms := expr.Terms.([]*Term) + + // Check that all input terms are safe. + for i, t := range terms[1:] { + if b.IsTargetPos(i) { + continue + } + vis := NewVarVisitor().WithParams(VarVisitorParams{ + SkipClosures: true, + SkipSets: true, + SkipObjectKeys: true, + SkipRefHead: true, + }) + vis.Walk(t) + unsafe := vis.Vars().Diff(output).Diff(safe) + if len(unsafe) > 0 { + return VarSet{} + } + } + + // Add vars in target positions to result. + for i, t := range terms[1:] { + if b.IsTargetPos(i) { + vis := NewVarVisitor().WithParams(VarVisitorParams{ + SkipRefHead: true, + SkipSets: true, + SkipObjectKeys: true, + SkipClosures: true, + }) + vis.Walk(t) + output.Update(vis.vars) + } + } + + return output +} + +func outputVarsForExprEq(expr *Expr, safe VarSet) VarSet { + if !validEqAssignArgCount(expr) { + return safe + } + output := outputVarsForExprRefs(expr, safe) + output.Update(safe) + output.Update(Unify(output, expr.Operand(0), expr.Operand(1))) + return output.Diff(safe) +} + +func outputVarsForExprCall(expr *Expr, builtins map[string]*Builtin, arity func(Ref) int, safe VarSet, terms []*Term) VarSet { + + output := outputVarsForExprRefs(expr, safe) + + ref, ok := terms[0].Value.(Ref) + if !ok { + return VarSet{} + } + + numArgs := arity(ref) + if numArgs == -1 { + return VarSet{} + } + + numInputTerms := numArgs + 1 + + if numInputTerms >= len(terms) { + return output + } + + vis := NewVarVisitor().WithParams(VarVisitorParams{ + SkipClosures: true, + SkipSets: true, + SkipObjectKeys: true, + SkipRefHead: true, + }) + + vis.Walk(Args(terms[:numInputTerms])) + unsafe := vis.Vars().Diff(output).Diff(safe) + + if len(unsafe) > 0 { + return VarSet{} + } + + vis = NewVarVisitor().WithParams(VarVisitorParams{ + SkipRefHead: true, + SkipSets: true, + SkipObjectKeys: true, + SkipClosures: true, + }) + + vis.Walk(Args(terms[numInputTerms:])) + output.Update(vis.vars) + return output +} + +func outputVarsForExprRefs(expr *Expr, safe VarSet) VarSet { + output := VarSet{} + WalkRefs(expr, func(r Ref) bool { + if safe.Contains(r[0].Value.(Var)) { + output.Update(r.OutputVars()) + return false + } + return true + }) + return output +} + +type equalityFactory struct { + gen *localVarGenerator +} + +func newEqualityFactory(gen *localVarGenerator) *equalityFactory { + return &equalityFactory{gen} +} + +func (f *equalityFactory) Generate(other *Term) *Expr { + term := NewTerm(f.gen.Generate()).SetLocation(other.Location) + expr := Equality.Expr(term, other) + expr.Generated = true + expr.Location = other.Location + return expr +} + +type localVarGenerator struct { + exclude VarSet + suffix string + next int +} + +func newLocalVarGeneratorForModuleSet(sorted []string, modules map[string]*Module) *localVarGenerator { + exclude := NewVarSet() + vis := &VarVisitor{vars: exclude} + for _, key := range sorted { + vis.Walk(modules[key]) + } + return &localVarGenerator{exclude: exclude, next: 0} +} + +func newLocalVarGenerator(suffix string, node interface{}) *localVarGenerator { + exclude := NewVarSet() + vis := &VarVisitor{vars: exclude} + vis.Walk(node) + return &localVarGenerator{exclude: exclude, suffix: suffix, next: 0} +} + +func (l *localVarGenerator) Generate() Var { + for { + result := Var("__local" + l.suffix + strconv.Itoa(l.next) + "__") + l.next++ + if !l.exclude.Contains(result) { + return result + } + } +} + +func getGlobals(pkg *Package, rules []Var, imports []*Import) map[Var]Ref { + + globals := map[Var]Ref{} + + // Populate globals with exports within the package. + for _, v := range rules { + global := append(Ref{}, pkg.Path...) + global = append(global, &Term{Value: String(v)}) + globals[v] = global + } + + // Populate globals with imports. + for _, i := range imports { + if len(i.Alias) > 0 { + path := i.Path.Value.(Ref) + globals[i.Alias] = path + } else { + path := i.Path.Value.(Ref) + if len(path) == 1 { + globals[path[0].Value.(Var)] = path + } else { + v := path[len(path)-1].Value.(String) + globals[Var(v)] = path + } + } + } + + return globals +} + +func requiresEval(x *Term) bool { + if x == nil { + return false + } + return ContainsRefs(x) || ContainsComprehensions(x) +} + +func resolveRef(globals map[Var]Ref, ignore *declaredVarStack, ref Ref) Ref { + + r := Ref{} + for i, x := range ref { + switch v := x.Value.(type) { + case Var: + if g, ok := globals[v]; ok && !ignore.Contains(v) { + cpy := g.Copy() + for i := range cpy { + cpy[i].SetLocation(x.Location) + } + if i == 0 { + r = cpy + } else { + r = append(r, NewTerm(cpy).SetLocation(x.Location)) + } + } else { + r = append(r, x) + } + case Ref, Array, Object, Set, *ArrayComprehension, *SetComprehension, *ObjectComprehension, Call: + r = append(r, resolveRefsInTerm(globals, ignore, x)) + default: + r = append(r, x) + } + } + + return r +} + +func resolveRefsInRule(globals map[Var]Ref, rule *Rule) error { + ignore := &declaredVarStack{} + + vars := NewVarSet() + var vis *GenericVisitor + var err error + + // Walk args to collect vars and transform body so that callers can shadow + // root documents. + vis = NewGenericVisitor(func(x interface{}) bool { + if err != nil { + return true + } + switch x := x.(type) { + case Var: + vars.Add(x) + + // Object keys cannot be pattern matched so only walk values. + case Object: + for _, k := range x.Keys() { + vis.Walk(x.Get(k)) + } + + // Skip terms that could contain vars that cannot be pattern matched. + case Set, *ArrayComprehension, *SetComprehension, *ObjectComprehension, Call: + return true + + case *Term: + if _, ok := x.Value.(Ref); ok { + if RootDocumentRefs.Contains(x) { + // We could support args named input, data, etc. however + // this would require rewriting terms in the head and body. + // Preventing root document shadowing is simpler, and + // arguably, will prevent confusing names from being used. + err = fmt.Errorf("args must not shadow %v (use a different variable name)", x) + return true + } + } + } + return false + }) + + vis.Walk(rule.Head.Args) + + if err != nil { + return err + } + + ignore.Push(vars) + ignore.Push(declaredVars(rule.Body)) + + if rule.Head.Key != nil { + rule.Head.Key = resolveRefsInTerm(globals, ignore, rule.Head.Key) + } + + if rule.Head.Value != nil { + rule.Head.Value = resolveRefsInTerm(globals, ignore, rule.Head.Value) + } + + rule.Body = resolveRefsInBody(globals, ignore, rule.Body) + return nil +} + +func resolveRefsInBody(globals map[Var]Ref, ignore *declaredVarStack, body Body) Body { + r := Body{} + for _, expr := range body { + r = append(r, resolveRefsInExpr(globals, ignore, expr)) + } + return r +} + +func resolveRefsInExpr(globals map[Var]Ref, ignore *declaredVarStack, expr *Expr) *Expr { + cpy := *expr + switch ts := expr.Terms.(type) { + case *Term: + cpy.Terms = resolveRefsInTerm(globals, ignore, ts) + case []*Term: + buf := make([]*Term, len(ts)) + for i := 0; i < len(ts); i++ { + buf[i] = resolveRefsInTerm(globals, ignore, ts[i]) + } + cpy.Terms = buf + } + for _, w := range cpy.With { + w.Target = resolveRefsInTerm(globals, ignore, w.Target) + w.Value = resolveRefsInTerm(globals, ignore, w.Value) + } + return &cpy +} + +func resolveRefsInTerm(globals map[Var]Ref, ignore *declaredVarStack, term *Term) *Term { + switch v := term.Value.(type) { + case Var: + if g, ok := globals[v]; ok && !ignore.Contains(v) { + cpy := g.Copy() + for i := range cpy { + cpy[i].SetLocation(term.Location) + } + return NewTerm(cpy).SetLocation(term.Location) + } + return term + case Ref: + fqn := resolveRef(globals, ignore, v) + cpy := *term + cpy.Value = fqn + return &cpy + case Object: + cpy := *term + cpy.Value, _ = v.Map(func(k, v *Term) (*Term, *Term, error) { + k = resolveRefsInTerm(globals, ignore, k) + v = resolveRefsInTerm(globals, ignore, v) + return k, v, nil + }) + return &cpy + case Array: + cpy := *term + cpy.Value = Array(resolveRefsInTermSlice(globals, ignore, v)) + return &cpy + case Call: + cpy := *term + cpy.Value = Call(resolveRefsInTermSlice(globals, ignore, v)) + return &cpy + case Set: + s, _ := v.Map(func(e *Term) (*Term, error) { + return resolveRefsInTerm(globals, ignore, e), nil + }) + cpy := *term + cpy.Value = s + return &cpy + case *ArrayComprehension: + ac := &ArrayComprehension{} + ignore.Push(declaredVars(v.Body)) + ac.Term = resolveRefsInTerm(globals, ignore, v.Term) + ac.Body = resolveRefsInBody(globals, ignore, v.Body) + cpy := *term + cpy.Value = ac + ignore.Pop() + return &cpy + case *ObjectComprehension: + oc := &ObjectComprehension{} + ignore.Push(declaredVars(v.Body)) + oc.Key = resolveRefsInTerm(globals, ignore, v.Key) + oc.Value = resolveRefsInTerm(globals, ignore, v.Value) + oc.Body = resolveRefsInBody(globals, ignore, v.Body) + cpy := *term + cpy.Value = oc + ignore.Pop() + return &cpy + case *SetComprehension: + sc := &SetComprehension{} + ignore.Push(declaredVars(v.Body)) + sc.Term = resolveRefsInTerm(globals, ignore, v.Term) + sc.Body = resolveRefsInBody(globals, ignore, v.Body) + cpy := *term + cpy.Value = sc + ignore.Pop() + return &cpy + default: + return term + } +} + +func resolveRefsInTermSlice(globals map[Var]Ref, ignore *declaredVarStack, terms []*Term) []*Term { + cpy := make([]*Term, len(terms)) + for i := 0; i < len(terms); i++ { + cpy[i] = resolveRefsInTerm(globals, ignore, terms[i]) + } + return cpy +} + +type declaredVarStack []VarSet + +func (s declaredVarStack) Contains(v Var) bool { + for i := len(s) - 1; i >= 0; i-- { + if _, ok := s[i][v]; ok { + return ok + } + } + return false +} + +func (s declaredVarStack) Add(v Var) { + s[len(s)-1].Add(v) +} + +func (s *declaredVarStack) Push(vs VarSet) { + *s = append(*s, vs) +} + +func (s *declaredVarStack) Pop() { + curr := *s + *s = curr[:len(curr)-1] +} + +func declaredVars(x interface{}) VarSet { + vars := NewVarSet() + vis := NewGenericVisitor(func(x interface{}) bool { + switch x := x.(type) { + case *Expr: + if x.IsAssignment() && validEqAssignArgCount(x) { + WalkVars(x.Operand(0), func(v Var) bool { + vars.Add(v) + return false + }) + } else if decl, ok := x.Terms.(*SomeDecl); ok { + for i := range decl.Symbols { + vars.Add(decl.Symbols[i].Value.(Var)) + } + } + case *ArrayComprehension, *SetComprehension, *ObjectComprehension: + return true + } + return false + }) + vis.Walk(x) + return vars +} + +// rewriteComprehensionTerms will rewrite comprehensions so that the term part +// is bound to a variable in the body. This allows any type of term to be used +// in the term part (even if the term requires evaluation.) +// +// For instance, given the following comprehension: +// +// [x[0] | x = y[_]; y = [1,2,3]] +// +// The comprehension would be rewritten as: +// +// [__local0__ | x = y[_]; y = [1,2,3]; __local0__ = x[0]] +func rewriteComprehensionTerms(f *equalityFactory, node interface{}) (interface{}, error) { + return TransformComprehensions(node, func(x interface{}) (Value, error) { + switch x := x.(type) { + case *ArrayComprehension: + if requiresEval(x.Term) { + expr := f.Generate(x.Term) + x.Term = expr.Operand(0) + x.Body.Append(expr) + } + return x, nil + case *SetComprehension: + if requiresEval(x.Term) { + expr := f.Generate(x.Term) + x.Term = expr.Operand(0) + x.Body.Append(expr) + } + return x, nil + case *ObjectComprehension: + if requiresEval(x.Key) { + expr := f.Generate(x.Key) + x.Key = expr.Operand(0) + x.Body.Append(expr) + } + if requiresEval(x.Value) { + expr := f.Generate(x.Value) + x.Value = expr.Operand(0) + x.Body.Append(expr) + } + return x, nil + } + panic("illegal type") + }) +} + +// rewriteEquals will rewrite exprs under x as unification calls instead of == +// calls. For example: +// +// data.foo == data.bar is rewritten as data.foo = data.bar +// +// This stage should only run the safety check (since == is a built-in with no +// outputs, so the inputs must not be marked as safe.) +// +// This stage is not executed by the query compiler by default because when +// callers specify == instead of = they expect to receive a true/false/undefined +// result back whereas with = the result is only ever true/undefined. For +// partial evaluation cases we do want to rewrite == to = to simplify the +// result. +func rewriteEquals(x interface{}) { + doubleEq := Equal.Ref() + unifyOp := Equality.Ref() + WalkExprs(x, func(x *Expr) bool { + if x.IsCall() { + operator := x.Operator() + if operator.Equal(doubleEq) && len(x.Operands()) == 2 { + x.SetOperator(NewTerm(unifyOp)) + } + } + return false + }) +} + +// rewriteDynamics will rewrite the body so that dynamic terms (i.e., refs and +// comprehensions) are bound to vars earlier in the query. This translation +// results in eager evaluation. +// +// For instance, given the following query: +// +// foo(data.bar) = 1 +// +// The rewritten version will be: +// +// __local0__ = data.bar; foo(__local0__) = 1 +func rewriteDynamics(f *equalityFactory, body Body) Body { + result := make(Body, 0, len(body)) + for _, expr := range body { + if expr.IsEquality() { + result = rewriteDynamicsEqExpr(f, expr, result) + } else if expr.IsCall() { + result = rewriteDynamicsCallExpr(f, expr, result) + } else { + result = rewriteDynamicsTermExpr(f, expr, result) + } + } + return result +} + +func appendExpr(body Body, expr *Expr) Body { + body.Append(expr) + return body +} + +func rewriteDynamicsEqExpr(f *equalityFactory, expr *Expr, result Body) Body { + if !validEqAssignArgCount(expr) { + return appendExpr(result, expr) + } + terms := expr.Terms.([]*Term) + result, terms[1] = rewriteDynamicsInTerm(expr, f, terms[1], result) + result, terms[2] = rewriteDynamicsInTerm(expr, f, terms[2], result) + return appendExpr(result, expr) +} + +func rewriteDynamicsCallExpr(f *equalityFactory, expr *Expr, result Body) Body { + terms := expr.Terms.([]*Term) + for i := 1; i < len(terms); i++ { + result, terms[i] = rewriteDynamicsOne(expr, f, terms[i], result) + } + return appendExpr(result, expr) +} + +func rewriteDynamicsTermExpr(f *equalityFactory, expr *Expr, result Body) Body { + term := expr.Terms.(*Term) + result, expr.Terms = rewriteDynamicsInTerm(expr, f, term, result) + return appendExpr(result, expr) +} + +func rewriteDynamicsInTerm(original *Expr, f *equalityFactory, term *Term, result Body) (Body, *Term) { + switch v := term.Value.(type) { + case Ref: + for i := 1; i < len(v); i++ { + result, v[i] = rewriteDynamicsOne(original, f, v[i], result) + } + case *ArrayComprehension: + v.Body = rewriteDynamics(f, v.Body) + case *SetComprehension: + v.Body = rewriteDynamics(f, v.Body) + case *ObjectComprehension: + v.Body = rewriteDynamics(f, v.Body) + default: + result, term = rewriteDynamicsOne(original, f, term, result) + } + return result, term +} + +func rewriteDynamicsOne(original *Expr, f *equalityFactory, term *Term, result Body) (Body, *Term) { + switch v := term.Value.(type) { + case Ref: + for i := 1; i < len(v); i++ { + result, v[i] = rewriteDynamicsOne(original, f, v[i], result) + } + generated := f.Generate(term) + generated.With = original.With + result.Append(generated) + return result, result[len(result)-1].Operand(0) + case Array: + for i := 0; i < len(v); i++ { + result, v[i] = rewriteDynamicsOne(original, f, v[i], result) + } + return result, term + case Object: + cpy := NewObject() + for _, key := range v.Keys() { + value := v.Get(key) + result, key = rewriteDynamicsOne(original, f, key, result) + result, value = rewriteDynamicsOne(original, f, value, result) + cpy.Insert(key, value) + } + return result, NewTerm(cpy).SetLocation(term.Location) + case Set: + cpy := NewSet() + for _, term := range v.Slice() { + var rw *Term + result, rw = rewriteDynamicsOne(original, f, term, result) + cpy.Add(rw) + } + return result, NewTerm(cpy).SetLocation(term.Location) + case *ArrayComprehension: + var extra *Expr + v.Body, extra = rewriteDynamicsComprehensionBody(original, f, v.Body, term) + result.Append(extra) + return result, result[len(result)-1].Operand(0) + case *SetComprehension: + var extra *Expr + v.Body, extra = rewriteDynamicsComprehensionBody(original, f, v.Body, term) + result.Append(extra) + return result, result[len(result)-1].Operand(0) + case *ObjectComprehension: + var extra *Expr + v.Body, extra = rewriteDynamicsComprehensionBody(original, f, v.Body, term) + result.Append(extra) + return result, result[len(result)-1].Operand(0) + } + return result, term +} + +func rewriteDynamicsComprehensionBody(original *Expr, f *equalityFactory, body Body, term *Term) (Body, *Expr) { + body = rewriteDynamics(f, body) + generated := f.Generate(term) + generated.With = original.With + return body, generated +} + +func rewriteExprTermsInHead(gen *localVarGenerator, rule *Rule) { + if rule.Head.Key != nil { + support, output := expandExprTerm(gen, rule.Head.Key) + for i := range support { + rule.Body.Append(support[i]) + } + rule.Head.Key = output + } + if rule.Head.Value != nil { + support, output := expandExprTerm(gen, rule.Head.Value) + for i := range support { + rule.Body.Append(support[i]) + } + rule.Head.Value = output + } +} + +func rewriteExprTermsInBody(gen *localVarGenerator, body Body) Body { + cpy := make(Body, 0, len(body)) + for i := 0; i < len(body); i++ { + for _, expr := range expandExpr(gen, body[i]) { + cpy.Append(expr) + } + } + return cpy +} + +func expandExpr(gen *localVarGenerator, expr *Expr) (result []*Expr) { + for i := range expr.With { + extras, value := expandExprTerm(gen, expr.With[i].Value) + expr.With[i].Value = value + result = append(result, extras...) + } + switch terms := expr.Terms.(type) { + case *Term: + extras, term := expandExprTerm(gen, terms) + if len(expr.With) > 0 { + for i := range extras { + extras[i].With = expr.With + } + } + result = append(result, extras...) + expr.Terms = term + result = append(result, expr) + case []*Term: + for i := 1; i < len(terms); i++ { + var extras []*Expr + extras, terms[i] = expandExprTerm(gen, terms[i]) + if len(expr.With) > 0 { + for i := range extras { + extras[i].With = expr.With + } + } + result = append(result, extras...) + } + result = append(result, expr) + } + return +} + +func expandExprTerm(gen *localVarGenerator, term *Term) (support []*Expr, output *Term) { + output = term + switch v := term.Value.(type) { + case Call: + for i := 1; i < len(v); i++ { + var extras []*Expr + extras, v[i] = expandExprTerm(gen, v[i]) + support = append(support, extras...) + } + output = NewTerm(gen.Generate()).SetLocation(term.Location) + expr := v.MakeExpr(output).SetLocation(term.Location) + expr.Generated = true + support = append(support, expr) + case Ref: + support = expandExprRef(gen, v) + case Array: + support = expandExprTermSlice(gen, v) + case Object: + cpy, _ := v.Map(func(k, v *Term) (*Term, *Term, error) { + extras1, expandedKey := expandExprTerm(gen, k) + extras2, expandedValue := expandExprTerm(gen, v) + support = append(support, extras1...) + support = append(support, extras2...) + return expandedKey, expandedValue, nil + }) + output = NewTerm(cpy).SetLocation(term.Location) + case Set: + cpy, _ := v.Map(func(x *Term) (*Term, error) { + extras, expanded := expandExprTerm(gen, x) + support = append(support, extras...) + return expanded, nil + }) + output = NewTerm(cpy).SetLocation(term.Location) + case *ArrayComprehension: + support, term := expandExprTerm(gen, v.Term) + for i := range support { + v.Body.Append(support[i]) + } + v.Term = term + v.Body = rewriteExprTermsInBody(gen, v.Body) + case *SetComprehension: + support, term := expandExprTerm(gen, v.Term) + for i := range support { + v.Body.Append(support[i]) + } + v.Term = term + v.Body = rewriteExprTermsInBody(gen, v.Body) + case *ObjectComprehension: + support, key := expandExprTerm(gen, v.Key) + for i := range support { + v.Body.Append(support[i]) + } + v.Key = key + support, value := expandExprTerm(gen, v.Value) + for i := range support { + v.Body.Append(support[i]) + } + v.Value = value + v.Body = rewriteExprTermsInBody(gen, v.Body) + } + return +} + +func expandExprRef(gen *localVarGenerator, v []*Term) (support []*Expr) { + // Start by calling a normal expandExprTerm on all terms. + support = expandExprTermSlice(gen, v) + + // Rewrite references in order to support indirect references. We rewrite + // e.g. + // + // [1, 2, 3][i] + // + // to + // + // __local_var = [1, 2, 3] + // __local_var[i] + // + // to support these. This only impacts the reference subject, i.e. the + // first item in the slice. + var subject = v[0] + switch subject.Value.(type) { + case Array, Object, Set, *ArrayComprehension, *SetComprehension, *ObjectComprehension, Call: + f := newEqualityFactory(gen) + assignToLocal := f.Generate(subject) + support = append(support, assignToLocal) + v[0] = assignToLocal.Operand(0) + } + return +} + +func expandExprTermSlice(gen *localVarGenerator, v []*Term) (support []*Expr) { + for i := 0; i < len(v); i++ { + var extras []*Expr + extras, v[i] = expandExprTerm(gen, v[i]) + support = append(support, extras...) + } + return +} + +type localDeclaredVars struct { + vars []*declaredVarSet + + // rewritten contains a mapping of *all* user-defined variables + // that have been rewritten whereas vars contains the state + // from the current query (not not any nested queries, and all + // vars seen). + rewritten map[Var]Var +} + +type varOccurrence int + +const ( + newVar varOccurrence = iota + argVar + seenVar + assignedVar + declaredVar +) + +type declaredVarSet struct { + vs map[Var]Var + reverse map[Var]Var + occurrence map[Var]varOccurrence +} + +func newDeclaredVarSet() *declaredVarSet { + return &declaredVarSet{ + vs: map[Var]Var{}, + reverse: map[Var]Var{}, + occurrence: map[Var]varOccurrence{}, + } +} + +func newLocalDeclaredVars() *localDeclaredVars { + return &localDeclaredVars{ + vars: []*declaredVarSet{newDeclaredVarSet()}, + rewritten: map[Var]Var{}, + } +} + +func (s *localDeclaredVars) Push() { + s.vars = append(s.vars, newDeclaredVarSet()) +} + +func (s *localDeclaredVars) Pop() *declaredVarSet { + sl := s.vars + curr := sl[len(sl)-1] + s.vars = sl[:len(sl)-1] + return curr +} + +func (s localDeclaredVars) Peek() *declaredVarSet { + return s.vars[len(s.vars)-1] +} + +func (s localDeclaredVars) Insert(x, y Var, occurrence varOccurrence) { + elem := s.vars[len(s.vars)-1] + elem.vs[x] = y + elem.reverse[y] = x + elem.occurrence[x] = occurrence + + // If the variable has been rewritten (where x != y, with y being + // the generated value), store it in the map of rewritten vars. + // Assume that the generated values are unique for the compilation. + if !x.Equal(y) { + s.rewritten[y] = x + } +} + +func (s localDeclaredVars) Declared(x Var) (y Var, ok bool) { + for i := len(s.vars) - 1; i >= 0; i-- { + if y, ok = s.vars[i].vs[x]; ok { + return + } + } + return +} + +// Occurrence returns a flag that indicates whether x has occurred in the +// current scope. +func (s localDeclaredVars) Occurrence(x Var) varOccurrence { + return s.vars[len(s.vars)-1].occurrence[x] +} + +// rewriteLocalVars rewrites bodies to remove assignment/declaration +// expressions. For example: +// +// a := 1; p[a] +// +// Is rewritten to: +// +// __local0__ = 1; p[__local0__] +// +// During rewriting, assignees are validated to prevent use before declaration. +func rewriteLocalVars(g *localVarGenerator, stack *localDeclaredVars, used VarSet, body Body) (Body, map[Var]Var, Errors) { + var errs Errors + body, errs = rewriteDeclaredVarsInBody(g, stack, used, body, errs) + return body, stack.Pop().vs, errs +} + +func rewriteDeclaredVarsInBody(g *localVarGenerator, stack *localDeclaredVars, used VarSet, body Body, errs Errors) (Body, Errors) { + + var cpy Body + + for i := range body { + var expr *Expr + if body[i].IsAssignment() { + expr, errs = rewriteDeclaredAssignment(g, stack, body[i], errs) + } else if decl, ok := body[i].Terms.(*SomeDecl); ok { + errs = rewriteSomeDeclStatement(g, stack, decl, errs) + } else { + expr, errs = rewriteDeclaredVarsInExpr(g, stack, body[i], errs) + } + if expr != nil { + cpy.Append(expr) + } + } + + // If the body only contained a var statement it will be empty at this + // point. Append true to the body to ensure that it's non-empty (zero length + // bodies are not supported.) + if len(cpy) == 0 { + cpy.Append(NewExpr(BooleanTerm(true))) + } + + return cpy, checkUnusedDeclaredVars(body[0].Loc(), stack, used, cpy, errs) +} + +func checkUnusedDeclaredVars(loc *Location, stack *localDeclaredVars, used VarSet, cpy Body, errs Errors) Errors { + + // NOTE(tsandall): Do not generate more errors if there are existing + // declaration errors. + if len(errs) > 0 { + return errs + } + + dvs := stack.Peek() + declared := NewVarSet() + + for v, occ := range dvs.occurrence { + if occ == declaredVar { + declared.Add(dvs.vs[v]) + } + } + + bodyvars := cpy.Vars(VarVisitorParams{}) + + for v := range used { + if gv, ok := stack.Declared(v); ok { + bodyvars.Add(gv) + } else { + bodyvars.Add(v) + } + } + + unused := declared.Diff(bodyvars).Diff(used) + + for _, gv := range unused.Sorted() { + errs = append(errs, NewError(CompileErr, loc, "declared var %v unused", dvs.reverse[gv])) + } + + return errs +} + +func rewriteSomeDeclStatement(g *localVarGenerator, stack *localDeclaredVars, decl *SomeDecl, errs Errors) Errors { + for i := range decl.Symbols { + v := decl.Symbols[i].Value.(Var) + if _, err := rewriteDeclaredVar(g, stack, v, declaredVar); err != nil { + errs = append(errs, NewError(CompileErr, decl.Loc(), err.Error())) + } + } + return errs +} + +func rewriteDeclaredVarsInExpr(g *localVarGenerator, stack *localDeclaredVars, expr *Expr, errs Errors) (*Expr, Errors) { + vis := NewGenericVisitor(func(x interface{}) bool { + var stop bool + switch x := x.(type) { + case *Term: + stop, errs = rewriteDeclaredVarsInTerm(g, stack, x, errs) + case *With: + _, errs = rewriteDeclaredVarsInTerm(g, stack, x.Value, errs) + stop = true + } + return stop + }) + vis.Walk(expr) + return expr, errs +} + +func rewriteDeclaredAssignment(g *localVarGenerator, stack *localDeclaredVars, expr *Expr, errs Errors) (*Expr, Errors) { + + if expr.Negated { + errs = append(errs, NewError(CompileErr, expr.Location, "cannot assign vars inside negated expression")) + return expr, errs + } + + numErrsBefore := len(errs) + + if !validEqAssignArgCount(expr) { + return expr, errs + } + + // Rewrite terms on right hand side capture seen vars and recursively + // process comprehensions before left hand side is processed. Also + // rewrite with modifier. + errs = rewriteDeclaredVarsInTermRecursive(g, stack, expr.Operand(1), errs) + + for _, w := range expr.With { + errs = rewriteDeclaredVarsInTermRecursive(g, stack, w.Value, errs) + } + + // Rewrite vars on left hand side with unique names. Catch redeclaration + // and invalid term types here. + var vis func(t *Term) bool + + vis = func(t *Term) bool { + switch v := t.Value.(type) { + case Var: + if gv, err := rewriteDeclaredVar(g, stack, v, assignedVar); err != nil { + errs = append(errs, NewError(CompileErr, t.Location, err.Error())) + } else { + t.Value = gv + } + return true + case Array: + return false + case Object: + v.Foreach(func(_, v *Term) { + WalkTerms(v, vis) + }) + return true + case Ref: + if RootDocumentRefs.Contains(t) { + if gv, err := rewriteDeclaredVar(g, stack, v[0].Value.(Var), assignedVar); err != nil { + errs = append(errs, NewError(CompileErr, t.Location, err.Error())) + } else { + t.Value = gv + } + return true + } + } + errs = append(errs, NewError(CompileErr, t.Location, "cannot assign to %v", TypeName(t.Value))) + return true + } + + WalkTerms(expr.Operand(0), vis) + + if len(errs) == numErrsBefore { + loc := expr.Operator()[0].Location + expr.SetOperator(RefTerm(VarTerm(Equality.Name).SetLocation(loc)).SetLocation(loc)) + } + + return expr, errs +} + +func rewriteDeclaredVarsInTerm(g *localVarGenerator, stack *localDeclaredVars, term *Term, errs Errors) (bool, Errors) { + switch v := term.Value.(type) { + case Var: + if gv, ok := stack.Declared(v); ok { + term.Value = gv + } else if stack.Occurrence(v) == newVar { + stack.Insert(v, v, seenVar) + } + case Ref: + if RootDocumentRefs.Contains(term) { + if gv, ok := stack.Declared(v[0].Value.(Var)); ok { + term.Value = gv + } + return true, errs + } + return false, errs + case Object: + cpy, _ := v.Map(func(k, v *Term) (*Term, *Term, error) { + kcpy := k.Copy() + errs = rewriteDeclaredVarsInTermRecursive(g, stack, kcpy, errs) + errs = rewriteDeclaredVarsInTermRecursive(g, stack, v, errs) + return kcpy, v, nil + }) + term.Value = cpy + case Set: + cpy, _ := v.Map(func(elem *Term) (*Term, error) { + elemcpy := elem.Copy() + errs = rewriteDeclaredVarsInTermRecursive(g, stack, elemcpy, errs) + return elemcpy, nil + }) + term.Value = cpy + case *ArrayComprehension: + errs = rewriteDeclaredVarsInArrayComprehension(g, stack, v, errs) + case *SetComprehension: + errs = rewriteDeclaredVarsInSetComprehension(g, stack, v, errs) + case *ObjectComprehension: + errs = rewriteDeclaredVarsInObjectComprehension(g, stack, v, errs) + default: + return false, errs + } + return true, errs +} + +func rewriteDeclaredVarsInTermRecursive(g *localVarGenerator, stack *localDeclaredVars, term *Term, errs Errors) Errors { + WalkNodes(term, func(n Node) bool { + var stop bool + switch n := n.(type) { + case *With: + _, errs = rewriteDeclaredVarsInTerm(g, stack, n.Value, errs) + stop = true + case *Term: + stop, errs = rewriteDeclaredVarsInTerm(g, stack, n, errs) + } + return stop + }) + return errs +} + +func rewriteDeclaredVarsInArrayComprehension(g *localVarGenerator, stack *localDeclaredVars, v *ArrayComprehension, errs Errors) Errors { + stack.Push() + v.Body, errs = rewriteDeclaredVarsInBody(g, stack, nil, v.Body, errs) + errs = rewriteDeclaredVarsInTermRecursive(g, stack, v.Term, errs) + stack.Pop() + return errs +} + +func rewriteDeclaredVarsInSetComprehension(g *localVarGenerator, stack *localDeclaredVars, v *SetComprehension, errs Errors) Errors { + stack.Push() + v.Body, errs = rewriteDeclaredVarsInBody(g, stack, nil, v.Body, errs) + errs = rewriteDeclaredVarsInTermRecursive(g, stack, v.Term, errs) + stack.Pop() + return errs +} + +func rewriteDeclaredVarsInObjectComprehension(g *localVarGenerator, stack *localDeclaredVars, v *ObjectComprehension, errs Errors) Errors { + stack.Push() + v.Body, errs = rewriteDeclaredVarsInBody(g, stack, nil, v.Body, errs) + errs = rewriteDeclaredVarsInTermRecursive(g, stack, v.Key, errs) + errs = rewriteDeclaredVarsInTermRecursive(g, stack, v.Value, errs) + stack.Pop() + return errs +} + +func rewriteDeclaredVar(g *localVarGenerator, stack *localDeclaredVars, v Var, occ varOccurrence) (gv Var, err error) { + switch stack.Occurrence(v) { + case seenVar: + return gv, fmt.Errorf("var %v referenced above", v) + case assignedVar: + return gv, fmt.Errorf("var %v assigned above", v) + case declaredVar: + return gv, fmt.Errorf("var %v declared above", v) + case argVar: + return gv, fmt.Errorf("arg %v redeclared", v) + } + gv = g.Generate() + stack.Insert(v, gv, occ) + return +} + +// rewriteWithModifiersInBody will rewrite the body so that with modifiers do +// not contain terms that require evaluation as values. If this function +// encounters an invalid with modifier target then it will raise an error. +func rewriteWithModifiersInBody(c *Compiler, f *equalityFactory, body Body) (Body, *Error) { + var result Body + for i := range body { + exprs, err := rewriteWithModifier(c, f, body[i]) + if err != nil { + return nil, err + } + if len(exprs) > 0 { + for _, expr := range exprs { + result.Append(expr) + } + } else { + result.Append(body[i]) + } + } + return result, nil +} + +func rewriteWithModifier(c *Compiler, f *equalityFactory, expr *Expr) ([]*Expr, *Error) { + + var result []*Expr + for i := range expr.With { + err := validateTarget(c, expr.With[i].Target) + if err != nil { + return nil, err + } + + if requiresEval(expr.With[i].Value) { + eq := f.Generate(expr.With[i].Value) + result = append(result, eq) + expr.With[i].Value = eq.Operand(0) + } + } + + // If any of the with modifiers in this expression were rewritten then result + // will be non-empty. In this case, the expression will have been modified and + // it should also be added to the result. + if len(result) > 0 { + result = append(result, expr) + } + return result, nil +} + +func validateTarget(c *Compiler, term *Term) *Error { + if !isInputRef(term) && !isDataRef(term) { + return NewError(TypeErr, term.Location, "with keyword target must start with %v or %v", InputRootDocument, DefaultRootDocument) + } + + if isDataRef(term) { + ref := term.Value.(Ref) + node := c.RuleTree + for i := 0; i < len(ref)-1; i++ { + child := node.Child(ref[i].Value) + if child == nil { + break + } else if len(child.Values) > 0 { + return NewError(CompileErr, term.Loc(), "with keyword cannot partially replace virtual document(s)") + } + node = child + } + + if node != nil { + if child := node.Child(ref[len(ref)-1].Value); child != nil { + for _, value := range child.Values { + if len(value.(*Rule).Head.Args) > 0 { + return NewError(CompileErr, term.Loc(), "with keyword cannot replace functions") + } + } + } + } + + } + return nil +} + +func isInputRef(term *Term) bool { + if ref, ok := term.Value.(Ref); ok { + if ref.HasPrefix(InputRootRef) { + return true + } + } + return false +} + +func isDataRef(term *Term) bool { + if ref, ok := term.Value.(Ref); ok { + if ref.HasPrefix(DefaultRootRef) { + return true + } + } + return false +} + +func isVirtual(node *TreeNode, ref Ref) bool { + for i := 0; i < len(ref); i++ { + child := node.Child(ref[i].Value) + if child == nil { + return false + } else if len(child.Values) > 0 { + return true + } + node = child + } + return true +} + +func safetyErrorSlice(unsafe unsafeVars) (result Errors) { + + if len(unsafe) == 0 { + return + } + + for _, pair := range unsafe.Vars() { + if !pair.Var.IsGenerated() { + result = append(result, NewError(UnsafeVarErr, pair.Loc, "var %v is unsafe", pair.Var)) + } + } + + if len(result) > 0 { + return + } + + // If the expression contains unsafe generated variables, report which + // expressions are unsafe instead of the variables that are unsafe (since + // the latter are not meaningful to the user.) + pairs := unsafe.Slice() + + sort.Slice(pairs, func(i, j int) bool { + return pairs[i].Expr.Location.Compare(pairs[j].Expr.Location) < 0 + }) + + // Report at most one error per generated variable. + seen := NewVarSet() + + for _, expr := range pairs { + before := len(seen) + for v := range expr.Vars { + if v.IsGenerated() { + seen.Add(v) + } + } + if len(seen) > before { + result = append(result, NewError(UnsafeVarErr, expr.Expr.Location, "expression is unsafe")) + } + } + + return +} + +func checkUnsafeBuiltins(unsafeBuiltinsMap map[string]struct{}, node interface{}) Errors { + errs := make(Errors, 0) + WalkExprs(node, func(x *Expr) bool { + if x.IsCall() { + operator := x.Operator().String() + if _, ok := unsafeBuiltinsMap[operator]; ok { + errs = append(errs, NewError(TypeErr, x.Loc(), "unsafe built-in function calls in expression: %v", operator)) + } + } + return false + }) + return errs +} + +func rewriteVarsInRef(vars ...map[Var]Var) func(Ref) Ref { + return func(node Ref) Ref { + i, _ := TransformVars(node, func(v Var) (Value, error) { + for _, m := range vars { + if u, ok := m[v]; ok { + return u, nil + } + } + return v, nil + }) + return i.(Ref) + } +} + +func rewriteVarsNop(node Ref) Ref { + return node +} diff --git a/vendor/github.com/open-policy-agent/opa/ast/compilehelper.go b/vendor/github.com/open-policy-agent/opa/ast/compilehelper.go new file mode 100644 index 000000000..37a81ddc9 --- /dev/null +++ b/vendor/github.com/open-policy-agent/opa/ast/compilehelper.go @@ -0,0 +1,42 @@ +// Copyright 2016 The OPA Authors. All rights reserved. +// Use of this source code is governed by an Apache2 +// license that can be found in the LICENSE file. + +package ast + +// CompileModules takes a set of Rego modules represented as strings and +// compiles them for evaluation. The keys of the map are used as filenames. +func CompileModules(modules map[string]string) (*Compiler, error) { + + parsed := make(map[string]*Module, len(modules)) + + for f, module := range modules { + var pm *Module + var err error + if pm, err = ParseModule(f, module); err != nil { + return nil, err + } + parsed[f] = pm + } + + compiler := NewCompiler() + compiler.Compile(parsed) + + if compiler.Failed() { + return nil, compiler.Errors + } + + return compiler, nil +} + +// MustCompileModules compiles a set of Rego modules represented as strings. If +// the compilation process fails, this function panics. +func MustCompileModules(modules map[string]string) *Compiler { + + compiler, err := CompileModules(modules) + if err != nil { + panic(err) + } + + return compiler +} diff --git a/vendor/github.com/open-policy-agent/opa/ast/conflicts.go b/vendor/github.com/open-policy-agent/opa/ast/conflicts.go new file mode 100644 index 000000000..d1013cced --- /dev/null +++ b/vendor/github.com/open-policy-agent/opa/ast/conflicts.go @@ -0,0 +1,48 @@ +// Copyright 2019 The OPA Authors. All rights reserved. +// Use of this source code is governed by an Apache2 +// license that can be found in the LICENSE file. + +package ast + +import ( + "strings" +) + +// CheckPathConflicts returns a set of errors indicating paths that +// are in conflict with the result of the provided callable. +func CheckPathConflicts(c *Compiler, exists func([]string) (bool, error)) Errors { + var errs Errors + + root := c.RuleTree.Child(DefaultRootDocument.Value) + if root == nil { + return nil + } + + for _, node := range root.Children { + errs = append(errs, checkDocumentConflicts(node, exists, nil)...) + } + + return errs +} + +func checkDocumentConflicts(node *TreeNode, exists func([]string) (bool, error), path []string) Errors { + + path = append(path, string(node.Key.(String))) + + if len(node.Values) > 0 { + s := strings.Join(path, "/") + if ok, err := exists(path); err != nil { + return Errors{NewError(CompileErr, node.Values[0].(*Rule).Loc(), "conflict check for data path %v: %v", s, err.Error())} + } else if ok { + return Errors{NewError(CompileErr, node.Values[0].(*Rule).Loc(), "conflicting rule for data path %v found", s)} + } + } + + var errs Errors + + for _, child := range node.Children { + errs = append(errs, checkDocumentConflicts(child, exists, path)...) + } + + return errs +} diff --git a/vendor/github.com/open-policy-agent/opa/ast/doc.go b/vendor/github.com/open-policy-agent/opa/ast/doc.go new file mode 100644 index 000000000..363660cf9 --- /dev/null +++ b/vendor/github.com/open-policy-agent/opa/ast/doc.go @@ -0,0 +1,36 @@ +// Copyright 2016 The OPA Authors. All rights reserved. +// Use of this source code is governed by an Apache2 +// license that can be found in the LICENSE file. + +// Package ast declares Rego syntax tree types and also includes a parser and compiler for preparing policies for execution in the policy engine. +// +// Rego policies are defined using a relatively small set of types: modules, package and import declarations, rules, expressions, and terms. At their core, policies consist of rules that are defined by one or more expressions over documents available to the policy engine. The expressions are defined by intrinsic values (terms) such as strings, objects, variables, etc. +// +// Rego policies are typically defined in text files and then parsed and compiled by the policy engine at runtime. The parsing stage takes the text or string representation of the policy and converts it into an abstract syntax tree (AST) that consists of the types mentioned above. The AST is organized as follows: +// +// Module +// | +// +--- Package (Reference) +// | +// +--- Imports +// | | +// | +--- Import (Term) +// | +// +--- Rules +// | +// +--- Rule +// | +// +--- Head +// | | +// | +--- Name (Variable) +// | | +// | +--- Key (Term) +// | | +// | +--- Value (Term) +// | +// +--- Body +// | +// +--- Expression (Term | Terms | Variable Declaration) +// +// At query time, the policy engine expects policies to have been compiled. The compilation stage takes one or more modules and compiles them into a format that the policy engine supports. +package ast diff --git a/vendor/github.com/open-policy-agent/opa/ast/env.go b/vendor/github.com/open-policy-agent/opa/ast/env.go new file mode 100644 index 000000000..0519c30be --- /dev/null +++ b/vendor/github.com/open-policy-agent/opa/ast/env.go @@ -0,0 +1,323 @@ +// Copyright 2017 The OPA Authors. All rights reserved. +// Use of this source code is governed by an Apache2 +// license that can be found in the LICENSE file. + +package ast + +import ( + "github.com/open-policy-agent/opa/types" + "github.com/open-policy-agent/opa/util" +) + +// TypeEnv contains type info for static analysis such as type checking. +type TypeEnv struct { + tree *typeTreeNode + next *TypeEnv +} + +// NewTypeEnv returns an empty TypeEnv. +func NewTypeEnv() *TypeEnv { + return &TypeEnv{ + tree: newTypeTree(), + } +} + +// Get returns the type of x. +func (env *TypeEnv) Get(x interface{}) types.Type { + + if term, ok := x.(*Term); ok { + x = term.Value + } + + switch x := x.(type) { + + // Scalars. + case Null: + return types.NewNull() + case Boolean: + return types.NewBoolean() + case Number: + return types.NewNumber() + case String: + return types.NewString() + + // Composites. + case Array: + static := make([]types.Type, len(x)) + for i := range static { + tpe := env.Get(x[i].Value) + static[i] = tpe + } + + var dynamic types.Type + if len(static) == 0 { + dynamic = types.A + } + + return types.NewArray(static, dynamic) + + case Object: + static := []*types.StaticProperty{} + var dynamic *types.DynamicProperty + + x.Foreach(func(k, v *Term) { + if IsConstant(k.Value) { + kjson, err := JSON(k.Value) + if err == nil { + tpe := env.Get(v) + static = append(static, types.NewStaticProperty(kjson, tpe)) + return + } + } + // Can't handle it as a static property, fallback to dynamic + typeK := env.Get(k.Value) + typeV := env.Get(v.Value) + dynamic = types.NewDynamicProperty(typeK, typeV) + }) + + if len(static) == 0 && dynamic == nil { + dynamic = types.NewDynamicProperty(types.A, types.A) + } + + return types.NewObject(static, dynamic) + + case Set: + var tpe types.Type + x.Foreach(func(elem *Term) { + other := env.Get(elem.Value) + tpe = types.Or(tpe, other) + }) + if tpe == nil { + tpe = types.A + } + return types.NewSet(tpe) + + // Comprehensions. + case *ArrayComprehension: + checker := newTypeChecker() + cpy, errs := checker.CheckBody(env, x.Body) + if len(errs) == 0 { + return types.NewArray(nil, cpy.Get(x.Term)) + } + return nil + case *ObjectComprehension: + checker := newTypeChecker() + cpy, errs := checker.CheckBody(env, x.Body) + if len(errs) == 0 { + return types.NewObject(nil, types.NewDynamicProperty(cpy.Get(x.Key), cpy.Get(x.Value))) + } + return nil + case *SetComprehension: + checker := newTypeChecker() + cpy, errs := checker.CheckBody(env, x.Body) + if len(errs) == 0 { + return types.NewSet(cpy.Get(x.Term)) + } + return nil + + // Refs. + case Ref: + return env.getRef(x) + + // Vars. + case Var: + if node := env.tree.Child(x); node != nil { + return node.Value() + } + if env.next != nil { + return env.next.Get(x) + } + return nil + + default: + panic("unreachable") + } +} + +func (env *TypeEnv) getRef(ref Ref) types.Type { + + node := env.tree.Child(ref[0].Value) + if node == nil { + return env.getRefFallback(ref) + } + + return env.getRefRec(node, ref, ref[1:]) +} + +func (env *TypeEnv) getRefFallback(ref Ref) types.Type { + + if env.next != nil { + return env.next.Get(ref) + } + + if RootDocumentNames.Contains(ref[0]) { + return types.A + } + + return nil +} + +func (env *TypeEnv) getRefRec(node *typeTreeNode, ref, tail Ref) types.Type { + if len(tail) == 0 { + return env.getRefRecExtent(node) + } + + if node.Leaf() { + return selectRef(node.Value(), tail) + } + + if !IsConstant(tail[0].Value) { + return selectRef(env.getRefRecExtent(node), tail) + } + + child := node.Child(tail[0].Value) + if child == nil { + return env.getRefFallback(ref) + } + + return env.getRefRec(child, ref, tail[1:]) +} + +func (env *TypeEnv) getRefRecExtent(node *typeTreeNode) types.Type { + + if node.Leaf() { + return node.Value() + } + + children := []*types.StaticProperty{} + + node.Children().Iter(func(k, v util.T) bool { + key := k.(Value) + child := v.(*typeTreeNode) + + tpe := env.getRefRecExtent(child) + // TODO(tsandall): handle non-string keys? + if s, ok := key.(String); ok { + children = append(children, types.NewStaticProperty(string(s), tpe)) + } + return false + }) + + // TODO(tsandall): for now, these objects can have any dynamic properties + // because we don't have schema for base docs. Once schemas are supported + // we can improve this. + return types.NewObject(children, types.NewDynamicProperty(types.S, types.A)) +} + +func (env *TypeEnv) wrap() *TypeEnv { + cpy := *env + cpy.next = env + cpy.tree = newTypeTree() + return &cpy +} + +// typeTreeNode is used to store type information in a tree. +type typeTreeNode struct { + key Value + value types.Type + children *util.HashMap +} + +func newTypeTree() *typeTreeNode { + return &typeTreeNode{ + key: nil, + value: nil, + children: util.NewHashMap(valueEq, valueHash), + } +} + +func (n *typeTreeNode) Child(key Value) *typeTreeNode { + value, ok := n.children.Get(key) + if !ok { + return nil + } + return value.(*typeTreeNode) +} + +func (n *typeTreeNode) Children() *util.HashMap { + return n.children +} + +func (n *typeTreeNode) Get(path Ref) types.Type { + curr := n + for _, term := range path { + child, ok := curr.children.Get(term.Value) + if !ok { + return nil + } + curr = child.(*typeTreeNode) + } + return curr.Value() +} + +func (n *typeTreeNode) Leaf() bool { + return n.value != nil +} + +func (n *typeTreeNode) PutOne(key Value, tpe types.Type) { + c, ok := n.children.Get(key) + + var child *typeTreeNode + if !ok { + child = newTypeTree() + child.key = key + n.children.Put(key, child) + } else { + child = c.(*typeTreeNode) + } + + child.value = tpe +} + +func (n *typeTreeNode) Put(path Ref, tpe types.Type) { + curr := n + for _, term := range path { + c, ok := curr.children.Get(term.Value) + + var child *typeTreeNode + if !ok { + child = newTypeTree() + child.key = term.Value + curr.children.Put(child.key, child) + } else { + child = c.(*typeTreeNode) + } + + curr = child + } + curr.value = tpe +} + +func (n *typeTreeNode) Value() types.Type { + return n.value +} + +// selectConstant returns the attribute of the type referred to by the term. If +// the attribute type cannot be determined, nil is returned. +func selectConstant(tpe types.Type, term *Term) types.Type { + x, err := JSON(term.Value) + if err == nil { + return types.Select(tpe, x) + } + return nil +} + +// selectRef returns the type of the nested attribute referred to by ref. If +// the attribute type cannot be determined, nil is returned. If the ref +// contains vars or refs, then the returned type will be a union of the +// possible types. +func selectRef(tpe types.Type, ref Ref) types.Type { + + if tpe == nil || len(ref) == 0 { + return tpe + } + + head, tail := ref[0], ref[1:] + + switch head.Value.(type) { + case Var, Ref, Array, Object, Set: + return selectRef(types.Values(tpe), tail) + default: + return selectRef(selectConstant(tpe, head), tail) + } +} diff --git a/vendor/github.com/open-policy-agent/opa/ast/errors.go b/vendor/github.com/open-policy-agent/opa/ast/errors.go new file mode 100644 index 000000000..76a084214 --- /dev/null +++ b/vendor/github.com/open-policy-agent/opa/ast/errors.go @@ -0,0 +1,133 @@ +// Copyright 2016 The OPA Authors. All rights reserved. +// Use of this source code is governed by an Apache2 +// license that can be found in the LICENSE file. + +package ast + +import ( + "fmt" + "sort" + "strings" +) + +// Errors represents a series of errors encountered during parsing, compiling, +// etc. +type Errors []*Error + +func (e Errors) Error() string { + + if len(e) == 0 { + return "no error(s)" + } + + if len(e) == 1 { + return fmt.Sprintf("1 error occurred: %v", e[0].Error()) + } + + s := []string{} + for _, err := range e { + s = append(s, err.Error()) + } + + return fmt.Sprintf("%d errors occurred:\n%s", len(e), strings.Join(s, "\n")) +} + +// Sort sorts the error slice by location. If the locations are equal then the +// error message is compared. +func (e Errors) Sort() { + sort.Slice(e, func(i, j int) bool { + a := e[i] + b := e[j] + + if cmp := a.Location.Compare(b.Location); cmp != 0 { + return cmp < 0 + } + + return a.Error() < b.Error() + }) +} + +const ( + // ParseErr indicates an unclassified parse error occurred. + ParseErr = "rego_parse_error" + + // CompileErr indicates an unclassified compile error occurred. + CompileErr = "rego_compile_error" + + // TypeErr indicates a type error was caught. + TypeErr = "rego_type_error" + + // UnsafeVarErr indicates an unsafe variable was found during compilation. + UnsafeVarErr = "rego_unsafe_var_error" + + // RecursionErr indicates recursion was found during compilation. + RecursionErr = "rego_recursion_error" +) + +// IsError returns true if err is an AST error with code. +func IsError(code string, err error) bool { + if err, ok := err.(*Error); ok { + return err.Code == code + } + return false +} + +// ErrorDetails defines the interface for detailed error messages. +type ErrorDetails interface { + Lines() []string +} + +// Error represents a single error caught during parsing, compiling, etc. +type Error struct { + Code string `json:"code"` + Message string `json:"message"` + Location *Location `json:"location,omitempty"` + Details ErrorDetails `json:"details,omitempty"` +} + +func (e *Error) Error() string { + + var prefix string + + if e.Location != nil { + + if len(e.Location.File) > 0 { + prefix += e.Location.File + ":" + fmt.Sprint(e.Location.Row) + } else { + prefix += fmt.Sprint(e.Location.Row) + ":" + fmt.Sprint(e.Location.Col) + } + } + + msg := fmt.Sprintf("%v: %v", e.Code, e.Message) + + if len(prefix) > 0 { + msg = prefix + ": " + msg + } + + if e.Details != nil { + for _, line := range e.Details.Lines() { + msg += "\n\t" + line + } + } + + return msg +} + +// NewError returns a new Error object. +func NewError(code string, loc *Location, f string, a ...interface{}) *Error { + return &Error{ + Code: code, + Location: loc, + Message: fmt.Sprintf(f, a...), + } +} + +var ( + errPartialRuleAssignOperator = fmt.Errorf("partial rules must use = operator (not := operator)") + errElseAssignOperator = fmt.Errorf("else keyword cannot be used on rule declared with := operator") + errFunctionAssignOperator = fmt.Errorf("functions must use = operator (not := operator)") +) + +func errTermAssignOperator(x interface{}) error { + return fmt.Errorf("cannot assign to %v", TypeName(x)) +} diff --git a/vendor/github.com/open-policy-agent/opa/ast/fuzz.go b/vendor/github.com/open-policy-agent/opa/ast/fuzz.go new file mode 100644 index 000000000..449729b9e --- /dev/null +++ b/vendor/github.com/open-policy-agent/opa/ast/fuzz.go @@ -0,0 +1,28 @@ +// +build gofuzz + +package ast + +import ( + "regexp" +) + +// nested { and [ tokens cause the parse time to explode. +// see: https://github.com/mna/pigeon/issues/75 +var blacklistRegexp = regexp.MustCompile(`[{(\[]{5,}`) + +func Fuzz(data []byte) int { + + if blacklistRegexp.Match(data) { + return -1 + } + + str := string(data) + _, _, err := ParseStatements("", str) + + if err == nil { + CompileModules(map[string]string{"": str}) + return 1 + } + + return 0 +} diff --git a/vendor/github.com/open-policy-agent/opa/ast/index.go b/vendor/github.com/open-policy-agent/opa/ast/index.go new file mode 100644 index 000000000..d387353ab --- /dev/null +++ b/vendor/github.com/open-policy-agent/opa/ast/index.go @@ -0,0 +1,798 @@ +// Copyright 2017 The OPA Authors. All rights reserved. +// Use of this source code is governed by an Apache2 +// license that can be found in the LICENSE file. + +package ast + +import ( + "fmt" + "io" + "sort" + "strings" + + "github.com/open-policy-agent/opa/util" +) + +// RuleIndex defines the interface for rule indices. +type RuleIndex interface { + + // Build tries to construct an index for the given rules. If the index was + // constructed, ok is true, otherwise false. + Build(rules []*Rule) (ok bool) + + // Lookup searches the index for rules that will match the provided + // resolver. If the resolver returns an error, it is returned via err. + Lookup(resolver ValueResolver) (result *IndexResult, err error) + + // AllRules traverses the index and returns all rules that will match + // the provided resolver without any optimizations (effectively with + // indexing disabled). If the resolver returns an error, it is returned + // via err. + AllRules(resolver ValueResolver) (result *IndexResult, err error) +} + +// IndexResult contains the result of an index lookup. +type IndexResult struct { + Kind DocKind + Rules []*Rule + Else map[*Rule][]*Rule + Default *Rule +} + +// NewIndexResult returns a new IndexResult object. +func NewIndexResult(kind DocKind) *IndexResult { + return &IndexResult{ + Kind: kind, + Else: map[*Rule][]*Rule{}, + } +} + +// Empty returns true if there are no rules to evaluate. +func (ir *IndexResult) Empty() bool { + return len(ir.Rules) == 0 && ir.Default == nil +} + +type baseDocEqIndex struct { + isVirtual func(Ref) bool + root *trieNode + defaultRule *Rule + kind DocKind +} + +func newBaseDocEqIndex(isVirtual func(Ref) bool) *baseDocEqIndex { + return &baseDocEqIndex{ + isVirtual: isVirtual, + root: newTrieNodeImpl(), + } +} + +func (i *baseDocEqIndex) Build(rules []*Rule) bool { + if len(rules) == 0 { + return false + } + + i.kind = rules[0].Head.DocKind() + indices := newrefindices(i.isVirtual) + + // build indices for each rule. + for idx := range rules { + WalkRules(rules[idx], func(rule *Rule) bool { + if rule.Default { + i.defaultRule = rule + return false + } + for _, expr := range rule.Body { + indices.Update(rule, expr) + } + return false + }) + } + + // build trie out of indices. + for idx := range rules { + var prio int + WalkRules(rules[idx], func(rule *Rule) bool { + if rule.Default { + return false + } + node := i.root + if indices.Indexed(rule) { + for _, ref := range indices.Sorted() { + node = node.Insert(ref, indices.Value(rule, ref), indices.Mapper(rule, ref)) + } + } + // Insert rule into trie with (insertion order, priority order) + // tuple. Retaining the insertion order allows us to return rules + // in the order they were passed to this function. + node.rules = append(node.rules, &ruleNode{[...]int{idx, prio}, rule}) + prio++ + return false + }) + + } + + return true +} + +func (i *baseDocEqIndex) Lookup(resolver ValueResolver) (*IndexResult, error) { + + tr := newTrieTraversalResult() + + err := i.root.Traverse(resolver, tr) + if err != nil { + return nil, err + } + + result := NewIndexResult(i.kind) + result.Default = i.defaultRule + result.Rules = make([]*Rule, 0, len(tr.ordering)) + + for _, pos := range tr.ordering { + sort.Slice(tr.unordered[pos], func(i, j int) bool { + return tr.unordered[pos][i].prio[1] < tr.unordered[pos][j].prio[1] + }) + nodes := tr.unordered[pos] + root := nodes[0].rule + result.Rules = append(result.Rules, root) + if len(nodes) > 1 { + result.Else[root] = make([]*Rule, len(nodes)-1) + for i := 1; i < len(nodes); i++ { + result.Else[root][i-1] = nodes[i].rule + } + } + } + + return result, nil +} + +func (i *baseDocEqIndex) AllRules(resolver ValueResolver) (*IndexResult, error) { + tr := newTrieTraversalResult() + + // Walk over the rule trie and accumulate _all_ rules + rw := &ruleWalker{result: tr} + i.root.Do(rw) + + result := NewIndexResult(i.kind) + result.Default = i.defaultRule + result.Rules = make([]*Rule, 0, len(tr.ordering)) + + for _, pos := range tr.ordering { + sort.Slice(tr.unordered[pos], func(i, j int) bool { + return tr.unordered[pos][i].prio[1] < tr.unordered[pos][j].prio[1] + }) + nodes := tr.unordered[pos] + root := nodes[0].rule + result.Rules = append(result.Rules, root) + if len(nodes) > 1 { + result.Else[root] = make([]*Rule, len(nodes)-1) + for i := 1; i < len(nodes); i++ { + result.Else[root][i-1] = nodes[i].rule + } + } + } + + return result, nil +} + +type ruleWalker struct { + result *trieTraversalResult +} + +func (r *ruleWalker) Do(x interface{}) trieWalker { + tn := x.(*trieNode) + for _, rn := range tn.rules { + r.result.Add(rn) + } + return r +} + +type valueMapper func(Value) Value + +type refindex struct { + Ref Ref + Value Value + Mapper func(Value) Value +} + +type refindices struct { + isVirtual func(Ref) bool + rules map[*Rule][]*refindex + frequency *util.HashMap + sorted []Ref +} + +func newrefindices(isVirtual func(Ref) bool) *refindices { + return &refindices{ + isVirtual: isVirtual, + rules: map[*Rule][]*refindex{}, + frequency: util.NewHashMap(func(a, b util.T) bool { + r1, r2 := a.(Ref), b.(Ref) + return r1.Equal(r2) + }, func(x util.T) int { + return x.(Ref).Hash() + }), + } +} + +// Update attempts to update the refindices for the given expression in the +// given rule. If the expression cannot be indexed the update does not affect +// the indices. +func (i *refindices) Update(rule *Rule, expr *Expr) { + + if expr.Negated { + return + } + + if len(expr.With) > 0 { + // NOTE(tsandall): In the future, we may need to consider expressions + // that have with statements applied to them. + return + } + + op := expr.Operator() + + if op.Equal(Equality.Ref()) || op.Equal(Equal.Ref()) { + + i.updateEq(rule, expr) + + } else if op.Equal(GlobMatch.Ref()) { + + i.updateGlobMatch(rule, expr) + } +} + +// Sorted returns a sorted list of references that the indices were built from. +// References that appear more frequently in the indexed rules are ordered +// before less frequently appearing references. +func (i *refindices) Sorted() []Ref { + + if i.sorted == nil { + counts := make([]int, 0, i.frequency.Len()) + i.sorted = make([]Ref, 0, i.frequency.Len()) + + i.frequency.Iter(func(k, v util.T) bool { + counts = append(counts, v.(int)) + i.sorted = append(i.sorted, k.(Ref)) + return false + }) + + sort.Slice(i.sorted, func(i, j int) bool { + return counts[i] > counts[j] + }) + } + + return i.sorted +} + +func (i *refindices) Indexed(rule *Rule) bool { + return len(i.rules[rule]) > 0 +} + +func (i *refindices) Value(rule *Rule, ref Ref) Value { + if index := i.index(rule, ref); index != nil { + return index.Value + } + return nil +} + +func (i *refindices) Mapper(rule *Rule, ref Ref) valueMapper { + if index := i.index(rule, ref); index != nil { + return index.Mapper + } + return nil +} + +func (i *refindices) updateEq(rule *Rule, expr *Expr) { + a, b := expr.Operand(0), expr.Operand(1) + if ref, value, ok := eqOperandsToRefAndValue(i.isVirtual, a, b); ok { + i.insert(rule, &refindex{ + Ref: ref, + Value: value, + }) + } else if ref, value, ok := eqOperandsToRefAndValue(i.isVirtual, b, a); ok { + i.insert(rule, &refindex{ + Ref: ref, + Value: value, + }) + } +} + +func (i *refindices) updateGlobMatch(rule *Rule, expr *Expr) { + + delim, ok := globDelimiterToString(expr.Operand(1)) + if !ok { + return + } + + if arr := globPatternToArray(expr.Operand(0), delim); arr != nil { + // The 3rd operand of glob.match is the value to match. We assume the + // 3rd operand was a reference that has been rewritten and bound to a + // variable earlier in the query. + match := expr.Operand(2) + if _, ok := match.Value.(Var); ok { + for _, other := range i.rules[rule] { + if _, ok := other.Value.(Var); ok && other.Value.Compare(match.Value) == 0 { + i.insert(rule, &refindex{ + Ref: other.Ref, + Value: arr.Value, + Mapper: func(v Value) Value { + if s, ok := v.(String); ok { + return stringSliceToArray(splitStringEscaped(string(s), delim)) + } + return v + }, + }) + } + } + } + } +} + +func (i *refindices) insert(rule *Rule, index *refindex) { + + count, ok := i.frequency.Get(index.Ref) + if !ok { + count = 0 + } + + i.frequency.Put(index.Ref, count.(int)+1) + + for pos, other := range i.rules[rule] { + if other.Ref.Equal(index.Ref) { + i.rules[rule][pos] = index + return + } + } + + i.rules[rule] = append(i.rules[rule], index) +} + +func (i *refindices) index(rule *Rule, ref Ref) *refindex { + for _, index := range i.rules[rule] { + if index.Ref.Equal(ref) { + return index + } + } + return nil +} + +type trieWalker interface { + Do(x interface{}) trieWalker +} + +type trieTraversalResult struct { + unordered map[int][]*ruleNode + ordering []int +} + +func newTrieTraversalResult() *trieTraversalResult { + return &trieTraversalResult{ + unordered: map[int][]*ruleNode{}, + } +} + +func (tr *trieTraversalResult) Add(node *ruleNode) { + root := node.prio[0] + nodes, ok := tr.unordered[root] + if !ok { + tr.ordering = append(tr.ordering, root) + } + tr.unordered[root] = append(nodes, node) +} + +type trieNode struct { + ref Ref + mapper valueMapper + next *trieNode + any *trieNode + undefined *trieNode + scalars map[Value]*trieNode + array *trieNode + rules []*ruleNode +} + +func (node *trieNode) String() string { + var flags []string + flags = append(flags, fmt.Sprintf("self:%p", node)) + if len(node.ref) > 0 { + flags = append(flags, node.ref.String()) + } + if node.next != nil { + flags = append(flags, fmt.Sprintf("next:%p", node.next)) + } + if node.any != nil { + flags = append(flags, fmt.Sprintf("any:%p", node.any)) + } + if node.undefined != nil { + flags = append(flags, fmt.Sprintf("undefined:%p", node.undefined)) + } + if node.array != nil { + flags = append(flags, fmt.Sprintf("array:%p", node.array)) + } + if len(node.scalars) > 0 { + buf := []string{} + for k, v := range node.scalars { + buf = append(buf, fmt.Sprintf("scalar(%v):%p", k, v)) + } + sort.Strings(buf) + flags = append(flags, strings.Join(buf, " ")) + } + if len(node.rules) > 0 { + flags = append(flags, fmt.Sprintf("%d rule(s)", len(node.rules))) + } + if node.mapper != nil { + flags = append(flags, "mapper") + } + return strings.Join(flags, " ") +} + +type ruleNode struct { + prio [2]int + rule *Rule +} + +func newTrieNodeImpl() *trieNode { + return &trieNode{ + scalars: map[Value]*trieNode{}, + } +} + +func (node *trieNode) Do(walker trieWalker) { + next := walker.Do(node) + if next == nil { + return + } + if node.any != nil { + node.any.Do(next) + } + if node.undefined != nil { + node.undefined.Do(next) + } + for _, child := range node.scalars { + child.Do(next) + } + if node.array != nil { + node.array.Do(next) + } + if node.next != nil { + node.next.Do(next) + } +} + +func (node *trieNode) Insert(ref Ref, value Value, mapper valueMapper) *trieNode { + + if node.next == nil { + node.next = newTrieNodeImpl() + node.next.ref = ref + } + + node.next.mapper = mapper + + return node.next.insertValue(value) +} + +func (node *trieNode) Traverse(resolver ValueResolver, tr *trieTraversalResult) error { + + if node == nil { + return nil + } + + for i := range node.rules { + tr.Add(node.rules[i]) + } + + return node.next.traverse(resolver, tr) +} + +func (node *trieNode) insertValue(value Value) *trieNode { + + switch value := value.(type) { + case nil: + if node.undefined == nil { + node.undefined = newTrieNodeImpl() + } + return node.undefined + case Var: + if node.any == nil { + node.any = newTrieNodeImpl() + } + return node.any + case Null, Boolean, Number, String: + child, ok := node.scalars[value] + if !ok { + child = newTrieNodeImpl() + node.scalars[value] = child + } + return child + case Array: + if node.array == nil { + node.array = newTrieNodeImpl() + } + return node.array.insertArray(value) + } + + panic("illegal value") +} + +func (node *trieNode) insertArray(arr Array) *trieNode { + + if len(arr) == 0 { + return node + } + + switch head := arr[0].Value.(type) { + case Var: + if node.any == nil { + node.any = newTrieNodeImpl() + } + return node.any.insertArray(arr[1:]) + case Null, Boolean, Number, String: + child, ok := node.scalars[head] + if !ok { + child = newTrieNodeImpl() + node.scalars[head] = child + } + return child.insertArray(arr[1:]) + } + + panic("illegal value") +} + +func (node *trieNode) traverse(resolver ValueResolver, tr *trieTraversalResult) error { + + if node == nil { + return nil + } + + v, err := resolver.Resolve(node.ref) + if err != nil { + if IsUnknownValueErr(err) { + return node.traverseUnknown(resolver, tr) + } + return err + } + + if node.undefined != nil { + node.undefined.Traverse(resolver, tr) + } + + if v == nil { + return nil + } + + if node.any != nil { + node.any.Traverse(resolver, tr) + } + + if node.mapper != nil { + v = node.mapper(v) + } + + return node.traverseValue(resolver, tr, v) +} + +func (node *trieNode) traverseValue(resolver ValueResolver, tr *trieTraversalResult, value Value) error { + + switch value := value.(type) { + case Array: + if node.array == nil { + return nil + } + return node.array.traverseArray(resolver, tr, value) + + case Null, Boolean, Number, String: + child, ok := node.scalars[value] + if !ok { + return nil + } + return child.Traverse(resolver, tr) + } + + return nil +} + +func (node *trieNode) traverseArray(resolver ValueResolver, tr *trieTraversalResult, arr Array) error { + + if len(arr) == 0 { + return node.Traverse(resolver, tr) + } + + head := arr[0].Value + + if !IsScalar(head) { + return nil + } + + if node.any != nil { + node.any.traverseArray(resolver, tr, arr[1:]) + } + + child, ok := node.scalars[head] + if !ok { + return nil + } + + return child.traverseArray(resolver, tr, arr[1:]) +} + +func (node *trieNode) traverseUnknown(resolver ValueResolver, tr *trieTraversalResult) error { + + if node == nil { + return nil + } + + if err := node.Traverse(resolver, tr); err != nil { + return err + } + + if err := node.undefined.traverseUnknown(resolver, tr); err != nil { + return err + } + + if err := node.any.traverseUnknown(resolver, tr); err != nil { + return err + } + + if err := node.array.traverseUnknown(resolver, tr); err != nil { + return err + } + + for _, child := range node.scalars { + if err := child.traverseUnknown(resolver, tr); err != nil { + return err + } + } + + return nil +} + +type triePrinter struct { + depth int + w io.Writer +} + +func (p triePrinter) Do(x interface{}) trieWalker { + padding := strings.Repeat(" ", p.depth) + fmt.Fprintf(p.w, "%v%v\n", padding, x) + p.depth++ + return p +} + +func eqOperandsToRefAndValue(isVirtual func(Ref) bool, a, b *Term) (Ref, Value, bool) { + + ref, ok := a.Value.(Ref) + if !ok { + return nil, nil, false + } + + if !RootDocumentNames.Contains(ref[0]) { + return nil, nil, false + } + + if isVirtual(ref) { + return nil, nil, false + } + + if ref.IsNested() || !ref.IsGround() { + return nil, nil, false + } + + switch b := b.Value.(type) { + case Null, Boolean, Number, String, Var: + return ref, b, true + case Array: + stop := false + first := true + vis := NewGenericVisitor(func(x interface{}) bool { + if first { + first = false + return false + } + switch x.(type) { + // No nested structures or values that require evaluation (other than var). + case Array, Object, Set, *ArrayComprehension, *ObjectComprehension, *SetComprehension, Ref: + stop = true + } + return stop + }) + vis.Walk(b) + if !stop { + return ref, b, true + } + } + + return nil, nil, false +} + +func globDelimiterToString(delim *Term) (string, bool) { + + arr, ok := delim.Value.(Array) + if !ok { + return "", false + } + + var result string + + if len(arr) == 0 { + result = "." + } else { + for _, term := range arr { + s, ok := term.Value.(String) + if !ok { + return "", false + } + result += string(s) + } + } + + return result, true +} + +func globPatternToArray(pattern *Term, delim string) *Term { + + s, ok := pattern.Value.(String) + if !ok { + return nil + } + + parts := splitStringEscaped(string(s), delim) + result := make(Array, len(parts)) + + for i := range parts { + if parts[i] == "*" { + result[i] = VarTerm("$globwildcard") + } else { + var escaped bool + for _, c := range parts[i] { + if c == '\\' { + escaped = !escaped + continue + } + if !escaped { + switch c { + case '[', '?', '{', '*': + // TODO(tsandall): super glob and character pattern + // matching not supported yet. + return nil + } + } + escaped = false + } + result[i] = StringTerm(parts[i]) + } + } + + return NewTerm(result) +} + +// splits s on characters in delim except if delim characters have been escaped +// with reverse solidus. +func splitStringEscaped(s string, delim string) []string { + + var last, curr int + var escaped bool + var result []string + + for ; curr < len(s); curr++ { + if s[curr] == '\\' || escaped { + escaped = !escaped + continue + } + if strings.ContainsRune(delim, rune(s[curr])) { + result = append(result, s[last:curr]) + last = curr + 1 + } + } + + result = append(result, s[last:]) + + return result +} + +func stringSliceToArray(s []string) (result Array) { + result = make(Array, len(s)) + for i := range s { + result[i] = StringTerm(s[i]) + } + return +} diff --git a/vendor/github.com/open-policy-agent/opa/ast/map.go b/vendor/github.com/open-policy-agent/opa/ast/map.go new file mode 100644 index 000000000..b0cc9eb60 --- /dev/null +++ b/vendor/github.com/open-policy-agent/opa/ast/map.go @@ -0,0 +1,133 @@ +// Copyright 2016 The OPA Authors. All rights reserved. +// Use of this source code is governed by an Apache2 +// license that can be found in the LICENSE file. + +package ast + +import ( + "encoding/json" + + "github.com/open-policy-agent/opa/util" +) + +// ValueMap represents a key/value map between AST term values. Any type of term +// can be used as a key in the map. +type ValueMap struct { + hashMap *util.HashMap +} + +// NewValueMap returns a new ValueMap. +func NewValueMap() *ValueMap { + vs := &ValueMap{ + hashMap: util.NewHashMap(valueEq, valueHash), + } + return vs +} + +// MarshalJSON provides a custom marshaller for the ValueMap which +// will include the key, value, and value type. +func (vs *ValueMap) MarshalJSON() ([]byte, error) { + var tmp []map[string]interface{} + vs.Iter(func(k Value, v Value) bool { + tmp = append(tmp, map[string]interface{}{ + "name": k.String(), + "type": TypeName(v), + "value": v, + }) + return false + }) + return json.Marshal(tmp) +} + +// Copy returns a shallow copy of the ValueMap. +func (vs *ValueMap) Copy() *ValueMap { + if vs == nil { + return nil + } + cpy := NewValueMap() + cpy.hashMap = vs.hashMap.Copy() + return cpy +} + +// Equal returns true if this ValueMap equals the other. +func (vs *ValueMap) Equal(other *ValueMap) bool { + if vs == nil { + return other == nil || other.Len() == 0 + } + if other == nil { + return vs == nil || vs.Len() == 0 + } + return vs.hashMap.Equal(other.hashMap) +} + +// Len returns the number of elements in the map. +func (vs *ValueMap) Len() int { + if vs == nil { + return 0 + } + return vs.hashMap.Len() +} + +// Get returns the value in the map for k. +func (vs *ValueMap) Get(k Value) Value { + if vs != nil { + if v, ok := vs.hashMap.Get(k); ok { + return v.(Value) + } + } + return nil +} + +// Hash returns a hash code for this ValueMap. +func (vs *ValueMap) Hash() int { + if vs == nil { + return 0 + } + return vs.hashMap.Hash() +} + +// Iter calls the iter function for each key/value pair in the map. If the iter +// function returns true, iteration stops. +func (vs *ValueMap) Iter(iter func(Value, Value) bool) bool { + if vs == nil { + return false + } + return vs.hashMap.Iter(func(kt, vt util.T) bool { + k := kt.(Value) + v := vt.(Value) + return iter(k, v) + }) +} + +// Put inserts a key k into the map with value v. +func (vs *ValueMap) Put(k, v Value) { + if vs == nil { + panic("put on nil value map") + } + vs.hashMap.Put(k, v) +} + +// Delete removes a key k from the map. +func (vs *ValueMap) Delete(k Value) { + if vs == nil { + return + } + vs.hashMap.Delete(k) +} + +func (vs *ValueMap) String() string { + if vs == nil { + return "{}" + } + return vs.hashMap.String() +} + +func valueHash(v util.T) int { + return v.(Value).Hash() +} + +func valueEq(a, b util.T) bool { + av := a.(Value) + bv := b.(Value) + return av.Compare(bv) == 0 +} diff --git a/vendor/github.com/open-policy-agent/opa/ast/parser.go b/vendor/github.com/open-policy-agent/opa/ast/parser.go new file mode 100644 index 000000000..d2236de7e --- /dev/null +++ b/vendor/github.com/open-policy-agent/opa/ast/parser.go @@ -0,0 +1,5150 @@ +// Code generated by pigeon; DO NOT EDIT. + +package ast + +import ( + "bytes" + "errors" + "fmt" + "io" + "io/ioutil" + "math" + "os" + "sort" + "strconv" + "strings" + "unicode" + "unicode/utf8" +) + +var g = &grammar{ + rules: []*rule{ + { + name: "Program", + pos: position{line: 5, col: 1, offset: 17}, + expr: &actionExpr{ + pos: position{line: 5, col: 12, offset: 28}, + run: (*parser).callonProgram1, + expr: &seqExpr{ + pos: position{line: 5, col: 12, offset: 28}, + exprs: []interface{}{ + &ruleRefExpr{ + pos: position{line: 5, col: 12, offset: 28}, + name: "_", + }, + &labeledExpr{ + pos: position{line: 5, col: 14, offset: 30}, + label: "vals", + expr: &zeroOrOneExpr{ + pos: position{line: 5, col: 19, offset: 35}, + expr: &seqExpr{ + pos: position{line: 5, col: 20, offset: 36}, + exprs: []interface{}{ + &ruleRefExpr{ + pos: position{line: 5, col: 20, offset: 36}, + name: "Stmt", + }, + &zeroOrMoreExpr{ + pos: position{line: 5, col: 25, offset: 41}, + expr: &seqExpr{ + pos: position{line: 5, col: 26, offset: 42}, + exprs: []interface{}{ + &ruleRefExpr{ + pos: position{line: 5, col: 26, offset: 42}, + name: "ws", + }, + &ruleRefExpr{ + pos: position{line: 5, col: 29, offset: 45}, + name: "Stmt", + }, + }, + }, + }, + }, + }, + }, + }, + &ruleRefExpr{ + pos: position{line: 5, col: 38, offset: 54}, + name: "_", + }, + &ruleRefExpr{ + pos: position{line: 5, col: 40, offset: 56}, + name: "EOF", + }, + }, + }, + }, + }, + { + name: "Stmt", + pos: position{line: 9, col: 1, offset: 97}, + expr: &actionExpr{ + pos: position{line: 9, col: 9, offset: 105}, + run: (*parser).callonStmt1, + expr: &labeledExpr{ + pos: position{line: 9, col: 9, offset: 105}, + label: "val", + expr: &choiceExpr{ + pos: position{line: 9, col: 14, offset: 110}, + alternatives: []interface{}{ + &ruleRefExpr{ + pos: position{line: 9, col: 14, offset: 110}, + name: "Package", + }, + &ruleRefExpr{ + pos: position{line: 9, col: 24, offset: 120}, + name: "Import", + }, + &ruleRefExpr{ + pos: position{line: 9, col: 33, offset: 129}, + name: "Rules", + }, + &ruleRefExpr{ + pos: position{line: 9, col: 41, offset: 137}, + name: "Body", + }, + &ruleRefExpr{ + pos: position{line: 9, col: 48, offset: 144}, + name: "Comment", + }, + }, + }, + }, + }, + }, + { + name: "Package", + pos: position{line: 13, col: 1, offset: 178}, + expr: &actionExpr{ + pos: position{line: 13, col: 12, offset: 189}, + run: (*parser).callonPackage1, + expr: &seqExpr{ + pos: position{line: 13, col: 12, offset: 189}, + exprs: []interface{}{ + &litMatcher{ + pos: position{line: 13, col: 12, offset: 189}, + val: "package", + ignoreCase: false, + }, + &ruleRefExpr{ + pos: position{line: 13, col: 22, offset: 199}, + name: "ws", + }, + &labeledExpr{ + pos: position{line: 13, col: 25, offset: 202}, + label: "val", + expr: &choiceExpr{ + pos: position{line: 13, col: 30, offset: 207}, + alternatives: []interface{}{ + &ruleRefExpr{ + pos: position{line: 13, col: 30, offset: 207}, + name: "Ref", + }, + &ruleRefExpr{ + pos: position{line: 13, col: 36, offset: 213}, + name: "Var", + }, + }, + }, + }, + }, + }, + }, + }, + { + name: "Import", + pos: position{line: 17, col: 1, offset: 271}, + expr: &actionExpr{ + pos: position{line: 17, col: 11, offset: 281}, + run: (*parser).callonImport1, + expr: &seqExpr{ + pos: position{line: 17, col: 11, offset: 281}, + exprs: []interface{}{ + &litMatcher{ + pos: position{line: 17, col: 11, offset: 281}, + val: "import", + ignoreCase: false, + }, + &ruleRefExpr{ + pos: position{line: 17, col: 20, offset: 290}, + name: "ws", + }, + &labeledExpr{ + pos: position{line: 17, col: 23, offset: 293}, + label: "path", + expr: &choiceExpr{ + pos: position{line: 17, col: 29, offset: 299}, + alternatives: []interface{}{ + &ruleRefExpr{ + pos: position{line: 17, col: 29, offset: 299}, + name: "Ref", + }, + &ruleRefExpr{ + pos: position{line: 17, col: 35, offset: 305}, + name: "Var", + }, + }, + }, + }, + &labeledExpr{ + pos: position{line: 17, col: 40, offset: 310}, + label: "alias", + expr: &zeroOrOneExpr{ + pos: position{line: 17, col: 46, offset: 316}, + expr: &seqExpr{ + pos: position{line: 17, col: 47, offset: 317}, + exprs: []interface{}{ + &ruleRefExpr{ + pos: position{line: 17, col: 47, offset: 317}, + name: "ws", + }, + &litMatcher{ + pos: position{line: 17, col: 50, offset: 320}, + val: "as", + ignoreCase: false, + }, + &ruleRefExpr{ + pos: position{line: 17, col: 55, offset: 325}, + name: "ws", + }, + &ruleRefExpr{ + pos: position{line: 17, col: 58, offset: 328}, + name: "Var", + }, + }, + }, + }, + }, + }, + }, + }, + }, + { + name: "Rules", + pos: position{line: 21, col: 1, offset: 394}, + expr: &choiceExpr{ + pos: position{line: 21, col: 10, offset: 403}, + alternatives: []interface{}{ + &ruleRefExpr{ + pos: position{line: 21, col: 10, offset: 403}, + name: "DefaultRules", + }, + &ruleRefExpr{ + pos: position{line: 21, col: 25, offset: 418}, + name: "NormalRules", + }, + }, + }, + }, + { + name: "DefaultRules", + pos: position{line: 23, col: 1, offset: 431}, + expr: &actionExpr{ + pos: position{line: 23, col: 17, offset: 447}, + run: (*parser).callonDefaultRules1, + expr: &seqExpr{ + pos: position{line: 23, col: 17, offset: 447}, + exprs: []interface{}{ + &litMatcher{ + pos: position{line: 23, col: 17, offset: 447}, + val: "default", + ignoreCase: false, + }, + &ruleRefExpr{ + pos: position{line: 23, col: 27, offset: 457}, + name: "ws", + }, + &labeledExpr{ + pos: position{line: 23, col: 30, offset: 460}, + label: "name", + expr: &ruleRefExpr{ + pos: position{line: 23, col: 35, offset: 465}, + name: "Var", + }, + }, + &ruleRefExpr{ + pos: position{line: 23, col: 39, offset: 469}, + name: "_", + }, + &labeledExpr{ + pos: position{line: 23, col: 41, offset: 471}, + label: "operator", + expr: &choiceExpr{ + pos: position{line: 23, col: 52, offset: 482}, + alternatives: []interface{}{ + &litMatcher{ + pos: position{line: 23, col: 52, offset: 482}, + val: ":=", + ignoreCase: false, + }, + &litMatcher{ + pos: position{line: 23, col: 59, offset: 489}, + val: "=", + ignoreCase: false, + }, + }, + }, + }, + &ruleRefExpr{ + pos: position{line: 23, col: 65, offset: 495}, + name: "_", + }, + &labeledExpr{ + pos: position{line: 23, col: 67, offset: 497}, + label: "value", + expr: &ruleRefExpr{ + pos: position{line: 23, col: 73, offset: 503}, + name: "Term", + }, + }, + }, + }, + }, + }, + { + name: "NormalRules", + pos: position{line: 27, col: 1, offset: 583}, + expr: &actionExpr{ + pos: position{line: 27, col: 16, offset: 598}, + run: (*parser).callonNormalRules1, + expr: &seqExpr{ + pos: position{line: 27, col: 16, offset: 598}, + exprs: []interface{}{ + &labeledExpr{ + pos: position{line: 27, col: 16, offset: 598}, + label: "head", + expr: &choiceExpr{ + pos: position{line: 27, col: 22, offset: 604}, + alternatives: []interface{}{ + &ruleRefExpr{ + pos: position{line: 27, col: 22, offset: 604}, + name: "PartialRuleHead", + }, + &ruleRefExpr{ + pos: position{line: 27, col: 40, offset: 622}, + name: "RuleHead", + }, + }, + }, + }, + &ruleRefExpr{ + pos: position{line: 27, col: 50, offset: 632}, + name: "_", + }, + &labeledExpr{ + pos: position{line: 27, col: 52, offset: 634}, + label: "rest", + expr: &seqExpr{ + pos: position{line: 27, col: 58, offset: 640}, + exprs: []interface{}{ + &ruleRefExpr{ + pos: position{line: 27, col: 58, offset: 640}, + name: "NonEmptyBraceEnclosedBody", + }, + &zeroOrMoreExpr{ + pos: position{line: 27, col: 84, offset: 666}, + expr: &seqExpr{ + pos: position{line: 27, col: 86, offset: 668}, + exprs: []interface{}{ + &ruleRefExpr{ + pos: position{line: 27, col: 86, offset: 668}, + name: "_", + }, + &ruleRefExpr{ + pos: position{line: 27, col: 88, offset: 670}, + name: "RuleExt", + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + { + name: "PartialRuleHead", + pos: position{line: 31, col: 1, offset: 739}, + expr: &actionExpr{ + pos: position{line: 31, col: 20, offset: 758}, + run: (*parser).callonPartialRuleHead1, + expr: &seqExpr{ + pos: position{line: 31, col: 20, offset: 758}, + exprs: []interface{}{ + &labeledExpr{ + pos: position{line: 31, col: 20, offset: 758}, + label: "name", + expr: &ruleRefExpr{ + pos: position{line: 31, col: 25, offset: 763}, + name: "Var", + }, + }, + &labeledExpr{ + pos: position{line: 31, col: 29, offset: 767}, + label: "args", + expr: &seqExpr{ + pos: position{line: 31, col: 36, offset: 774}, + exprs: []interface{}{ + &ruleRefExpr{ + pos: position{line: 31, col: 36, offset: 774}, + name: "_", + }, + &litMatcher{ + pos: position{line: 31, col: 38, offset: 776}, + val: "(", + ignoreCase: false, + }, + &ruleRefExpr{ + pos: position{line: 31, col: 42, offset: 780}, + name: "_", + }, + &ruleRefExpr{ + pos: position{line: 31, col: 44, offset: 782}, + name: "Args", + }, + &ruleRefExpr{ + pos: position{line: 31, col: 49, offset: 787}, + name: "_", + }, + &litMatcher{ + pos: position{line: 31, col: 51, offset: 789}, + val: ")", + ignoreCase: false, + }, + &ruleRefExpr{ + pos: position{line: 31, col: 55, offset: 793}, + name: "_", + }, + }, + }, + }, + &labeledExpr{ + pos: position{line: 31, col: 59, offset: 797}, + label: "value", + expr: &zeroOrOneExpr{ + pos: position{line: 31, col: 65, offset: 803}, + expr: &seqExpr{ + pos: position{line: 31, col: 67, offset: 805}, + exprs: []interface{}{ + &ruleRefExpr{ + pos: position{line: 31, col: 67, offset: 805}, + name: "_", + }, + &choiceExpr{ + pos: position{line: 31, col: 71, offset: 809}, + alternatives: []interface{}{ + &litMatcher{ + pos: position{line: 31, col: 71, offset: 809}, + val: ":=", + ignoreCase: false, + }, + &litMatcher{ + pos: position{line: 31, col: 78, offset: 816}, + val: "=", + ignoreCase: false, + }, + }, + }, + &ruleRefExpr{ + pos: position{line: 31, col: 84, offset: 822}, + name: "_", + }, + &ruleRefExpr{ + pos: position{line: 31, col: 86, offset: 824}, + name: "ExprTerm", + }, + }, + }, + }, + }, + }, + }, + }, + }, + { + name: "RuleHead", + pos: position{line: 35, col: 1, offset: 909}, + expr: &actionExpr{ + pos: position{line: 35, col: 13, offset: 921}, + run: (*parser).callonRuleHead1, + expr: &seqExpr{ + pos: position{line: 35, col: 13, offset: 921}, + exprs: []interface{}{ + &labeledExpr{ + pos: position{line: 35, col: 13, offset: 921}, + label: "name", + expr: &ruleRefExpr{ + pos: position{line: 35, col: 18, offset: 926}, + name: "Var", + }, + }, + &labeledExpr{ + pos: position{line: 35, col: 22, offset: 930}, + label: "key", + expr: &zeroOrOneExpr{ + pos: position{line: 35, col: 26, offset: 934}, + expr: &seqExpr{ + pos: position{line: 35, col: 28, offset: 936}, + exprs: []interface{}{ + &ruleRefExpr{ + pos: position{line: 35, col: 28, offset: 936}, + name: "_", + }, + &litMatcher{ + pos: position{line: 35, col: 30, offset: 938}, + val: "[", + ignoreCase: false, + }, + &ruleRefExpr{ + pos: position{line: 35, col: 34, offset: 942}, + name: "_", + }, + &ruleRefExpr{ + pos: position{line: 35, col: 36, offset: 944}, + name: "ExprTerm", + }, + &ruleRefExpr{ + pos: position{line: 35, col: 45, offset: 953}, + name: "_", + }, + &litMatcher{ + pos: position{line: 35, col: 47, offset: 955}, + val: "]", + ignoreCase: false, + }, + &ruleRefExpr{ + pos: position{line: 35, col: 51, offset: 959}, + name: "_", + }, + }, + }, + }, + }, + &labeledExpr{ + pos: position{line: 35, col: 56, offset: 964}, + label: "value", + expr: &zeroOrOneExpr{ + pos: position{line: 35, col: 62, offset: 970}, + expr: &seqExpr{ + pos: position{line: 35, col: 64, offset: 972}, + exprs: []interface{}{ + &ruleRefExpr{ + pos: position{line: 35, col: 64, offset: 972}, + name: "_", + }, + &choiceExpr{ + pos: position{line: 35, col: 68, offset: 976}, + alternatives: []interface{}{ + &litMatcher{ + pos: position{line: 35, col: 68, offset: 976}, + val: ":=", + ignoreCase: false, + }, + &litMatcher{ + pos: position{line: 35, col: 75, offset: 983}, + val: "=", + ignoreCase: false, + }, + }, + }, + &ruleRefExpr{ + pos: position{line: 35, col: 81, offset: 989}, + name: "_", + }, + &ruleRefExpr{ + pos: position{line: 35, col: 83, offset: 991}, + name: "ExprTerm", + }, + }, + }, + }, + }, + }, + }, + }, + }, + { + name: "Args", + pos: position{line: 39, col: 1, offset: 1075}, + expr: &actionExpr{ + pos: position{line: 39, col: 9, offset: 1083}, + run: (*parser).callonArgs1, + expr: &labeledExpr{ + pos: position{line: 39, col: 9, offset: 1083}, + label: "list", + expr: &ruleRefExpr{ + pos: position{line: 39, col: 14, offset: 1088}, + name: "ExprTermList", + }, + }, + }, + }, + { + name: "Else", + pos: position{line: 43, col: 1, offset: 1132}, + expr: &actionExpr{ + pos: position{line: 43, col: 9, offset: 1140}, + run: (*parser).callonElse1, + expr: &seqExpr{ + pos: position{line: 43, col: 9, offset: 1140}, + exprs: []interface{}{ + &litMatcher{ + pos: position{line: 43, col: 9, offset: 1140}, + val: "else", + ignoreCase: false, + }, + &labeledExpr{ + pos: position{line: 43, col: 16, offset: 1147}, + label: "value", + expr: &zeroOrOneExpr{ + pos: position{line: 43, col: 22, offset: 1153}, + expr: &seqExpr{ + pos: position{line: 43, col: 24, offset: 1155}, + exprs: []interface{}{ + &ruleRefExpr{ + pos: position{line: 43, col: 24, offset: 1155}, + name: "_", + }, + &litMatcher{ + pos: position{line: 43, col: 26, offset: 1157}, + val: "=", + ignoreCase: false, + }, + &ruleRefExpr{ + pos: position{line: 43, col: 30, offset: 1161}, + name: "_", + }, + &ruleRefExpr{ + pos: position{line: 43, col: 32, offset: 1163}, + name: "Term", + }, + }, + }, + }, + }, + &labeledExpr{ + pos: position{line: 43, col: 40, offset: 1171}, + label: "body", + expr: &seqExpr{ + pos: position{line: 43, col: 47, offset: 1178}, + exprs: []interface{}{ + &ruleRefExpr{ + pos: position{line: 43, col: 47, offset: 1178}, + name: "_", + }, + &ruleRefExpr{ + pos: position{line: 43, col: 49, offset: 1180}, + name: "NonEmptyBraceEnclosedBody", + }, + }, + }, + }, + }, + }, + }, + }, + { + name: "RuleDup", + pos: position{line: 47, col: 1, offset: 1269}, + expr: &actionExpr{ + pos: position{line: 47, col: 12, offset: 1280}, + run: (*parser).callonRuleDup1, + expr: &labeledExpr{ + pos: position{line: 47, col: 12, offset: 1280}, + label: "b", + expr: &ruleRefExpr{ + pos: position{line: 47, col: 14, offset: 1282}, + name: "NonEmptyBraceEnclosedBody", + }, + }, + }, + }, + { + name: "RuleExt", + pos: position{line: 51, col: 1, offset: 1378}, + expr: &choiceExpr{ + pos: position{line: 51, col: 12, offset: 1389}, + alternatives: []interface{}{ + &ruleRefExpr{ + pos: position{line: 51, col: 12, offset: 1389}, + name: "Else", + }, + &ruleRefExpr{ + pos: position{line: 51, col: 19, offset: 1396}, + name: "RuleDup", + }, + }, + }, + }, + { + name: "Body", + pos: position{line: 53, col: 1, offset: 1405}, + expr: &choiceExpr{ + pos: position{line: 53, col: 9, offset: 1413}, + alternatives: []interface{}{ + &ruleRefExpr{ + pos: position{line: 53, col: 9, offset: 1413}, + name: "NonWhitespaceBody", + }, + &ruleRefExpr{ + pos: position{line: 53, col: 29, offset: 1433}, + name: "BraceEnclosedBody", + }, + }, + }, + }, + { + name: "NonEmptyBraceEnclosedBody", + pos: position{line: 55, col: 1, offset: 1452}, + expr: &actionExpr{ + pos: position{line: 55, col: 30, offset: 1481}, + run: (*parser).callonNonEmptyBraceEnclosedBody1, + expr: &seqExpr{ + pos: position{line: 55, col: 30, offset: 1481}, + exprs: []interface{}{ + &litMatcher{ + pos: position{line: 55, col: 30, offset: 1481}, + val: "{", + ignoreCase: false, + }, + &ruleRefExpr{ + pos: position{line: 55, col: 34, offset: 1485}, + name: "_", + }, + &labeledExpr{ + pos: position{line: 55, col: 36, offset: 1487}, + label: "val", + expr: &zeroOrOneExpr{ + pos: position{line: 55, col: 40, offset: 1491}, + expr: &ruleRefExpr{ + pos: position{line: 55, col: 40, offset: 1491}, + name: "WhitespaceBody", + }, + }, + }, + &ruleRefExpr{ + pos: position{line: 55, col: 56, offset: 1507}, + name: "_", + }, + &litMatcher{ + pos: position{line: 55, col: 58, offset: 1509}, + val: "}", + ignoreCase: false, + }, + }, + }, + }, + }, + { + name: "BraceEnclosedBody", + pos: position{line: 62, col: 1, offset: 1621}, + expr: &actionExpr{ + pos: position{line: 62, col: 22, offset: 1642}, + run: (*parser).callonBraceEnclosedBody1, + expr: &seqExpr{ + pos: position{line: 62, col: 22, offset: 1642}, + exprs: []interface{}{ + &litMatcher{ + pos: position{line: 62, col: 22, offset: 1642}, + val: "{", + ignoreCase: false, + }, + &ruleRefExpr{ + pos: position{line: 62, col: 26, offset: 1646}, + name: "_", + }, + &labeledExpr{ + pos: position{line: 62, col: 28, offset: 1648}, + label: "val", + expr: &zeroOrOneExpr{ + pos: position{line: 62, col: 32, offset: 1652}, + expr: &ruleRefExpr{ + pos: position{line: 62, col: 32, offset: 1652}, + name: "WhitespaceBody", + }, + }, + }, + &ruleRefExpr{ + pos: position{line: 62, col: 48, offset: 1668}, + name: "_", + }, + &litMatcher{ + pos: position{line: 62, col: 50, offset: 1670}, + val: "}", + ignoreCase: false, + }, + }, + }, + }, + }, + { + name: "WhitespaceBody", + pos: position{line: 66, col: 1, offset: 1737}, + expr: &actionExpr{ + pos: position{line: 66, col: 19, offset: 1755}, + run: (*parser).callonWhitespaceBody1, + expr: &seqExpr{ + pos: position{line: 66, col: 19, offset: 1755}, + exprs: []interface{}{ + &labeledExpr{ + pos: position{line: 66, col: 19, offset: 1755}, + label: "head", + expr: &ruleRefExpr{ + pos: position{line: 66, col: 24, offset: 1760}, + name: "Literal", + }, + }, + &labeledExpr{ + pos: position{line: 66, col: 32, offset: 1768}, + label: "tail", + expr: &zeroOrMoreExpr{ + pos: position{line: 66, col: 37, offset: 1773}, + expr: &seqExpr{ + pos: position{line: 66, col: 38, offset: 1774}, + exprs: []interface{}{ + &ruleRefExpr{ + pos: position{line: 66, col: 38, offset: 1774}, + name: "WhitespaceLiteralSeparator", + }, + &ruleRefExpr{ + pos: position{line: 66, col: 65, offset: 1801}, + name: "_", + }, + &ruleRefExpr{ + pos: position{line: 66, col: 67, offset: 1803}, + name: "Literal", + }, + }, + }, + }, + }, + }, + }, + }, + }, + { + name: "NonWhitespaceBody", + pos: position{line: 70, col: 1, offset: 1853}, + expr: &actionExpr{ + pos: position{line: 70, col: 22, offset: 1874}, + run: (*parser).callonNonWhitespaceBody1, + expr: &seqExpr{ + pos: position{line: 70, col: 22, offset: 1874}, + exprs: []interface{}{ + &labeledExpr{ + pos: position{line: 70, col: 22, offset: 1874}, + label: "head", + expr: &ruleRefExpr{ + pos: position{line: 70, col: 27, offset: 1879}, + name: "Literal", + }, + }, + &labeledExpr{ + pos: position{line: 70, col: 35, offset: 1887}, + label: "tail", + expr: &zeroOrMoreExpr{ + pos: position{line: 70, col: 40, offset: 1892}, + expr: &seqExpr{ + pos: position{line: 70, col: 42, offset: 1894}, + exprs: []interface{}{ + &ruleRefExpr{ + pos: position{line: 70, col: 42, offset: 1894}, + name: "_", + }, + &ruleRefExpr{ + pos: position{line: 70, col: 44, offset: 1896}, + name: "NonWhitespaceLiteralSeparator", + }, + &ruleRefExpr{ + pos: position{line: 70, col: 74, offset: 1926}, + name: "_", + }, + &ruleRefExpr{ + pos: position{line: 70, col: 76, offset: 1928}, + name: "Literal", + }, + }, + }, + }, + }, + }, + }, + }, + }, + { + name: "WhitespaceLiteralSeparator", + pos: position{line: 74, col: 1, offset: 1978}, + expr: &seqExpr{ + pos: position{line: 74, col: 31, offset: 2008}, + exprs: []interface{}{ + &zeroOrMoreExpr{ + pos: position{line: 74, col: 31, offset: 2008}, + expr: &charClassMatcher{ + pos: position{line: 74, col: 31, offset: 2008}, + val: "[ \\t]", + chars: []rune{' ', '\t'}, + ignoreCase: false, + inverted: false, + }, + }, + &choiceExpr{ + pos: position{line: 74, col: 39, offset: 2016}, + alternatives: []interface{}{ + &seqExpr{ + pos: position{line: 74, col: 40, offset: 2017}, + exprs: []interface{}{ + &ruleRefExpr{ + pos: position{line: 74, col: 40, offset: 2017}, + name: "NonWhitespaceLiteralSeparator", + }, + &zeroOrOneExpr{ + pos: position{line: 74, col: 70, offset: 2047}, + expr: &ruleRefExpr{ + pos: position{line: 74, col: 70, offset: 2047}, + name: "Comment", + }, + }, + }, + }, + &seqExpr{ + pos: position{line: 74, col: 83, offset: 2060}, + exprs: []interface{}{ + &zeroOrOneExpr{ + pos: position{line: 74, col: 83, offset: 2060}, + expr: &ruleRefExpr{ + pos: position{line: 74, col: 83, offset: 2060}, + name: "Comment", + }, + }, + &charClassMatcher{ + pos: position{line: 74, col: 92, offset: 2069}, + val: "[\\r\\n]", + chars: []rune{'\r', '\n'}, + ignoreCase: false, + inverted: false, + }, + }, + }, + }, + }, + }, + }, + }, + { + name: "NonWhitespaceLiteralSeparator", + pos: position{line: 76, col: 1, offset: 2079}, + expr: &litMatcher{ + pos: position{line: 76, col: 34, offset: 2112}, + val: ";", + ignoreCase: false, + }, + }, + { + name: "Literal", + pos: position{line: 78, col: 1, offset: 2117}, + expr: &choiceExpr{ + pos: position{line: 78, col: 12, offset: 2128}, + alternatives: []interface{}{ + &ruleRefExpr{ + pos: position{line: 78, col: 12, offset: 2128}, + name: "TermExpr", + }, + &ruleRefExpr{ + pos: position{line: 78, col: 23, offset: 2139}, + name: "SomeDecl", + }, + }, + }, + }, + { + name: "SomeDecl", + pos: position{line: 80, col: 1, offset: 2149}, + expr: &actionExpr{ + pos: position{line: 80, col: 13, offset: 2161}, + run: (*parser).callonSomeDecl1, + expr: &seqExpr{ + pos: position{line: 80, col: 13, offset: 2161}, + exprs: []interface{}{ + &litMatcher{ + pos: position{line: 80, col: 13, offset: 2161}, + val: "some", + ignoreCase: false, + }, + &ruleRefExpr{ + pos: position{line: 80, col: 20, offset: 2168}, + name: "ws", + }, + &labeledExpr{ + pos: position{line: 80, col: 23, offset: 2171}, + label: "symbols", + expr: &ruleRefExpr{ + pos: position{line: 80, col: 31, offset: 2179}, + name: "SomeDeclList", + }, + }, + }, + }, + }, + }, + { + name: "SomeDeclList", + pos: position{line: 84, col: 1, offset: 2257}, + expr: &actionExpr{ + pos: position{line: 84, col: 17, offset: 2273}, + run: (*parser).callonSomeDeclList1, + expr: &seqExpr{ + pos: position{line: 84, col: 17, offset: 2273}, + exprs: []interface{}{ + &labeledExpr{ + pos: position{line: 84, col: 17, offset: 2273}, + label: "head", + expr: &ruleRefExpr{ + pos: position{line: 84, col: 22, offset: 2278}, + name: "Var", + }, + }, + &labeledExpr{ + pos: position{line: 84, col: 26, offset: 2282}, + label: "rest", + expr: &zeroOrMoreExpr{ + pos: position{line: 84, col: 31, offset: 2287}, + expr: &seqExpr{ + pos: position{line: 84, col: 33, offset: 2289}, + exprs: []interface{}{ + &ruleRefExpr{ + pos: position{line: 84, col: 33, offset: 2289}, + name: "_", + }, + &litMatcher{ + pos: position{line: 84, col: 35, offset: 2291}, + val: ",", + ignoreCase: false, + }, + &ruleRefExpr{ + pos: position{line: 84, col: 39, offset: 2295}, + name: "_", + }, + &ruleRefExpr{ + pos: position{line: 84, col: 41, offset: 2297}, + name: "Var", + }, + }, + }, + }, + }, + }, + }, + }, + }, + { + name: "TermExpr", + pos: position{line: 88, col: 1, offset: 2351}, + expr: &actionExpr{ + pos: position{line: 88, col: 13, offset: 2363}, + run: (*parser).callonTermExpr1, + expr: &seqExpr{ + pos: position{line: 88, col: 13, offset: 2363}, + exprs: []interface{}{ + &labeledExpr{ + pos: position{line: 88, col: 13, offset: 2363}, + label: "negated", + expr: &zeroOrOneExpr{ + pos: position{line: 88, col: 21, offset: 2371}, + expr: &ruleRefExpr{ + pos: position{line: 88, col: 21, offset: 2371}, + name: "NotKeyword", + }, + }, + }, + &labeledExpr{ + pos: position{line: 88, col: 33, offset: 2383}, + label: "value", + expr: &ruleRefExpr{ + pos: position{line: 88, col: 39, offset: 2389}, + name: "LiteralExpr", + }, + }, + &labeledExpr{ + pos: position{line: 88, col: 51, offset: 2401}, + label: "with", + expr: &zeroOrOneExpr{ + pos: position{line: 88, col: 56, offset: 2406}, + expr: &ruleRefExpr{ + pos: position{line: 88, col: 56, offset: 2406}, + name: "WithKeywordList", + }, + }, + }, + }, + }, + }, + }, + { + name: "LiteralExpr", + pos: position{line: 92, col: 1, offset: 2473}, + expr: &actionExpr{ + pos: position{line: 92, col: 16, offset: 2488}, + run: (*parser).callonLiteralExpr1, + expr: &seqExpr{ + pos: position{line: 92, col: 16, offset: 2488}, + exprs: []interface{}{ + &labeledExpr{ + pos: position{line: 92, col: 16, offset: 2488}, + label: "lhs", + expr: &ruleRefExpr{ + pos: position{line: 92, col: 20, offset: 2492}, + name: "ExprTerm", + }, + }, + &labeledExpr{ + pos: position{line: 92, col: 29, offset: 2501}, + label: "rest", + expr: &zeroOrOneExpr{ + pos: position{line: 92, col: 34, offset: 2506}, + expr: &seqExpr{ + pos: position{line: 92, col: 36, offset: 2508}, + exprs: []interface{}{ + &ruleRefExpr{ + pos: position{line: 92, col: 36, offset: 2508}, + name: "_", + }, + &ruleRefExpr{ + pos: position{line: 92, col: 38, offset: 2510}, + name: "LiteralExprOperator", + }, + &ruleRefExpr{ + pos: position{line: 92, col: 58, offset: 2530}, + name: "_", + }, + &ruleRefExpr{ + pos: position{line: 92, col: 60, offset: 2532}, + name: "ExprTerm", + }, + }, + }, + }, + }, + }, + }, + }, + }, + { + name: "LiteralExprOperator", + pos: position{line: 96, col: 1, offset: 2606}, + expr: &actionExpr{ + pos: position{line: 96, col: 24, offset: 2629}, + run: (*parser).callonLiteralExprOperator1, + expr: &labeledExpr{ + pos: position{line: 96, col: 24, offset: 2629}, + label: "val", + expr: &choiceExpr{ + pos: position{line: 96, col: 30, offset: 2635}, + alternatives: []interface{}{ + &litMatcher{ + pos: position{line: 96, col: 30, offset: 2635}, + val: ":=", + ignoreCase: false, + }, + &litMatcher{ + pos: position{line: 96, col: 37, offset: 2642}, + val: "=", + ignoreCase: false, + }, + }, + }, + }, + }, + }, + { + name: "NotKeyword", + pos: position{line: 100, col: 1, offset: 2710}, + expr: &actionExpr{ + pos: position{line: 100, col: 15, offset: 2724}, + run: (*parser).callonNotKeyword1, + expr: &labeledExpr{ + pos: position{line: 100, col: 15, offset: 2724}, + label: "val", + expr: &zeroOrOneExpr{ + pos: position{line: 100, col: 19, offset: 2728}, + expr: &seqExpr{ + pos: position{line: 100, col: 20, offset: 2729}, + exprs: []interface{}{ + &litMatcher{ + pos: position{line: 100, col: 20, offset: 2729}, + val: "not", + ignoreCase: false, + }, + &ruleRefExpr{ + pos: position{line: 100, col: 26, offset: 2735}, + name: "ws", + }, + }, + }, + }, + }, + }, + }, + { + name: "WithKeywordList", + pos: position{line: 104, col: 1, offset: 2772}, + expr: &actionExpr{ + pos: position{line: 104, col: 20, offset: 2791}, + run: (*parser).callonWithKeywordList1, + expr: &seqExpr{ + pos: position{line: 104, col: 20, offset: 2791}, + exprs: []interface{}{ + &ruleRefExpr{ + pos: position{line: 104, col: 20, offset: 2791}, + name: "ws", + }, + &labeledExpr{ + pos: position{line: 104, col: 23, offset: 2794}, + label: "head", + expr: &ruleRefExpr{ + pos: position{line: 104, col: 28, offset: 2799}, + name: "WithKeyword", + }, + }, + &labeledExpr{ + pos: position{line: 104, col: 40, offset: 2811}, + label: "rest", + expr: &zeroOrMoreExpr{ + pos: position{line: 104, col: 45, offset: 2816}, + expr: &seqExpr{ + pos: position{line: 104, col: 47, offset: 2818}, + exprs: []interface{}{ + &ruleRefExpr{ + pos: position{line: 104, col: 47, offset: 2818}, + name: "ws", + }, + &ruleRefExpr{ + pos: position{line: 104, col: 50, offset: 2821}, + name: "WithKeyword", + }, + }, + }, + }, + }, + }, + }, + }, + }, + { + name: "WithKeyword", + pos: position{line: 108, col: 1, offset: 2884}, + expr: &actionExpr{ + pos: position{line: 108, col: 16, offset: 2899}, + run: (*parser).callonWithKeyword1, + expr: &seqExpr{ + pos: position{line: 108, col: 16, offset: 2899}, + exprs: []interface{}{ + &litMatcher{ + pos: position{line: 108, col: 16, offset: 2899}, + val: "with", + ignoreCase: false, + }, + &ruleRefExpr{ + pos: position{line: 108, col: 23, offset: 2906}, + name: "ws", + }, + &labeledExpr{ + pos: position{line: 108, col: 26, offset: 2909}, + label: "target", + expr: &ruleRefExpr{ + pos: position{line: 108, col: 33, offset: 2916}, + name: "ExprTerm", + }, + }, + &ruleRefExpr{ + pos: position{line: 108, col: 42, offset: 2925}, + name: "ws", + }, + &litMatcher{ + pos: position{line: 108, col: 45, offset: 2928}, + val: "as", + ignoreCase: false, + }, + &ruleRefExpr{ + pos: position{line: 108, col: 50, offset: 2933}, + name: "ws", + }, + &labeledExpr{ + pos: position{line: 108, col: 53, offset: 2936}, + label: "value", + expr: &ruleRefExpr{ + pos: position{line: 108, col: 59, offset: 2942}, + name: "ExprTerm", + }, + }, + }, + }, + }, + }, + { + name: "ExprTerm", + pos: position{line: 112, col: 1, offset: 3018}, + expr: &actionExpr{ + pos: position{line: 112, col: 13, offset: 3030}, + run: (*parser).callonExprTerm1, + expr: &seqExpr{ + pos: position{line: 112, col: 13, offset: 3030}, + exprs: []interface{}{ + &labeledExpr{ + pos: position{line: 112, col: 13, offset: 3030}, + label: "lhs", + expr: &ruleRefExpr{ + pos: position{line: 112, col: 17, offset: 3034}, + name: "RelationExpr", + }, + }, + &labeledExpr{ + pos: position{line: 112, col: 30, offset: 3047}, + label: "rest", + expr: &zeroOrMoreExpr{ + pos: position{line: 112, col: 35, offset: 3052}, + expr: &seqExpr{ + pos: position{line: 112, col: 37, offset: 3054}, + exprs: []interface{}{ + &ruleRefExpr{ + pos: position{line: 112, col: 37, offset: 3054}, + name: "_", + }, + &ruleRefExpr{ + pos: position{line: 112, col: 39, offset: 3056}, + name: "RelationOperator", + }, + &ruleRefExpr{ + pos: position{line: 112, col: 56, offset: 3073}, + name: "_", + }, + &ruleRefExpr{ + pos: position{line: 112, col: 58, offset: 3075}, + name: "RelationExpr", + }, + }, + }, + }, + }, + }, + }, + }, + }, + { + name: "ExprTermPairList", + pos: position{line: 116, col: 1, offset: 3151}, + expr: &actionExpr{ + pos: position{line: 116, col: 21, offset: 3171}, + run: (*parser).callonExprTermPairList1, + expr: &seqExpr{ + pos: position{line: 116, col: 21, offset: 3171}, + exprs: []interface{}{ + &labeledExpr{ + pos: position{line: 116, col: 21, offset: 3171}, + label: "head", + expr: &zeroOrOneExpr{ + pos: position{line: 116, col: 26, offset: 3176}, + expr: &ruleRefExpr{ + pos: position{line: 116, col: 26, offset: 3176}, + name: "ExprTermPair", + }, + }, + }, + &labeledExpr{ + pos: position{line: 116, col: 40, offset: 3190}, + label: "tail", + expr: &zeroOrMoreExpr{ + pos: position{line: 116, col: 45, offset: 3195}, + expr: &seqExpr{ + pos: position{line: 116, col: 47, offset: 3197}, + exprs: []interface{}{ + &ruleRefExpr{ + pos: position{line: 116, col: 47, offset: 3197}, + name: "_", + }, + &litMatcher{ + pos: position{line: 116, col: 49, offset: 3199}, + val: ",", + ignoreCase: false, + }, + &ruleRefExpr{ + pos: position{line: 116, col: 53, offset: 3203}, + name: "_", + }, + &ruleRefExpr{ + pos: position{line: 116, col: 55, offset: 3205}, + name: "ExprTermPair", + }, + }, + }, + }, + }, + &ruleRefExpr{ + pos: position{line: 116, col: 71, offset: 3221}, + name: "_", + }, + &zeroOrOneExpr{ + pos: position{line: 116, col: 73, offset: 3223}, + expr: &litMatcher{ + pos: position{line: 116, col: 73, offset: 3223}, + val: ",", + ignoreCase: false, + }, + }, + }, + }, + }, + }, + { + name: "ExprTermList", + pos: position{line: 120, col: 1, offset: 3277}, + expr: &actionExpr{ + pos: position{line: 120, col: 17, offset: 3293}, + run: (*parser).callonExprTermList1, + expr: &seqExpr{ + pos: position{line: 120, col: 17, offset: 3293}, + exprs: []interface{}{ + &labeledExpr{ + pos: position{line: 120, col: 17, offset: 3293}, + label: "head", + expr: &zeroOrOneExpr{ + pos: position{line: 120, col: 22, offset: 3298}, + expr: &ruleRefExpr{ + pos: position{line: 120, col: 22, offset: 3298}, + name: "ExprTerm", + }, + }, + }, + &labeledExpr{ + pos: position{line: 120, col: 32, offset: 3308}, + label: "tail", + expr: &zeroOrMoreExpr{ + pos: position{line: 120, col: 37, offset: 3313}, + expr: &seqExpr{ + pos: position{line: 120, col: 39, offset: 3315}, + exprs: []interface{}{ + &ruleRefExpr{ + pos: position{line: 120, col: 39, offset: 3315}, + name: "_", + }, + &litMatcher{ + pos: position{line: 120, col: 41, offset: 3317}, + val: ",", + ignoreCase: false, + }, + &ruleRefExpr{ + pos: position{line: 120, col: 45, offset: 3321}, + name: "_", + }, + &ruleRefExpr{ + pos: position{line: 120, col: 47, offset: 3323}, + name: "ExprTerm", + }, + }, + }, + }, + }, + &ruleRefExpr{ + pos: position{line: 120, col: 59, offset: 3335}, + name: "_", + }, + &zeroOrOneExpr{ + pos: position{line: 120, col: 61, offset: 3337}, + expr: &litMatcher{ + pos: position{line: 120, col: 61, offset: 3337}, + val: ",", + ignoreCase: false, + }, + }, + }, + }, + }, + }, + { + name: "ExprTermPair", + pos: position{line: 124, col: 1, offset: 3388}, + expr: &actionExpr{ + pos: position{line: 124, col: 17, offset: 3404}, + run: (*parser).callonExprTermPair1, + expr: &seqExpr{ + pos: position{line: 124, col: 17, offset: 3404}, + exprs: []interface{}{ + &labeledExpr{ + pos: position{line: 124, col: 17, offset: 3404}, + label: "key", + expr: &ruleRefExpr{ + pos: position{line: 124, col: 21, offset: 3408}, + name: "ExprTerm", + }, + }, + &ruleRefExpr{ + pos: position{line: 124, col: 30, offset: 3417}, + name: "_", + }, + &litMatcher{ + pos: position{line: 124, col: 32, offset: 3419}, + val: ":", + ignoreCase: false, + }, + &ruleRefExpr{ + pos: position{line: 124, col: 36, offset: 3423}, + name: "_", + }, + &labeledExpr{ + pos: position{line: 124, col: 38, offset: 3425}, + label: "value", + expr: &ruleRefExpr{ + pos: position{line: 124, col: 44, offset: 3431}, + name: "ExprTerm", + }, + }, + }, + }, + }, + }, + { + name: "RelationOperator", + pos: position{line: 128, col: 1, offset: 3485}, + expr: &actionExpr{ + pos: position{line: 128, col: 21, offset: 3505}, + run: (*parser).callonRelationOperator1, + expr: &labeledExpr{ + pos: position{line: 128, col: 21, offset: 3505}, + label: "val", + expr: &choiceExpr{ + pos: position{line: 128, col: 26, offset: 3510}, + alternatives: []interface{}{ + &litMatcher{ + pos: position{line: 128, col: 26, offset: 3510}, + val: "==", + ignoreCase: false, + }, + &litMatcher{ + pos: position{line: 128, col: 33, offset: 3517}, + val: "!=", + ignoreCase: false, + }, + &litMatcher{ + pos: position{line: 128, col: 40, offset: 3524}, + val: "<=", + ignoreCase: false, + }, + &litMatcher{ + pos: position{line: 128, col: 47, offset: 3531}, + val: ">=", + ignoreCase: false, + }, + &litMatcher{ + pos: position{line: 128, col: 54, offset: 3538}, + val: ">", + ignoreCase: false, + }, + &litMatcher{ + pos: position{line: 128, col: 60, offset: 3544}, + val: "<", + ignoreCase: false, + }, + }, + }, + }, + }, + }, + { + name: "RelationExpr", + pos: position{line: 132, col: 1, offset: 3611}, + expr: &actionExpr{ + pos: position{line: 132, col: 17, offset: 3627}, + run: (*parser).callonRelationExpr1, + expr: &seqExpr{ + pos: position{line: 132, col: 17, offset: 3627}, + exprs: []interface{}{ + &labeledExpr{ + pos: position{line: 132, col: 17, offset: 3627}, + label: "lhs", + expr: &ruleRefExpr{ + pos: position{line: 132, col: 21, offset: 3631}, + name: "BitwiseOrExpr", + }, + }, + &labeledExpr{ + pos: position{line: 132, col: 35, offset: 3645}, + label: "rest", + expr: &zeroOrMoreExpr{ + pos: position{line: 132, col: 40, offset: 3650}, + expr: &seqExpr{ + pos: position{line: 132, col: 42, offset: 3652}, + exprs: []interface{}{ + &ruleRefExpr{ + pos: position{line: 132, col: 42, offset: 3652}, + name: "_", + }, + &ruleRefExpr{ + pos: position{line: 132, col: 44, offset: 3654}, + name: "BitwiseOrOperator", + }, + &ruleRefExpr{ + pos: position{line: 132, col: 62, offset: 3672}, + name: "_", + }, + &ruleRefExpr{ + pos: position{line: 132, col: 64, offset: 3674}, + name: "BitwiseOrExpr", + }, + }, + }, + }, + }, + }, + }, + }, + }, + { + name: "BitwiseOrOperator", + pos: position{line: 136, col: 1, offset: 3750}, + expr: &actionExpr{ + pos: position{line: 136, col: 22, offset: 3771}, + run: (*parser).callonBitwiseOrOperator1, + expr: &labeledExpr{ + pos: position{line: 136, col: 22, offset: 3771}, + label: "val", + expr: &litMatcher{ + pos: position{line: 136, col: 26, offset: 3775}, + val: "|", + ignoreCase: false, + }, + }, + }, + }, + { + name: "BitwiseOrExpr", + pos: position{line: 140, col: 1, offset: 3841}, + expr: &actionExpr{ + pos: position{line: 140, col: 18, offset: 3858}, + run: (*parser).callonBitwiseOrExpr1, + expr: &seqExpr{ + pos: position{line: 140, col: 18, offset: 3858}, + exprs: []interface{}{ + &labeledExpr{ + pos: position{line: 140, col: 18, offset: 3858}, + label: "lhs", + expr: &ruleRefExpr{ + pos: position{line: 140, col: 22, offset: 3862}, + name: "BitwiseAndExpr", + }, + }, + &labeledExpr{ + pos: position{line: 140, col: 37, offset: 3877}, + label: "rest", + expr: &zeroOrMoreExpr{ + pos: position{line: 140, col: 42, offset: 3882}, + expr: &seqExpr{ + pos: position{line: 140, col: 44, offset: 3884}, + exprs: []interface{}{ + &ruleRefExpr{ + pos: position{line: 140, col: 44, offset: 3884}, + name: "_", + }, + &ruleRefExpr{ + pos: position{line: 140, col: 46, offset: 3886}, + name: "BitwiseAndOperator", + }, + &ruleRefExpr{ + pos: position{line: 140, col: 65, offset: 3905}, + name: "_", + }, + &ruleRefExpr{ + pos: position{line: 140, col: 67, offset: 3907}, + name: "BitwiseAndExpr", + }, + }, + }, + }, + }, + }, + }, + }, + }, + { + name: "BitwiseAndOperator", + pos: position{line: 144, col: 1, offset: 3984}, + expr: &actionExpr{ + pos: position{line: 144, col: 23, offset: 4006}, + run: (*parser).callonBitwiseAndOperator1, + expr: &labeledExpr{ + pos: position{line: 144, col: 23, offset: 4006}, + label: "val", + expr: &litMatcher{ + pos: position{line: 144, col: 27, offset: 4010}, + val: "&", + ignoreCase: false, + }, + }, + }, + }, + { + name: "BitwiseAndExpr", + pos: position{line: 148, col: 1, offset: 4076}, + expr: &actionExpr{ + pos: position{line: 148, col: 19, offset: 4094}, + run: (*parser).callonBitwiseAndExpr1, + expr: &seqExpr{ + pos: position{line: 148, col: 19, offset: 4094}, + exprs: []interface{}{ + &labeledExpr{ + pos: position{line: 148, col: 19, offset: 4094}, + label: "lhs", + expr: &ruleRefExpr{ + pos: position{line: 148, col: 23, offset: 4098}, + name: "ArithExpr", + }, + }, + &labeledExpr{ + pos: position{line: 148, col: 33, offset: 4108}, + label: "rest", + expr: &zeroOrMoreExpr{ + pos: position{line: 148, col: 38, offset: 4113}, + expr: &seqExpr{ + pos: position{line: 148, col: 40, offset: 4115}, + exprs: []interface{}{ + &ruleRefExpr{ + pos: position{line: 148, col: 40, offset: 4115}, + name: "_", + }, + &ruleRefExpr{ + pos: position{line: 148, col: 42, offset: 4117}, + name: "ArithOperator", + }, + &ruleRefExpr{ + pos: position{line: 148, col: 56, offset: 4131}, + name: "_", + }, + &ruleRefExpr{ + pos: position{line: 148, col: 58, offset: 4133}, + name: "ArithExpr", + }, + }, + }, + }, + }, + }, + }, + }, + }, + { + name: "ArithOperator", + pos: position{line: 152, col: 1, offset: 4205}, + expr: &actionExpr{ + pos: position{line: 152, col: 18, offset: 4222}, + run: (*parser).callonArithOperator1, + expr: &labeledExpr{ + pos: position{line: 152, col: 18, offset: 4222}, + label: "val", + expr: &choiceExpr{ + pos: position{line: 152, col: 23, offset: 4227}, + alternatives: []interface{}{ + &litMatcher{ + pos: position{line: 152, col: 23, offset: 4227}, + val: "+", + ignoreCase: false, + }, + &litMatcher{ + pos: position{line: 152, col: 29, offset: 4233}, + val: "-", + ignoreCase: false, + }, + }, + }, + }, + }, + }, + { + name: "ArithExpr", + pos: position{line: 156, col: 1, offset: 4300}, + expr: &actionExpr{ + pos: position{line: 156, col: 14, offset: 4313}, + run: (*parser).callonArithExpr1, + expr: &seqExpr{ + pos: position{line: 156, col: 14, offset: 4313}, + exprs: []interface{}{ + &labeledExpr{ + pos: position{line: 156, col: 14, offset: 4313}, + label: "lhs", + expr: &ruleRefExpr{ + pos: position{line: 156, col: 18, offset: 4317}, + name: "FactorExpr", + }, + }, + &labeledExpr{ + pos: position{line: 156, col: 29, offset: 4328}, + label: "rest", + expr: &zeroOrMoreExpr{ + pos: position{line: 156, col: 34, offset: 4333}, + expr: &seqExpr{ + pos: position{line: 156, col: 36, offset: 4335}, + exprs: []interface{}{ + &ruleRefExpr{ + pos: position{line: 156, col: 36, offset: 4335}, + name: "_", + }, + &ruleRefExpr{ + pos: position{line: 156, col: 38, offset: 4337}, + name: "FactorOperator", + }, + &ruleRefExpr{ + pos: position{line: 156, col: 53, offset: 4352}, + name: "_", + }, + &ruleRefExpr{ + pos: position{line: 156, col: 55, offset: 4354}, + name: "FactorExpr", + }, + }, + }, + }, + }, + }, + }, + }, + }, + { + name: "FactorOperator", + pos: position{line: 160, col: 1, offset: 4428}, + expr: &actionExpr{ + pos: position{line: 160, col: 19, offset: 4446}, + run: (*parser).callonFactorOperator1, + expr: &labeledExpr{ + pos: position{line: 160, col: 19, offset: 4446}, + label: "val", + expr: &choiceExpr{ + pos: position{line: 160, col: 24, offset: 4451}, + alternatives: []interface{}{ + &litMatcher{ + pos: position{line: 160, col: 24, offset: 4451}, + val: "*", + ignoreCase: false, + }, + &litMatcher{ + pos: position{line: 160, col: 30, offset: 4457}, + val: "/", + ignoreCase: false, + }, + &litMatcher{ + pos: position{line: 160, col: 36, offset: 4463}, + val: "%", + ignoreCase: false, + }, + }, + }, + }, + }, + }, + { + name: "FactorExpr", + pos: position{line: 164, col: 1, offset: 4529}, + expr: &choiceExpr{ + pos: position{line: 164, col: 15, offset: 4543}, + alternatives: []interface{}{ + &actionExpr{ + pos: position{line: 164, col: 15, offset: 4543}, + run: (*parser).callonFactorExpr2, + expr: &seqExpr{ + pos: position{line: 164, col: 17, offset: 4545}, + exprs: []interface{}{ + &litMatcher{ + pos: position{line: 164, col: 17, offset: 4545}, + val: "(", + ignoreCase: false, + }, + &ruleRefExpr{ + pos: position{line: 164, col: 21, offset: 4549}, + name: "_", + }, + &labeledExpr{ + pos: position{line: 164, col: 23, offset: 4551}, + label: "expr", + expr: &ruleRefExpr{ + pos: position{line: 164, col: 28, offset: 4556}, + name: "ExprTerm", + }, + }, + &ruleRefExpr{ + pos: position{line: 164, col: 37, offset: 4565}, + name: "_", + }, + &litMatcher{ + pos: position{line: 164, col: 39, offset: 4567}, + val: ")", + ignoreCase: false, + }, + }, + }, + }, + &actionExpr{ + pos: position{line: 166, col: 5, offset: 4600}, + run: (*parser).callonFactorExpr10, + expr: &labeledExpr{ + pos: position{line: 166, col: 5, offset: 4600}, + label: "term", + expr: &ruleRefExpr{ + pos: position{line: 166, col: 10, offset: 4605}, + name: "Term", + }, + }, + }, + }, + }, + }, + { + name: "Call", + pos: position{line: 170, col: 1, offset: 4636}, + expr: &actionExpr{ + pos: position{line: 170, col: 9, offset: 4644}, + run: (*parser).callonCall1, + expr: &seqExpr{ + pos: position{line: 170, col: 9, offset: 4644}, + exprs: []interface{}{ + &labeledExpr{ + pos: position{line: 170, col: 9, offset: 4644}, + label: "operator", + expr: &choiceExpr{ + pos: position{line: 170, col: 19, offset: 4654}, + alternatives: []interface{}{ + &ruleRefExpr{ + pos: position{line: 170, col: 19, offset: 4654}, + name: "Ref", + }, + &ruleRefExpr{ + pos: position{line: 170, col: 25, offset: 4660}, + name: "Var", + }, + }, + }, + }, + &litMatcher{ + pos: position{line: 170, col: 30, offset: 4665}, + val: "(", + ignoreCase: false, + }, + &ruleRefExpr{ + pos: position{line: 170, col: 34, offset: 4669}, + name: "_", + }, + &labeledExpr{ + pos: position{line: 170, col: 36, offset: 4671}, + label: "args", + expr: &ruleRefExpr{ + pos: position{line: 170, col: 41, offset: 4676}, + name: "ExprTermList", + }, + }, + &ruleRefExpr{ + pos: position{line: 170, col: 54, offset: 4689}, + name: "_", + }, + &litMatcher{ + pos: position{line: 170, col: 56, offset: 4691}, + val: ")", + ignoreCase: false, + }, + }, + }, + }, + }, + { + name: "Term", + pos: position{line: 174, col: 1, offset: 4756}, + expr: &actionExpr{ + pos: position{line: 174, col: 9, offset: 4764}, + run: (*parser).callonTerm1, + expr: &seqExpr{ + pos: position{line: 174, col: 9, offset: 4764}, + exprs: []interface{}{ + &labeledExpr{ + pos: position{line: 174, col: 9, offset: 4764}, + label: "val", + expr: &choiceExpr{ + pos: position{line: 174, col: 15, offset: 4770}, + alternatives: []interface{}{ + &ruleRefExpr{ + pos: position{line: 174, col: 15, offset: 4770}, + name: "Comprehension", + }, + &ruleRefExpr{ + pos: position{line: 174, col: 31, offset: 4786}, + name: "Composite", + }, + &ruleRefExpr{ + pos: position{line: 174, col: 43, offset: 4798}, + name: "Scalar", + }, + &ruleRefExpr{ + pos: position{line: 174, col: 52, offset: 4807}, + name: "Call", + }, + &ruleRefExpr{ + pos: position{line: 174, col: 59, offset: 4814}, + name: "Var", + }, + }, + }, + }, + &labeledExpr{ + pos: position{line: 174, col: 65, offset: 4820}, + label: "refs", + expr: &zeroOrMoreExpr{ + pos: position{line: 174, col: 70, offset: 4825}, + expr: &ruleRefExpr{ + pos: position{line: 174, col: 70, offset: 4825}, + name: "RefOperand", + }, + }, + }, + }, + }, + }, + }, + { + name: "TermPair", + pos: position{line: 178, col: 1, offset: 4892}, + expr: &actionExpr{ + pos: position{line: 178, col: 13, offset: 4904}, + run: (*parser).callonTermPair1, + expr: &seqExpr{ + pos: position{line: 178, col: 13, offset: 4904}, + exprs: []interface{}{ + &labeledExpr{ + pos: position{line: 178, col: 13, offset: 4904}, + label: "key", + expr: &ruleRefExpr{ + pos: position{line: 178, col: 17, offset: 4908}, + name: "Term", + }, + }, + &ruleRefExpr{ + pos: position{line: 178, col: 22, offset: 4913}, + name: "_", + }, + &litMatcher{ + pos: position{line: 178, col: 24, offset: 4915}, + val: ":", + ignoreCase: false, + }, + &ruleRefExpr{ + pos: position{line: 178, col: 28, offset: 4919}, + name: "_", + }, + &labeledExpr{ + pos: position{line: 178, col: 30, offset: 4921}, + label: "value", + expr: &ruleRefExpr{ + pos: position{line: 178, col: 36, offset: 4927}, + name: "Term", + }, + }, + }, + }, + }, + }, + { + name: "Comprehension", + pos: position{line: 182, col: 1, offset: 4977}, + expr: &choiceExpr{ + pos: position{line: 182, col: 18, offset: 4994}, + alternatives: []interface{}{ + &ruleRefExpr{ + pos: position{line: 182, col: 18, offset: 4994}, + name: "ArrayComprehension", + }, + &ruleRefExpr{ + pos: position{line: 182, col: 39, offset: 5015}, + name: "ObjectComprehension", + }, + &ruleRefExpr{ + pos: position{line: 182, col: 61, offset: 5037}, + name: "SetComprehension", + }, + }, + }, + }, + { + name: "ArrayComprehension", + pos: position{line: 184, col: 1, offset: 5055}, + expr: &actionExpr{ + pos: position{line: 184, col: 23, offset: 5077}, + run: (*parser).callonArrayComprehension1, + expr: &seqExpr{ + pos: position{line: 184, col: 23, offset: 5077}, + exprs: []interface{}{ + &litMatcher{ + pos: position{line: 184, col: 23, offset: 5077}, + val: "[", + ignoreCase: false, + }, + &ruleRefExpr{ + pos: position{line: 184, col: 27, offset: 5081}, + name: "_", + }, + &labeledExpr{ + pos: position{line: 184, col: 29, offset: 5083}, + label: "head", + expr: &ruleRefExpr{ + pos: position{line: 184, col: 34, offset: 5088}, + name: "Term", + }, + }, + &ruleRefExpr{ + pos: position{line: 184, col: 39, offset: 5093}, + name: "_", + }, + &litMatcher{ + pos: position{line: 184, col: 41, offset: 5095}, + val: "|", + ignoreCase: false, + }, + &ruleRefExpr{ + pos: position{line: 184, col: 45, offset: 5099}, + name: "_", + }, + &labeledExpr{ + pos: position{line: 184, col: 47, offset: 5101}, + label: "body", + expr: &ruleRefExpr{ + pos: position{line: 184, col: 52, offset: 5106}, + name: "WhitespaceBody", + }, + }, + &ruleRefExpr{ + pos: position{line: 184, col: 67, offset: 5121}, + name: "_", + }, + &litMatcher{ + pos: position{line: 184, col: 69, offset: 5123}, + val: "]", + ignoreCase: false, + }, + }, + }, + }, + }, + { + name: "ObjectComprehension", + pos: position{line: 188, col: 1, offset: 5198}, + expr: &actionExpr{ + pos: position{line: 188, col: 24, offset: 5221}, + run: (*parser).callonObjectComprehension1, + expr: &seqExpr{ + pos: position{line: 188, col: 24, offset: 5221}, + exprs: []interface{}{ + &litMatcher{ + pos: position{line: 188, col: 24, offset: 5221}, + val: "{", + ignoreCase: false, + }, + &ruleRefExpr{ + pos: position{line: 188, col: 28, offset: 5225}, + name: "_", + }, + &labeledExpr{ + pos: position{line: 188, col: 30, offset: 5227}, + label: "head", + expr: &ruleRefExpr{ + pos: position{line: 188, col: 35, offset: 5232}, + name: "TermPair", + }, + }, + &ruleRefExpr{ + pos: position{line: 188, col: 45, offset: 5242}, + name: "_", + }, + &litMatcher{ + pos: position{line: 188, col: 47, offset: 5244}, + val: "|", + ignoreCase: false, + }, + &ruleRefExpr{ + pos: position{line: 188, col: 51, offset: 5248}, + name: "_", + }, + &labeledExpr{ + pos: position{line: 188, col: 53, offset: 5250}, + label: "body", + expr: &ruleRefExpr{ + pos: position{line: 188, col: 58, offset: 5255}, + name: "WhitespaceBody", + }, + }, + &ruleRefExpr{ + pos: position{line: 188, col: 73, offset: 5270}, + name: "_", + }, + &litMatcher{ + pos: position{line: 188, col: 75, offset: 5272}, + val: "}", + ignoreCase: false, + }, + }, + }, + }, + }, + { + name: "SetComprehension", + pos: position{line: 192, col: 1, offset: 5348}, + expr: &actionExpr{ + pos: position{line: 192, col: 21, offset: 5368}, + run: (*parser).callonSetComprehension1, + expr: &seqExpr{ + pos: position{line: 192, col: 21, offset: 5368}, + exprs: []interface{}{ + &litMatcher{ + pos: position{line: 192, col: 21, offset: 5368}, + val: "{", + ignoreCase: false, + }, + &ruleRefExpr{ + pos: position{line: 192, col: 25, offset: 5372}, + name: "_", + }, + &labeledExpr{ + pos: position{line: 192, col: 27, offset: 5374}, + label: "head", + expr: &ruleRefExpr{ + pos: position{line: 192, col: 32, offset: 5379}, + name: "Term", + }, + }, + &ruleRefExpr{ + pos: position{line: 192, col: 37, offset: 5384}, + name: "_", + }, + &litMatcher{ + pos: position{line: 192, col: 39, offset: 5386}, + val: "|", + ignoreCase: false, + }, + &ruleRefExpr{ + pos: position{line: 192, col: 43, offset: 5390}, + name: "_", + }, + &labeledExpr{ + pos: position{line: 192, col: 45, offset: 5392}, + label: "body", + expr: &ruleRefExpr{ + pos: position{line: 192, col: 50, offset: 5397}, + name: "WhitespaceBody", + }, + }, + &ruleRefExpr{ + pos: position{line: 192, col: 65, offset: 5412}, + name: "_", + }, + &litMatcher{ + pos: position{line: 192, col: 67, offset: 5414}, + val: "}", + ignoreCase: false, + }, + }, + }, + }, + }, + { + name: "Composite", + pos: position{line: 196, col: 1, offset: 5487}, + expr: &choiceExpr{ + pos: position{line: 196, col: 14, offset: 5500}, + alternatives: []interface{}{ + &ruleRefExpr{ + pos: position{line: 196, col: 14, offset: 5500}, + name: "Object", + }, + &ruleRefExpr{ + pos: position{line: 196, col: 23, offset: 5509}, + name: "Array", + }, + &ruleRefExpr{ + pos: position{line: 196, col: 31, offset: 5517}, + name: "Set", + }, + }, + }, + }, + { + name: "Scalar", + pos: position{line: 198, col: 1, offset: 5522}, + expr: &choiceExpr{ + pos: position{line: 198, col: 11, offset: 5532}, + alternatives: []interface{}{ + &ruleRefExpr{ + pos: position{line: 198, col: 11, offset: 5532}, + name: "Number", + }, + &ruleRefExpr{ + pos: position{line: 198, col: 20, offset: 5541}, + name: "String", + }, + &ruleRefExpr{ + pos: position{line: 198, col: 29, offset: 5550}, + name: "Bool", + }, + &ruleRefExpr{ + pos: position{line: 198, col: 36, offset: 5557}, + name: "Null", + }, + }, + }, + }, + { + name: "Object", + pos: position{line: 200, col: 1, offset: 5563}, + expr: &actionExpr{ + pos: position{line: 200, col: 11, offset: 5573}, + run: (*parser).callonObject1, + expr: &seqExpr{ + pos: position{line: 200, col: 11, offset: 5573}, + exprs: []interface{}{ + &litMatcher{ + pos: position{line: 200, col: 11, offset: 5573}, + val: "{", + ignoreCase: false, + }, + &ruleRefExpr{ + pos: position{line: 200, col: 15, offset: 5577}, + name: "_", + }, + &labeledExpr{ + pos: position{line: 200, col: 17, offset: 5579}, + label: "list", + expr: &ruleRefExpr{ + pos: position{line: 200, col: 22, offset: 5584}, + name: "ExprTermPairList", + }, + }, + &ruleRefExpr{ + pos: position{line: 200, col: 39, offset: 5601}, + name: "_", + }, + &litMatcher{ + pos: position{line: 200, col: 41, offset: 5603}, + val: "}", + ignoreCase: false, + }, + }, + }, + }, + }, + { + name: "Array", + pos: position{line: 204, col: 1, offset: 5660}, + expr: &actionExpr{ + pos: position{line: 204, col: 10, offset: 5669}, + run: (*parser).callonArray1, + expr: &seqExpr{ + pos: position{line: 204, col: 10, offset: 5669}, + exprs: []interface{}{ + &litMatcher{ + pos: position{line: 204, col: 10, offset: 5669}, + val: "[", + ignoreCase: false, + }, + &ruleRefExpr{ + pos: position{line: 204, col: 14, offset: 5673}, + name: "_", + }, + &labeledExpr{ + pos: position{line: 204, col: 16, offset: 5675}, + label: "list", + expr: &ruleRefExpr{ + pos: position{line: 204, col: 21, offset: 5680}, + name: "ExprTermList", + }, + }, + &ruleRefExpr{ + pos: position{line: 204, col: 34, offset: 5693}, + name: "_", + }, + &litMatcher{ + pos: position{line: 204, col: 36, offset: 5695}, + val: "]", + ignoreCase: false, + }, + }, + }, + }, + }, + { + name: "Set", + pos: position{line: 208, col: 1, offset: 5751}, + expr: &choiceExpr{ + pos: position{line: 208, col: 8, offset: 5758}, + alternatives: []interface{}{ + &ruleRefExpr{ + pos: position{line: 208, col: 8, offset: 5758}, + name: "SetEmpty", + }, + &ruleRefExpr{ + pos: position{line: 208, col: 19, offset: 5769}, + name: "SetNonEmpty", + }, + }, + }, + }, + { + name: "SetEmpty", + pos: position{line: 210, col: 1, offset: 5782}, + expr: &actionExpr{ + pos: position{line: 210, col: 13, offset: 5794}, + run: (*parser).callonSetEmpty1, + expr: &seqExpr{ + pos: position{line: 210, col: 13, offset: 5794}, + exprs: []interface{}{ + &litMatcher{ + pos: position{line: 210, col: 13, offset: 5794}, + val: "set(", + ignoreCase: false, + }, + &ruleRefExpr{ + pos: position{line: 210, col: 20, offset: 5801}, + name: "_", + }, + &litMatcher{ + pos: position{line: 210, col: 22, offset: 5803}, + val: ")", + ignoreCase: false, + }, + }, + }, + }, + }, + { + name: "SetNonEmpty", + pos: position{line: 215, col: 1, offset: 5880}, + expr: &actionExpr{ + pos: position{line: 215, col: 16, offset: 5895}, + run: (*parser).callonSetNonEmpty1, + expr: &seqExpr{ + pos: position{line: 215, col: 16, offset: 5895}, + exprs: []interface{}{ + &litMatcher{ + pos: position{line: 215, col: 16, offset: 5895}, + val: "{", + ignoreCase: false, + }, + &ruleRefExpr{ + pos: position{line: 215, col: 20, offset: 5899}, + name: "_", + }, + &labeledExpr{ + pos: position{line: 215, col: 22, offset: 5901}, + label: "list", + expr: &ruleRefExpr{ + pos: position{line: 215, col: 27, offset: 5906}, + name: "ExprTermList", + }, + }, + &ruleRefExpr{ + pos: position{line: 215, col: 40, offset: 5919}, + name: "_", + }, + &litMatcher{ + pos: position{line: 215, col: 42, offset: 5921}, + val: "}", + ignoreCase: false, + }, + }, + }, + }, + }, + { + name: "Ref", + pos: position{line: 219, col: 1, offset: 5975}, + expr: &actionExpr{ + pos: position{line: 219, col: 8, offset: 5982}, + run: (*parser).callonRef1, + expr: &seqExpr{ + pos: position{line: 219, col: 8, offset: 5982}, + exprs: []interface{}{ + &labeledExpr{ + pos: position{line: 219, col: 8, offset: 5982}, + label: "head", + expr: &choiceExpr{ + pos: position{line: 219, col: 14, offset: 5988}, + alternatives: []interface{}{ + &ruleRefExpr{ + pos: position{line: 219, col: 14, offset: 5988}, + name: "Composite", + }, + &ruleRefExpr{ + pos: position{line: 219, col: 26, offset: 6000}, + name: "Var", + }, + }, + }, + }, + &labeledExpr{ + pos: position{line: 219, col: 31, offset: 6005}, + label: "rest", + expr: &oneOrMoreExpr{ + pos: position{line: 219, col: 36, offset: 6010}, + expr: &ruleRefExpr{ + pos: position{line: 219, col: 36, offset: 6010}, + name: "RefOperand", + }, + }, + }, + }, + }, + }, + }, + { + name: "RefOperand", + pos: position{line: 223, col: 1, offset: 6078}, + expr: &choiceExpr{ + pos: position{line: 223, col: 15, offset: 6092}, + alternatives: []interface{}{ + &ruleRefExpr{ + pos: position{line: 223, col: 15, offset: 6092}, + name: "RefOperandDot", + }, + &ruleRefExpr{ + pos: position{line: 223, col: 31, offset: 6108}, + name: "RefOperandCanonical", + }, + }, + }, + }, + { + name: "RefOperandDot", + pos: position{line: 225, col: 1, offset: 6129}, + expr: &actionExpr{ + pos: position{line: 225, col: 18, offset: 6146}, + run: (*parser).callonRefOperandDot1, + expr: &seqExpr{ + pos: position{line: 225, col: 18, offset: 6146}, + exprs: []interface{}{ + &litMatcher{ + pos: position{line: 225, col: 18, offset: 6146}, + val: ".", + ignoreCase: false, + }, + &labeledExpr{ + pos: position{line: 225, col: 22, offset: 6150}, + label: "val", + expr: &ruleRefExpr{ + pos: position{line: 225, col: 26, offset: 6154}, + name: "Var", + }, + }, + }, + }, + }, + }, + { + name: "RefOperandCanonical", + pos: position{line: 229, col: 1, offset: 6217}, + expr: &actionExpr{ + pos: position{line: 229, col: 24, offset: 6240}, + run: (*parser).callonRefOperandCanonical1, + expr: &seqExpr{ + pos: position{line: 229, col: 24, offset: 6240}, + exprs: []interface{}{ + &litMatcher{ + pos: position{line: 229, col: 24, offset: 6240}, + val: "[", + ignoreCase: false, + }, + &labeledExpr{ + pos: position{line: 229, col: 28, offset: 6244}, + label: "val", + expr: &ruleRefExpr{ + pos: position{line: 229, col: 32, offset: 6248}, + name: "ExprTerm", + }, + }, + &litMatcher{ + pos: position{line: 229, col: 41, offset: 6257}, + val: "]", + ignoreCase: false, + }, + }, + }, + }, + }, + { + name: "Var", + pos: position{line: 233, col: 1, offset: 6286}, + expr: &actionExpr{ + pos: position{line: 233, col: 8, offset: 6293}, + run: (*parser).callonVar1, + expr: &labeledExpr{ + pos: position{line: 233, col: 8, offset: 6293}, + label: "val", + expr: &ruleRefExpr{ + pos: position{line: 233, col: 12, offset: 6297}, + name: "VarChecked", + }, + }, + }, + }, + { + name: "VarChecked", + pos: position{line: 237, col: 1, offset: 6352}, + expr: &seqExpr{ + pos: position{line: 237, col: 15, offset: 6366}, + exprs: []interface{}{ + &labeledExpr{ + pos: position{line: 237, col: 15, offset: 6366}, + label: "val", + expr: &ruleRefExpr{ + pos: position{line: 237, col: 19, offset: 6370}, + name: "VarUnchecked", + }, + }, + ¬CodeExpr{ + pos: position{line: 237, col: 32, offset: 6383}, + run: (*parser).callonVarChecked4, + }, + }, + }, + }, + { + name: "VarUnchecked", + pos: position{line: 241, col: 1, offset: 6448}, + expr: &actionExpr{ + pos: position{line: 241, col: 17, offset: 6464}, + run: (*parser).callonVarUnchecked1, + expr: &seqExpr{ + pos: position{line: 241, col: 17, offset: 6464}, + exprs: []interface{}{ + &ruleRefExpr{ + pos: position{line: 241, col: 17, offset: 6464}, + name: "VarStart", + }, + &zeroOrMoreExpr{ + pos: position{line: 241, col: 26, offset: 6473}, + expr: &ruleRefExpr{ + pos: position{line: 241, col: 26, offset: 6473}, + name: "VarChar", + }, + }, + }, + }, + }, + }, + { + name: "Number", + pos: position{line: 245, col: 1, offset: 6534}, + expr: &actionExpr{ + pos: position{line: 245, col: 11, offset: 6544}, + run: (*parser).callonNumber1, + expr: &seqExpr{ + pos: position{line: 245, col: 11, offset: 6544}, + exprs: []interface{}{ + &zeroOrOneExpr{ + pos: position{line: 245, col: 11, offset: 6544}, + expr: &litMatcher{ + pos: position{line: 245, col: 11, offset: 6544}, + val: "-", + ignoreCase: false, + }, + }, + &choiceExpr{ + pos: position{line: 245, col: 18, offset: 6551}, + alternatives: []interface{}{ + &ruleRefExpr{ + pos: position{line: 245, col: 18, offset: 6551}, + name: "Float", + }, + &ruleRefExpr{ + pos: position{line: 245, col: 26, offset: 6559}, + name: "Integer", + }, + }, + }, + }, + }, + }, + }, + { + name: "Float", + pos: position{line: 249, col: 1, offset: 6624}, + expr: &choiceExpr{ + pos: position{line: 249, col: 10, offset: 6633}, + alternatives: []interface{}{ + &ruleRefExpr{ + pos: position{line: 249, col: 10, offset: 6633}, + name: "ExponentFloat", + }, + &ruleRefExpr{ + pos: position{line: 249, col: 26, offset: 6649}, + name: "PointFloat", + }, + }, + }, + }, + { + name: "ExponentFloat", + pos: position{line: 251, col: 1, offset: 6661}, + expr: &seqExpr{ + pos: position{line: 251, col: 18, offset: 6678}, + exprs: []interface{}{ + &choiceExpr{ + pos: position{line: 251, col: 20, offset: 6680}, + alternatives: []interface{}{ + &ruleRefExpr{ + pos: position{line: 251, col: 20, offset: 6680}, + name: "PointFloat", + }, + &ruleRefExpr{ + pos: position{line: 251, col: 33, offset: 6693}, + name: "Integer", + }, + }, + }, + &ruleRefExpr{ + pos: position{line: 251, col: 43, offset: 6703}, + name: "Exponent", + }, + }, + }, + }, + { + name: "PointFloat", + pos: position{line: 253, col: 1, offset: 6713}, + expr: &seqExpr{ + pos: position{line: 253, col: 15, offset: 6727}, + exprs: []interface{}{ + &zeroOrOneExpr{ + pos: position{line: 253, col: 15, offset: 6727}, + expr: &ruleRefExpr{ + pos: position{line: 253, col: 15, offset: 6727}, + name: "Integer", + }, + }, + &ruleRefExpr{ + pos: position{line: 253, col: 24, offset: 6736}, + name: "Fraction", + }, + }, + }, + }, + { + name: "Fraction", + pos: position{line: 255, col: 1, offset: 6746}, + expr: &seqExpr{ + pos: position{line: 255, col: 13, offset: 6758}, + exprs: []interface{}{ + &litMatcher{ + pos: position{line: 255, col: 13, offset: 6758}, + val: ".", + ignoreCase: false, + }, + &oneOrMoreExpr{ + pos: position{line: 255, col: 17, offset: 6762}, + expr: &ruleRefExpr{ + pos: position{line: 255, col: 17, offset: 6762}, + name: "DecimalDigit", + }, + }, + }, + }, + }, + { + name: "Exponent", + pos: position{line: 257, col: 1, offset: 6777}, + expr: &seqExpr{ + pos: position{line: 257, col: 13, offset: 6789}, + exprs: []interface{}{ + &litMatcher{ + pos: position{line: 257, col: 13, offset: 6789}, + val: "e", + ignoreCase: true, + }, + &zeroOrOneExpr{ + pos: position{line: 257, col: 18, offset: 6794}, + expr: &charClassMatcher{ + pos: position{line: 257, col: 18, offset: 6794}, + val: "[+-]", + chars: []rune{'+', '-'}, + ignoreCase: false, + inverted: false, + }, + }, + &oneOrMoreExpr{ + pos: position{line: 257, col: 24, offset: 6800}, + expr: &ruleRefExpr{ + pos: position{line: 257, col: 24, offset: 6800}, + name: "DecimalDigit", + }, + }, + }, + }, + }, + { + name: "Integer", + pos: position{line: 259, col: 1, offset: 6815}, + expr: &choiceExpr{ + pos: position{line: 259, col: 12, offset: 6826}, + alternatives: []interface{}{ + &litMatcher{ + pos: position{line: 259, col: 12, offset: 6826}, + val: "0", + ignoreCase: false, + }, + &seqExpr{ + pos: position{line: 259, col: 20, offset: 6834}, + exprs: []interface{}{ + &ruleRefExpr{ + pos: position{line: 259, col: 20, offset: 6834}, + name: "NonZeroDecimalDigit", + }, + &zeroOrMoreExpr{ + pos: position{line: 259, col: 40, offset: 6854}, + expr: &ruleRefExpr{ + pos: position{line: 259, col: 40, offset: 6854}, + name: "DecimalDigit", + }, + }, + }, + }, + }, + }, + }, + { + name: "String", + pos: position{line: 261, col: 1, offset: 6871}, + expr: &choiceExpr{ + pos: position{line: 261, col: 11, offset: 6881}, + alternatives: []interface{}{ + &ruleRefExpr{ + pos: position{line: 261, col: 11, offset: 6881}, + name: "QuotedString", + }, + &ruleRefExpr{ + pos: position{line: 261, col: 26, offset: 6896}, + name: "RawString", + }, + }, + }, + }, + { + name: "QuotedString", + pos: position{line: 263, col: 1, offset: 6907}, + expr: &choiceExpr{ + pos: position{line: 263, col: 17, offset: 6923}, + alternatives: []interface{}{ + &actionExpr{ + pos: position{line: 263, col: 17, offset: 6923}, + run: (*parser).callonQuotedString2, + expr: &seqExpr{ + pos: position{line: 263, col: 17, offset: 6923}, + exprs: []interface{}{ + &litMatcher{ + pos: position{line: 263, col: 17, offset: 6923}, + val: "\"", + ignoreCase: false, + }, + &zeroOrMoreExpr{ + pos: position{line: 263, col: 21, offset: 6927}, + expr: &ruleRefExpr{ + pos: position{line: 263, col: 21, offset: 6927}, + name: "Char", + }, + }, + &litMatcher{ + pos: position{line: 263, col: 27, offset: 6933}, + val: "\"", + ignoreCase: false, + }, + }, + }, + }, + &actionExpr{ + pos: position{line: 265, col: 5, offset: 6993}, + run: (*parser).callonQuotedString8, + expr: &seqExpr{ + pos: position{line: 265, col: 5, offset: 6993}, + exprs: []interface{}{ + &litMatcher{ + pos: position{line: 265, col: 5, offset: 6993}, + val: "\"", + ignoreCase: false, + }, + &zeroOrMoreExpr{ + pos: position{line: 265, col: 9, offset: 6997}, + expr: &ruleRefExpr{ + pos: position{line: 265, col: 9, offset: 6997}, + name: "Char", + }, + }, + ¬Expr{ + pos: position{line: 265, col: 15, offset: 7003}, + expr: &litMatcher{ + pos: position{line: 265, col: 16, offset: 7004}, + val: "\"", + ignoreCase: false, + }, + }, + }, + }, + }, + }, + }, + }, + { + name: "RawString", + pos: position{line: 269, col: 1, offset: 7084}, + expr: &actionExpr{ + pos: position{line: 269, col: 14, offset: 7097}, + run: (*parser).callonRawString1, + expr: &seqExpr{ + pos: position{line: 269, col: 14, offset: 7097}, + exprs: []interface{}{ + &litMatcher{ + pos: position{line: 269, col: 14, offset: 7097}, + val: "`", + ignoreCase: false, + }, + &zeroOrMoreExpr{ + pos: position{line: 269, col: 18, offset: 7101}, + expr: &charClassMatcher{ + pos: position{line: 269, col: 18, offset: 7101}, + val: "[^`]", + chars: []rune{'`'}, + ignoreCase: false, + inverted: true, + }, + }, + &litMatcher{ + pos: position{line: 269, col: 24, offset: 7107}, + val: "`", + ignoreCase: false, + }, + }, + }, + }, + }, + { + name: "Bool", + pos: position{line: 273, col: 1, offset: 7169}, + expr: &actionExpr{ + pos: position{line: 273, col: 9, offset: 7177}, + run: (*parser).callonBool1, + expr: &seqExpr{ + pos: position{line: 273, col: 9, offset: 7177}, + exprs: []interface{}{ + &labeledExpr{ + pos: position{line: 273, col: 9, offset: 7177}, + label: "val", + expr: &choiceExpr{ + pos: position{line: 273, col: 14, offset: 7182}, + alternatives: []interface{}{ + &litMatcher{ + pos: position{line: 273, col: 14, offset: 7182}, + val: "true", + ignoreCase: false, + }, + &litMatcher{ + pos: position{line: 273, col: 23, offset: 7191}, + val: "false", + ignoreCase: false, + }, + }, + }, + }, + ¬Expr{ + pos: position{line: 273, col: 32, offset: 7200}, + expr: &ruleRefExpr{ + pos: position{line: 273, col: 33, offset: 7201}, + name: "VarChar", + }, + }, + }, + }, + }, + }, + { + name: "Null", + pos: position{line: 277, col: 1, offset: 7262}, + expr: &actionExpr{ + pos: position{line: 277, col: 9, offset: 7270}, + run: (*parser).callonNull1, + expr: &seqExpr{ + pos: position{line: 277, col: 9, offset: 7270}, + exprs: []interface{}{ + &litMatcher{ + pos: position{line: 277, col: 9, offset: 7270}, + val: "null", + ignoreCase: false, + }, + ¬Expr{ + pos: position{line: 277, col: 16, offset: 7277}, + expr: &ruleRefExpr{ + pos: position{line: 277, col: 17, offset: 7278}, + name: "VarChar", + }, + }, + }, + }, + }, + }, + { + name: "VarStart", + pos: position{line: 281, col: 1, offset: 7331}, + expr: &ruleRefExpr{ + pos: position{line: 281, col: 13, offset: 7343}, + name: "AsciiLetter", + }, + }, + { + name: "VarChar", + pos: position{line: 283, col: 1, offset: 7356}, + expr: &choiceExpr{ + pos: position{line: 283, col: 12, offset: 7367}, + alternatives: []interface{}{ + &ruleRefExpr{ + pos: position{line: 283, col: 12, offset: 7367}, + name: "AsciiLetter", + }, + &ruleRefExpr{ + pos: position{line: 283, col: 26, offset: 7381}, + name: "DecimalDigit", + }, + }, + }, + }, + { + name: "AsciiLetter", + pos: position{line: 285, col: 1, offset: 7395}, + expr: &charClassMatcher{ + pos: position{line: 285, col: 16, offset: 7410}, + val: "[A-Za-z_]", + chars: []rune{'_'}, + ranges: []rune{'A', 'Z', 'a', 'z'}, + ignoreCase: false, + inverted: false, + }, + }, + { + name: "Char", + pos: position{line: 287, col: 1, offset: 7421}, + expr: &choiceExpr{ + pos: position{line: 287, col: 9, offset: 7429}, + alternatives: []interface{}{ + &seqExpr{ + pos: position{line: 287, col: 11, offset: 7431}, + exprs: []interface{}{ + ¬Expr{ + pos: position{line: 287, col: 11, offset: 7431}, + expr: &ruleRefExpr{ + pos: position{line: 287, col: 12, offset: 7432}, + name: "EscapedChar", + }, + }, + &anyMatcher{ + line: 287, col: 24, offset: 7444, + }, + }, + }, + &seqExpr{ + pos: position{line: 287, col: 32, offset: 7452}, + exprs: []interface{}{ + &litMatcher{ + pos: position{line: 287, col: 32, offset: 7452}, + val: "\\", + ignoreCase: false, + }, + &ruleRefExpr{ + pos: position{line: 287, col: 37, offset: 7457}, + name: "EscapeSequence", + }, + }, + }, + }, + }, + }, + { + name: "EscapedChar", + pos: position{line: 289, col: 1, offset: 7475}, + expr: &charClassMatcher{ + pos: position{line: 289, col: 16, offset: 7490}, + val: "[\\x00-\\x1f\"\\\\]", + chars: []rune{'"', '\\'}, + ranges: []rune{'\x00', '\x1f'}, + ignoreCase: false, + inverted: false, + }, + }, + { + name: "EscapeSequence", + pos: position{line: 291, col: 1, offset: 7506}, + expr: &choiceExpr{ + pos: position{line: 291, col: 19, offset: 7524}, + alternatives: []interface{}{ + &ruleRefExpr{ + pos: position{line: 291, col: 19, offset: 7524}, + name: "SingleCharEscape", + }, + &ruleRefExpr{ + pos: position{line: 291, col: 38, offset: 7543}, + name: "UnicodeEscape", + }, + }, + }, + }, + { + name: "SingleCharEscape", + pos: position{line: 293, col: 1, offset: 7558}, + expr: &charClassMatcher{ + pos: position{line: 293, col: 21, offset: 7578}, + val: "[ \" \\\\ / b f n r t ]", + chars: []rune{' ', '"', ' ', '\\', ' ', '/', ' ', 'b', ' ', 'f', ' ', 'n', ' ', 'r', ' ', 't', ' '}, + ignoreCase: false, + inverted: false, + }, + }, + { + name: "UnicodeEscape", + pos: position{line: 295, col: 1, offset: 7600}, + expr: &seqExpr{ + pos: position{line: 295, col: 18, offset: 7617}, + exprs: []interface{}{ + &litMatcher{ + pos: position{line: 295, col: 18, offset: 7617}, + val: "u", + ignoreCase: false, + }, + &ruleRefExpr{ + pos: position{line: 295, col: 22, offset: 7621}, + name: "HexDigit", + }, + &ruleRefExpr{ + pos: position{line: 295, col: 31, offset: 7630}, + name: "HexDigit", + }, + &ruleRefExpr{ + pos: position{line: 295, col: 40, offset: 7639}, + name: "HexDigit", + }, + &ruleRefExpr{ + pos: position{line: 295, col: 49, offset: 7648}, + name: "HexDigit", + }, + }, + }, + }, + { + name: "DecimalDigit", + pos: position{line: 297, col: 1, offset: 7658}, + expr: &charClassMatcher{ + pos: position{line: 297, col: 17, offset: 7674}, + val: "[0-9]", + ranges: []rune{'0', '9'}, + ignoreCase: false, + inverted: false, + }, + }, + { + name: "NonZeroDecimalDigit", + pos: position{line: 299, col: 1, offset: 7681}, + expr: &charClassMatcher{ + pos: position{line: 299, col: 24, offset: 7704}, + val: "[1-9]", + ranges: []rune{'1', '9'}, + ignoreCase: false, + inverted: false, + }, + }, + { + name: "HexDigit", + pos: position{line: 301, col: 1, offset: 7711}, + expr: &charClassMatcher{ + pos: position{line: 301, col: 13, offset: 7723}, + val: "[0-9a-fA-F]", + ranges: []rune{'0', '9', 'a', 'f', 'A', 'F'}, + ignoreCase: false, + inverted: false, + }, + }, + { + name: "ws", + displayName: "\"whitespace\"", + pos: position{line: 303, col: 1, offset: 7736}, + expr: &oneOrMoreExpr{ + pos: position{line: 303, col: 20, offset: 7755}, + expr: &charClassMatcher{ + pos: position{line: 303, col: 20, offset: 7755}, + val: "[ \\t\\r\\n]", + chars: []rune{' ', '\t', '\r', '\n'}, + ignoreCase: false, + inverted: false, + }, + }, + }, + { + name: "_", + displayName: "\"whitespace\"", + pos: position{line: 305, col: 1, offset: 7767}, + expr: &zeroOrMoreExpr{ + pos: position{line: 305, col: 19, offset: 7785}, + expr: &choiceExpr{ + pos: position{line: 305, col: 21, offset: 7787}, + alternatives: []interface{}{ + &charClassMatcher{ + pos: position{line: 305, col: 21, offset: 7787}, + val: "[ \\t\\r\\n]", + chars: []rune{' ', '\t', '\r', '\n'}, + ignoreCase: false, + inverted: false, + }, + &ruleRefExpr{ + pos: position{line: 305, col: 33, offset: 7799}, + name: "Comment", + }, + }, + }, + }, + }, + { + name: "Comment", + pos: position{line: 307, col: 1, offset: 7811}, + expr: &actionExpr{ + pos: position{line: 307, col: 12, offset: 7822}, + run: (*parser).callonComment1, + expr: &seqExpr{ + pos: position{line: 307, col: 12, offset: 7822}, + exprs: []interface{}{ + &zeroOrMoreExpr{ + pos: position{line: 307, col: 12, offset: 7822}, + expr: &charClassMatcher{ + pos: position{line: 307, col: 12, offset: 7822}, + val: "[ \\t]", + chars: []rune{' ', '\t'}, + ignoreCase: false, + inverted: false, + }, + }, + &litMatcher{ + pos: position{line: 307, col: 19, offset: 7829}, + val: "#", + ignoreCase: false, + }, + &labeledExpr{ + pos: position{line: 307, col: 23, offset: 7833}, + label: "text", + expr: &zeroOrMoreExpr{ + pos: position{line: 307, col: 28, offset: 7838}, + expr: &charClassMatcher{ + pos: position{line: 307, col: 28, offset: 7838}, + val: "[^\\r\\n]", + chars: []rune{'\r', '\n'}, + ignoreCase: false, + inverted: true, + }, + }, + }, + }, + }, + }, + }, + { + name: "EOF", + pos: position{line: 311, col: 1, offset: 7885}, + expr: ¬Expr{ + pos: position{line: 311, col: 8, offset: 7892}, + expr: &anyMatcher{ + line: 311, col: 9, offset: 7893, + }, + }, + }, + }, +} + +func (c *current) onProgram1(vals interface{}) (interface{}, error) { + return makeProgram(c, vals) +} + +func (p *parser) callonProgram1() (interface{}, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onProgram1(stack["vals"]) +} + +func (c *current) onStmt1(val interface{}) (interface{}, error) { + return val, nil +} + +func (p *parser) callonStmt1() (interface{}, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onStmt1(stack["val"]) +} + +func (c *current) onPackage1(val interface{}) (interface{}, error) { + return makePackage(currentLocation(c), val) +} + +func (p *parser) callonPackage1() (interface{}, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onPackage1(stack["val"]) +} + +func (c *current) onImport1(path, alias interface{}) (interface{}, error) { + return makeImport(currentLocation(c), path, alias) +} + +func (p *parser) callonImport1() (interface{}, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onImport1(stack["path"], stack["alias"]) +} + +func (c *current) onDefaultRules1(name, operator, value interface{}) (interface{}, error) { + return makeDefaultRule(currentLocation(c), name, operator, value) +} + +func (p *parser) callonDefaultRules1() (interface{}, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onDefaultRules1(stack["name"], stack["operator"], stack["value"]) +} + +func (c *current) onNormalRules1(head, rest interface{}) (interface{}, error) { + return makeRule(currentLocation(c), head, rest) +} + +func (p *parser) callonNormalRules1() (interface{}, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onNormalRules1(stack["head"], stack["rest"]) +} + +func (c *current) onPartialRuleHead1(name, args, value interface{}) (interface{}, error) { + return makeRuleHead(currentLocation(c), name, args, nil, value) +} + +func (p *parser) callonPartialRuleHead1() (interface{}, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onPartialRuleHead1(stack["name"], stack["args"], stack["value"]) +} + +func (c *current) onRuleHead1(name, key, value interface{}) (interface{}, error) { + return makeRuleHead(currentLocation(c), name, nil, key, value) +} + +func (p *parser) callonRuleHead1() (interface{}, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onRuleHead1(stack["name"], stack["key"], stack["value"]) +} + +func (c *current) onArgs1(list interface{}) (interface{}, error) { + return makeArgs(list) +} + +func (p *parser) callonArgs1() (interface{}, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onArgs1(stack["list"]) +} + +func (c *current) onElse1(value, body interface{}) (interface{}, error) { + return makeRuleExt(currentLocation(c), value, body) +} + +func (p *parser) callonElse1() (interface{}, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onElse1(stack["value"], stack["body"]) +} + +func (c *current) onRuleDup1(b interface{}) (interface{}, error) { + return ruleExt{loc: currentLocation(c), body: b.(Body)}, nil +} + +func (p *parser) callonRuleDup1() (interface{}, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onRuleDup1(stack["b"]) +} + +func (c *current) onNonEmptyBraceEnclosedBody1(val interface{}) (interface{}, error) { + if val == nil { + return NewBody(), fmt.Errorf("found empty body") + } + return val, nil +} + +func (p *parser) callonNonEmptyBraceEnclosedBody1() (interface{}, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onNonEmptyBraceEnclosedBody1(stack["val"]) +} + +func (c *current) onBraceEnclosedBody1(val interface{}) (interface{}, error) { + return makeBraceEnclosedBody(currentLocation(c), val) +} + +func (p *parser) callonBraceEnclosedBody1() (interface{}, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onBraceEnclosedBody1(stack["val"]) +} + +func (c *current) onWhitespaceBody1(head, tail interface{}) (interface{}, error) { + return makeBody(head, tail, 2) +} + +func (p *parser) callonWhitespaceBody1() (interface{}, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onWhitespaceBody1(stack["head"], stack["tail"]) +} + +func (c *current) onNonWhitespaceBody1(head, tail interface{}) (interface{}, error) { + return makeBody(head, tail, 3) +} + +func (p *parser) callonNonWhitespaceBody1() (interface{}, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onNonWhitespaceBody1(stack["head"], stack["tail"]) +} + +func (c *current) onSomeDecl1(symbols interface{}) (interface{}, error) { + return makeSomeDeclLiteral(currentLocation(c), symbols) +} + +func (p *parser) callonSomeDecl1() (interface{}, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onSomeDecl1(stack["symbols"]) +} + +func (c *current) onSomeDeclList1(head, rest interface{}) (interface{}, error) { + return makeSomeDeclSymbols(head, rest) +} + +func (p *parser) callonSomeDeclList1() (interface{}, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onSomeDeclList1(stack["head"], stack["rest"]) +} + +func (c *current) onTermExpr1(negated, value, with interface{}) (interface{}, error) { + return makeLiteral(negated, value, with) +} + +func (p *parser) callonTermExpr1() (interface{}, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onTermExpr1(stack["negated"], stack["value"], stack["with"]) +} + +func (c *current) onLiteralExpr1(lhs, rest interface{}) (interface{}, error) { + return makeLiteralExpr(currentLocation(c), lhs, rest) +} + +func (p *parser) callonLiteralExpr1() (interface{}, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onLiteralExpr1(stack["lhs"], stack["rest"]) +} + +func (c *current) onLiteralExprOperator1(val interface{}) (interface{}, error) { + return makeInfixOperator(currentLocation(c), c.text) +} + +func (p *parser) callonLiteralExprOperator1() (interface{}, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onLiteralExprOperator1(stack["val"]) +} + +func (c *current) onNotKeyword1(val interface{}) (interface{}, error) { + return val != nil, nil +} + +func (p *parser) callonNotKeyword1() (interface{}, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onNotKeyword1(stack["val"]) +} + +func (c *current) onWithKeywordList1(head, rest interface{}) (interface{}, error) { + return makeWithKeywordList(head, rest) +} + +func (p *parser) callonWithKeywordList1() (interface{}, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onWithKeywordList1(stack["head"], stack["rest"]) +} + +func (c *current) onWithKeyword1(target, value interface{}) (interface{}, error) { + return makeWithKeyword(currentLocation(c), target, value) +} + +func (p *parser) callonWithKeyword1() (interface{}, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onWithKeyword1(stack["target"], stack["value"]) +} + +func (c *current) onExprTerm1(lhs, rest interface{}) (interface{}, error) { + return makeExprTerm(currentLocation(c), lhs, rest) +} + +func (p *parser) callonExprTerm1() (interface{}, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onExprTerm1(stack["lhs"], stack["rest"]) +} + +func (c *current) onExprTermPairList1(head, tail interface{}) (interface{}, error) { + return makeExprTermPairList(head, tail) +} + +func (p *parser) callonExprTermPairList1() (interface{}, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onExprTermPairList1(stack["head"], stack["tail"]) +} + +func (c *current) onExprTermList1(head, tail interface{}) (interface{}, error) { + return makeExprTermList(head, tail) +} + +func (p *parser) callonExprTermList1() (interface{}, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onExprTermList1(stack["head"], stack["tail"]) +} + +func (c *current) onExprTermPair1(key, value interface{}) (interface{}, error) { + return makeExprTermPair(key, value) +} + +func (p *parser) callonExprTermPair1() (interface{}, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onExprTermPair1(stack["key"], stack["value"]) +} + +func (c *current) onRelationOperator1(val interface{}) (interface{}, error) { + return makeInfixOperator(currentLocation(c), c.text) +} + +func (p *parser) callonRelationOperator1() (interface{}, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onRelationOperator1(stack["val"]) +} + +func (c *current) onRelationExpr1(lhs, rest interface{}) (interface{}, error) { + return makeExprTerm(currentLocation(c), lhs, rest) +} + +func (p *parser) callonRelationExpr1() (interface{}, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onRelationExpr1(stack["lhs"], stack["rest"]) +} + +func (c *current) onBitwiseOrOperator1(val interface{}) (interface{}, error) { + return makeInfixOperator(currentLocation(c), c.text) +} + +func (p *parser) callonBitwiseOrOperator1() (interface{}, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onBitwiseOrOperator1(stack["val"]) +} + +func (c *current) onBitwiseOrExpr1(lhs, rest interface{}) (interface{}, error) { + return makeExprTerm(currentLocation(c), lhs, rest) +} + +func (p *parser) callonBitwiseOrExpr1() (interface{}, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onBitwiseOrExpr1(stack["lhs"], stack["rest"]) +} + +func (c *current) onBitwiseAndOperator1(val interface{}) (interface{}, error) { + return makeInfixOperator(currentLocation(c), c.text) +} + +func (p *parser) callonBitwiseAndOperator1() (interface{}, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onBitwiseAndOperator1(stack["val"]) +} + +func (c *current) onBitwiseAndExpr1(lhs, rest interface{}) (interface{}, error) { + return makeExprTerm(currentLocation(c), lhs, rest) +} + +func (p *parser) callonBitwiseAndExpr1() (interface{}, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onBitwiseAndExpr1(stack["lhs"], stack["rest"]) +} + +func (c *current) onArithOperator1(val interface{}) (interface{}, error) { + return makeInfixOperator(currentLocation(c), c.text) +} + +func (p *parser) callonArithOperator1() (interface{}, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onArithOperator1(stack["val"]) +} + +func (c *current) onArithExpr1(lhs, rest interface{}) (interface{}, error) { + return makeExprTerm(currentLocation(c), lhs, rest) +} + +func (p *parser) callonArithExpr1() (interface{}, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onArithExpr1(stack["lhs"], stack["rest"]) +} + +func (c *current) onFactorOperator1(val interface{}) (interface{}, error) { + return makeInfixOperator(currentLocation(c), c.text) +} + +func (p *parser) callonFactorOperator1() (interface{}, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onFactorOperator1(stack["val"]) +} + +func (c *current) onFactorExpr2(expr interface{}) (interface{}, error) { + return expr, nil +} + +func (p *parser) callonFactorExpr2() (interface{}, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onFactorExpr2(stack["expr"]) +} + +func (c *current) onFactorExpr10(term interface{}) (interface{}, error) { + return term, nil +} + +func (p *parser) callonFactorExpr10() (interface{}, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onFactorExpr10(stack["term"]) +} + +func (c *current) onCall1(operator, args interface{}) (interface{}, error) { + return makeCall(currentLocation(c), operator, args) +} + +func (p *parser) callonCall1() (interface{}, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onCall1(stack["operator"], stack["args"]) +} + +func (c *current) onTerm1(val, refs interface{}) (interface{}, error) { + return makeRef(currentLocation(c), val, refs) +} + +func (p *parser) callonTerm1() (interface{}, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onTerm1(stack["val"], stack["refs"]) +} + +func (c *current) onTermPair1(key, value interface{}) (interface{}, error) { + return makeExprTermPair(key, value) +} + +func (p *parser) callonTermPair1() (interface{}, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onTermPair1(stack["key"], stack["value"]) +} + +func (c *current) onArrayComprehension1(head, body interface{}) (interface{}, error) { + return makeArrayComprehension(currentLocation(c), head, body) +} + +func (p *parser) callonArrayComprehension1() (interface{}, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onArrayComprehension1(stack["head"], stack["body"]) +} + +func (c *current) onObjectComprehension1(head, body interface{}) (interface{}, error) { + return makeObjectComprehension(currentLocation(c), head, body) +} + +func (p *parser) callonObjectComprehension1() (interface{}, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onObjectComprehension1(stack["head"], stack["body"]) +} + +func (c *current) onSetComprehension1(head, body interface{}) (interface{}, error) { + return makeSetComprehension(currentLocation(c), head, body) +} + +func (p *parser) callonSetComprehension1() (interface{}, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onSetComprehension1(stack["head"], stack["body"]) +} + +func (c *current) onObject1(list interface{}) (interface{}, error) { + return makeObject(currentLocation(c), list) +} + +func (p *parser) callonObject1() (interface{}, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onObject1(stack["list"]) +} + +func (c *current) onArray1(list interface{}) (interface{}, error) { + return makeArray(currentLocation(c), list) +} + +func (p *parser) callonArray1() (interface{}, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onArray1(stack["list"]) +} + +func (c *current) onSetEmpty1() (interface{}, error) { + var empty []*Term + return makeSet(currentLocation(c), empty) +} + +func (p *parser) callonSetEmpty1() (interface{}, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onSetEmpty1() +} + +func (c *current) onSetNonEmpty1(list interface{}) (interface{}, error) { + return makeSet(currentLocation(c), list) +} + +func (p *parser) callonSetNonEmpty1() (interface{}, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onSetNonEmpty1(stack["list"]) +} + +func (c *current) onRef1(head, rest interface{}) (interface{}, error) { + return makeRef(currentLocation(c), head, rest) +} + +func (p *parser) callonRef1() (interface{}, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onRef1(stack["head"], stack["rest"]) +} + +func (c *current) onRefOperandDot1(val interface{}) (interface{}, error) { + return makeRefOperandDot(currentLocation(c), val) +} + +func (p *parser) callonRefOperandDot1() (interface{}, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onRefOperandDot1(stack["val"]) +} + +func (c *current) onRefOperandCanonical1(val interface{}) (interface{}, error) { + return val, nil +} + +func (p *parser) callonRefOperandCanonical1() (interface{}, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onRefOperandCanonical1(stack["val"]) +} + +func (c *current) onVar1(val interface{}) (interface{}, error) { + return val.([]interface{})[0], nil +} + +func (p *parser) callonVar1() (interface{}, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onVar1(stack["val"]) +} + +func (c *current) onVarChecked4(val interface{}) (bool, error) { + return IsKeyword(string(val.(*Term).Value.(Var))), nil +} + +func (p *parser) callonVarChecked4() (bool, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onVarChecked4(stack["val"]) +} + +func (c *current) onVarUnchecked1() (interface{}, error) { + return makeVar(currentLocation(c), c.text) +} + +func (p *parser) callonVarUnchecked1() (interface{}, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onVarUnchecked1() +} + +func (c *current) onNumber1() (interface{}, error) { + return makeNumber(currentLocation(c), c.text) +} + +func (p *parser) callonNumber1() (interface{}, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onNumber1() +} + +func (c *current) onQuotedString2() (interface{}, error) { + return makeString(currentLocation(c), c.text) +} + +func (p *parser) callonQuotedString2() (interface{}, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onQuotedString2() +} + +func (c *current) onQuotedString8() (interface{}, error) { + return makeNonterminatedString(currentLocation(c), string(c.text)) +} + +func (p *parser) callonQuotedString8() (interface{}, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onQuotedString8() +} + +func (c *current) onRawString1() (interface{}, error) { + return makeRawString(currentLocation(c), c.text) +} + +func (p *parser) callonRawString1() (interface{}, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onRawString1() +} + +func (c *current) onBool1(val interface{}) (interface{}, error) { + return makeBool(currentLocation(c), c.text) +} + +func (p *parser) callonBool1() (interface{}, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onBool1(stack["val"]) +} + +func (c *current) onNull1() (interface{}, error) { + return makeNull(currentLocation(c)) +} + +func (p *parser) callonNull1() (interface{}, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onNull1() +} + +func (c *current) onComment1(text interface{}) (interface{}, error) { + return makeComments(c, text) +} + +func (p *parser) callonComment1() (interface{}, error) { + stack := p.vstack[len(p.vstack)-1] + _ = stack + return p.cur.onComment1(stack["text"]) +} + +var ( + // errNoRule is returned when the grammar to parse has no rule. + errNoRule = errors.New("grammar has no rule") + + // errInvalidEntrypoint is returned when the specified entrypoint rule + // does not exit. + errInvalidEntrypoint = errors.New("invalid entrypoint") + + // errInvalidEncoding is returned when the source is not properly + // utf8-encoded. + errInvalidEncoding = errors.New("invalid encoding") + + // errMaxExprCnt is used to signal that the maximum number of + // expressions have been parsed. + errMaxExprCnt = errors.New("max number of expresssions parsed") +) + +// Option is a function that can set an option on the parser. It returns +// the previous setting as an Option. +type Option func(*parser) Option + +// MaxExpressions creates an Option to stop parsing after the provided +// number of expressions have been parsed, if the value is 0 then the parser will +// parse for as many steps as needed (possibly an infinite number). +// +// The default for maxExprCnt is 0. +func MaxExpressions(maxExprCnt uint64) Option { + return func(p *parser) Option { + oldMaxExprCnt := p.maxExprCnt + p.maxExprCnt = maxExprCnt + return MaxExpressions(oldMaxExprCnt) + } +} + +// Entrypoint creates an Option to set the rule name to use as entrypoint. +// The rule name must have been specified in the -alternate-entrypoints +// if generating the parser with the -optimize-grammar flag, otherwise +// it may have been optimized out. Passing an empty string sets the +// entrypoint to the first rule in the grammar. +// +// The default is to start parsing at the first rule in the grammar. +func Entrypoint(ruleName string) Option { + return func(p *parser) Option { + oldEntrypoint := p.entrypoint + p.entrypoint = ruleName + if ruleName == "" { + p.entrypoint = g.rules[0].name + } + return Entrypoint(oldEntrypoint) + } +} + +// Statistics adds a user provided Stats struct to the parser to allow +// the user to process the results after the parsing has finished. +// Also the key for the "no match" counter is set. +// +// Example usage: +// +// input := "input" +// stats := Stats{} +// _, err := Parse("input-file", []byte(input), Statistics(&stats, "no match")) +// if err != nil { +// log.Panicln(err) +// } +// b, err := json.MarshalIndent(stats.ChoiceAltCnt, "", " ") +// if err != nil { +// log.Panicln(err) +// } +// fmt.Println(string(b)) +// +func Statistics(stats *Stats, choiceNoMatch string) Option { + return func(p *parser) Option { + oldStats := p.Stats + p.Stats = stats + oldChoiceNoMatch := p.choiceNoMatch + p.choiceNoMatch = choiceNoMatch + if p.Stats.ChoiceAltCnt == nil { + p.Stats.ChoiceAltCnt = make(map[string]map[string]int) + } + return Statistics(oldStats, oldChoiceNoMatch) + } +} + +// Debug creates an Option to set the debug flag to b. When set to true, +// debugging information is printed to stdout while parsing. +// +// The default is false. +func Debug(b bool) Option { + return func(p *parser) Option { + old := p.debug + p.debug = b + return Debug(old) + } +} + +// Memoize creates an Option to set the memoize flag to b. When set to true, +// the parser will cache all results so each expression is evaluated only +// once. This guarantees linear parsing time even for pathological cases, +// at the expense of more memory and slower times for typical cases. +// +// The default is false. +func Memoize(b bool) Option { + return func(p *parser) Option { + old := p.memoize + p.memoize = b + return Memoize(old) + } +} + +// AllowInvalidUTF8 creates an Option to allow invalid UTF-8 bytes. +// Every invalid UTF-8 byte is treated as a utf8.RuneError (U+FFFD) +// by character class matchers and is matched by the any matcher. +// The returned matched value, c.text and c.offset are NOT affected. +// +// The default is false. +func AllowInvalidUTF8(b bool) Option { + return func(p *parser) Option { + old := p.allowInvalidUTF8 + p.allowInvalidUTF8 = b + return AllowInvalidUTF8(old) + } +} + +// Recover creates an Option to set the recover flag to b. When set to +// true, this causes the parser to recover from panics and convert it +// to an error. Setting it to false can be useful while debugging to +// access the full stack trace. +// +// The default is true. +func Recover(b bool) Option { + return func(p *parser) Option { + old := p.recover + p.recover = b + return Recover(old) + } +} + +// GlobalStore creates an Option to set a key to a certain value in +// the globalStore. +func GlobalStore(key string, value interface{}) Option { + return func(p *parser) Option { + old := p.cur.globalStore[key] + p.cur.globalStore[key] = value + return GlobalStore(key, old) + } +} + +// InitState creates an Option to set a key to a certain value in +// the global "state" store. +func InitState(key string, value interface{}) Option { + return func(p *parser) Option { + old := p.cur.state[key] + p.cur.state[key] = value + return InitState(key, old) + } +} + +// ParseFile parses the file identified by filename. +func ParseFile(filename string, opts ...Option) (i interface{}, err error) { + f, err := os.Open(filename) + if err != nil { + return nil, err + } + defer func() { + if closeErr := f.Close(); closeErr != nil { + err = closeErr + } + }() + return ParseReader(filename, f, opts...) +} + +// ParseReader parses the data from r using filename as information in the +// error messages. +func ParseReader(filename string, r io.Reader, opts ...Option) (interface{}, error) { + b, err := ioutil.ReadAll(r) + if err != nil { + return nil, err + } + + return Parse(filename, b, opts...) +} + +// Parse parses the data from b using filename as information in the +// error messages. +func Parse(filename string, b []byte, opts ...Option) (interface{}, error) { + return newParser(filename, b, opts...).parse(g) +} + +// position records a position in the text. +type position struct { + line, col, offset int +} + +func (p position) String() string { + return fmt.Sprintf("%d:%d [%d]", p.line, p.col, p.offset) +} + +// savepoint stores all state required to go back to this point in the +// parser. +type savepoint struct { + position + rn rune + w int +} + +type current struct { + pos position // start position of the match + text []byte // raw text of the match + + // state is a store for arbitrary key,value pairs that the user wants to be + // tied to the backtracking of the parser. + // This is always rolled back if a parsing rule fails. + state storeDict + + // globalStore is a general store for the user to store arbitrary key-value + // pairs that they need to manage and that they do not want tied to the + // backtracking of the parser. This is only modified by the user and never + // rolled back by the parser. It is always up to the user to keep this in a + // consistent state. + globalStore storeDict +} + +type storeDict map[string]interface{} + +// the AST types... + +type grammar struct { + pos position + rules []*rule +} + +type rule struct { + pos position + name string + displayName string + expr interface{} +} + +type choiceExpr struct { + pos position + alternatives []interface{} +} + +type actionExpr struct { + pos position + expr interface{} + run func(*parser) (interface{}, error) +} + +type recoveryExpr struct { + pos position + expr interface{} + recoverExpr interface{} + failureLabel []string +} + +type seqExpr struct { + pos position + exprs []interface{} +} + +type throwExpr struct { + pos position + label string +} + +type labeledExpr struct { + pos position + label string + expr interface{} +} + +type expr struct { + pos position + expr interface{} +} + +type andExpr expr +type notExpr expr +type zeroOrOneExpr expr +type zeroOrMoreExpr expr +type oneOrMoreExpr expr + +type ruleRefExpr struct { + pos position + name string +} + +type stateCodeExpr struct { + pos position + run func(*parser) error +} + +type andCodeExpr struct { + pos position + run func(*parser) (bool, error) +} + +type notCodeExpr struct { + pos position + run func(*parser) (bool, error) +} + +type litMatcher struct { + pos position + val string + ignoreCase bool +} + +type charClassMatcher struct { + pos position + val string + basicLatinChars [128]bool + chars []rune + ranges []rune + classes []*unicode.RangeTable + ignoreCase bool + inverted bool +} + +type anyMatcher position + +// errList cumulates the errors found by the parser. +type errList []error + +func (e *errList) add(err error) { + *e = append(*e, err) +} + +func (e errList) err() error { + if len(e) == 0 { + return nil + } + e.dedupe() + return e +} + +func (e *errList) dedupe() { + var cleaned []error + set := make(map[string]bool) + for _, err := range *e { + if msg := err.Error(); !set[msg] { + set[msg] = true + cleaned = append(cleaned, err) + } + } + *e = cleaned +} + +func (e errList) Error() string { + switch len(e) { + case 0: + return "" + case 1: + return e[0].Error() + default: + var buf bytes.Buffer + + for i, err := range e { + if i > 0 { + buf.WriteRune('\n') + } + buf.WriteString(err.Error()) + } + return buf.String() + } +} + +// parserError wraps an error with a prefix indicating the rule in which +// the error occurred. The original error is stored in the Inner field. +type parserError struct { + Inner error + pos position + prefix string + expected []string +} + +// Error returns the error message. +func (p *parserError) Error() string { + return p.prefix + ": " + p.Inner.Error() +} + +// newParser creates a parser with the specified input source and options. +func newParser(filename string, b []byte, opts ...Option) *parser { + stats := Stats{ + ChoiceAltCnt: make(map[string]map[string]int), + } + + p := &parser{ + filename: filename, + errs: new(errList), + data: b, + pt: savepoint{position: position{line: 1}}, + recover: true, + cur: current{ + state: make(storeDict), + globalStore: make(storeDict), + }, + maxFailPos: position{col: 1, line: 1}, + maxFailExpected: make([]string, 0, 20), + Stats: &stats, + // start rule is rule [0] unless an alternate entrypoint is specified + entrypoint: g.rules[0].name, + } + p.setOptions(opts) + + if p.maxExprCnt == 0 { + p.maxExprCnt = math.MaxUint64 + } + + return p +} + +// setOptions applies the options to the parser. +func (p *parser) setOptions(opts []Option) { + for _, opt := range opts { + opt(p) + } +} + +type resultTuple struct { + v interface{} + b bool + end savepoint +} + +const choiceNoMatch = -1 + +// Stats stores some statistics, gathered during parsing +type Stats struct { + // ExprCnt counts the number of expressions processed during parsing + // This value is compared to the maximum number of expressions allowed + // (set by the MaxExpressions option). + ExprCnt uint64 + + // ChoiceAltCnt is used to count for each ordered choice expression, + // which alternative is used how may times. + // These numbers allow to optimize the order of the ordered choice expression + // to increase the performance of the parser + // + // The outer key of ChoiceAltCnt is composed of the name of the rule as well + // as the line and the column of the ordered choice. + // The inner key of ChoiceAltCnt is the number (one-based) of the matching alternative. + // For each alternative the number of matches are counted. If an ordered choice does not + // match, a special counter is incremented. The name of this counter is set with + // the parser option Statistics. + // For an alternative to be included in ChoiceAltCnt, it has to match at least once. + ChoiceAltCnt map[string]map[string]int +} + +type parser struct { + filename string + pt savepoint + cur current + + data []byte + errs *errList + + depth int + recover bool + debug bool + + memoize bool + // memoization table for the packrat algorithm: + // map[offset in source] map[expression or rule] {value, match} + memo map[int]map[interface{}]resultTuple + + // rules table, maps the rule identifier to the rule node + rules map[string]*rule + // variables stack, map of label to value + vstack []map[string]interface{} + // rule stack, allows identification of the current rule in errors + rstack []*rule + + // parse fail + maxFailPos position + maxFailExpected []string + maxFailInvertExpected bool + + // max number of expressions to be parsed + maxExprCnt uint64 + // entrypoint for the parser + entrypoint string + + allowInvalidUTF8 bool + + *Stats + + choiceNoMatch string + // recovery expression stack, keeps track of the currently available recovery expression, these are traversed in reverse + recoveryStack []map[string]interface{} +} + +// push a variable set on the vstack. +func (p *parser) pushV() { + if cap(p.vstack) == len(p.vstack) { + // create new empty slot in the stack + p.vstack = append(p.vstack, nil) + } else { + // slice to 1 more + p.vstack = p.vstack[:len(p.vstack)+1] + } + + // get the last args set + m := p.vstack[len(p.vstack)-1] + if m != nil && len(m) == 0 { + // empty map, all good + return + } + + m = make(map[string]interface{}) + p.vstack[len(p.vstack)-1] = m +} + +// pop a variable set from the vstack. +func (p *parser) popV() { + // if the map is not empty, clear it + m := p.vstack[len(p.vstack)-1] + if len(m) > 0 { + // GC that map + p.vstack[len(p.vstack)-1] = nil + } + p.vstack = p.vstack[:len(p.vstack)-1] +} + +// push a recovery expression with its labels to the recoveryStack +func (p *parser) pushRecovery(labels []string, expr interface{}) { + if cap(p.recoveryStack) == len(p.recoveryStack) { + // create new empty slot in the stack + p.recoveryStack = append(p.recoveryStack, nil) + } else { + // slice to 1 more + p.recoveryStack = p.recoveryStack[:len(p.recoveryStack)+1] + } + + m := make(map[string]interface{}, len(labels)) + for _, fl := range labels { + m[fl] = expr + } + p.recoveryStack[len(p.recoveryStack)-1] = m +} + +// pop a recovery expression from the recoveryStack +func (p *parser) popRecovery() { + // GC that map + p.recoveryStack[len(p.recoveryStack)-1] = nil + + p.recoveryStack = p.recoveryStack[:len(p.recoveryStack)-1] +} + +func (p *parser) print(prefix, s string) string { + if !p.debug { + return s + } + + fmt.Printf("%s %d:%d:%d: %s [%#U]\n", + prefix, p.pt.line, p.pt.col, p.pt.offset, s, p.pt.rn) + return s +} + +func (p *parser) in(s string) string { + p.depth++ + return p.print(strings.Repeat(" ", p.depth)+">", s) +} + +func (p *parser) out(s string) string { + p.depth-- + return p.print(strings.Repeat(" ", p.depth)+"<", s) +} + +func (p *parser) addErr(err error) { + p.addErrAt(err, p.pt.position, []string{}) +} + +func (p *parser) addErrAt(err error, pos position, expected []string) { + var buf bytes.Buffer + if p.filename != "" { + buf.WriteString(p.filename) + } + if buf.Len() > 0 { + buf.WriteString(":") + } + buf.WriteString(fmt.Sprintf("%d:%d (%d)", pos.line, pos.col, pos.offset)) + if len(p.rstack) > 0 { + if buf.Len() > 0 { + buf.WriteString(": ") + } + rule := p.rstack[len(p.rstack)-1] + if rule.displayName != "" { + buf.WriteString("rule " + rule.displayName) + } else { + buf.WriteString("rule " + rule.name) + } + } + pe := &parserError{Inner: err, pos: pos, prefix: buf.String(), expected: expected} + p.errs.add(pe) +} + +func (p *parser) failAt(fail bool, pos position, want string) { + // process fail if parsing fails and not inverted or parsing succeeds and invert is set + if fail == p.maxFailInvertExpected { + if pos.offset < p.maxFailPos.offset { + return + } + + if pos.offset > p.maxFailPos.offset { + p.maxFailPos = pos + p.maxFailExpected = p.maxFailExpected[:0] + } + + if p.maxFailInvertExpected { + want = "!" + want + } + p.maxFailExpected = append(p.maxFailExpected, want) + } +} + +// read advances the parser to the next rune. +func (p *parser) read() { + p.pt.offset += p.pt.w + rn, n := utf8.DecodeRune(p.data[p.pt.offset:]) + p.pt.rn = rn + p.pt.w = n + p.pt.col++ + if rn == '\n' { + p.pt.line++ + p.pt.col = 0 + } + + if rn == utf8.RuneError && n == 1 { // see utf8.DecodeRune + if !p.allowInvalidUTF8 { + p.addErr(errInvalidEncoding) + } + } +} + +// restore parser position to the savepoint pt. +func (p *parser) restore(pt savepoint) { + if p.debug { + defer p.out(p.in("restore")) + } + if pt.offset == p.pt.offset { + return + } + p.pt = pt +} + +// Cloner is implemented by any value that has a Clone method, which returns a +// copy of the value. This is mainly used for types which are not passed by +// value (e.g map, slice, chan) or structs that contain such types. +// +// This is used in conjunction with the global state feature to create proper +// copies of the state to allow the parser to properly restore the state in +// the case of backtracking. +type Cloner interface { + Clone() interface{} +} + +// clone and return parser current state. +func (p *parser) cloneState() storeDict { + if p.debug { + defer p.out(p.in("cloneState")) + } + + state := make(storeDict, len(p.cur.state)) + for k, v := range p.cur.state { + if c, ok := v.(Cloner); ok { + state[k] = c.Clone() + } else { + state[k] = v + } + } + return state +} + +// restore parser current state to the state storeDict. +// every restoreState should applied only one time for every cloned state +func (p *parser) restoreState(state storeDict) { + if p.debug { + defer p.out(p.in("restoreState")) + } + p.cur.state = state +} + +// get the slice of bytes from the savepoint start to the current position. +func (p *parser) sliceFrom(start savepoint) []byte { + return p.data[start.position.offset:p.pt.position.offset] +} + +func (p *parser) getMemoized(node interface{}) (resultTuple, bool) { + if len(p.memo) == 0 { + return resultTuple{}, false + } + m := p.memo[p.pt.offset] + if len(m) == 0 { + return resultTuple{}, false + } + res, ok := m[node] + return res, ok +} + +func (p *parser) setMemoized(pt savepoint, node interface{}, tuple resultTuple) { + if p.memo == nil { + p.memo = make(map[int]map[interface{}]resultTuple) + } + m := p.memo[pt.offset] + if m == nil { + m = make(map[interface{}]resultTuple) + p.memo[pt.offset] = m + } + m[node] = tuple +} + +func (p *parser) buildRulesTable(g *grammar) { + p.rules = make(map[string]*rule, len(g.rules)) + for _, r := range g.rules { + p.rules[r.name] = r + } +} + +func (p *parser) parse(g *grammar) (val interface{}, err error) { + if len(g.rules) == 0 { + p.addErr(errNoRule) + return nil, p.errs.err() + } + + // TODO : not super critical but this could be generated + p.buildRulesTable(g) + + if p.recover { + // panic can be used in action code to stop parsing immediately + // and return the panic as an error. + defer func() { + if e := recover(); e != nil { + if p.debug { + defer p.out(p.in("panic handler")) + } + val = nil + switch e := e.(type) { + case error: + p.addErr(e) + default: + p.addErr(fmt.Errorf("%v", e)) + } + err = p.errs.err() + } + }() + } + + startRule, ok := p.rules[p.entrypoint] + if !ok { + p.addErr(errInvalidEntrypoint) + return nil, p.errs.err() + } + + p.read() // advance to first rune + val, ok = p.parseRule(startRule) + if !ok { + if len(*p.errs) == 0 { + // If parsing fails, but no errors have been recorded, the expected values + // for the farthest parser position are returned as error. + maxFailExpectedMap := make(map[string]struct{}, len(p.maxFailExpected)) + for _, v := range p.maxFailExpected { + maxFailExpectedMap[v] = struct{}{} + } + expected := make([]string, 0, len(maxFailExpectedMap)) + eof := false + if _, ok := maxFailExpectedMap["!."]; ok { + delete(maxFailExpectedMap, "!.") + eof = true + } + for k := range maxFailExpectedMap { + expected = append(expected, k) + } + sort.Strings(expected) + if eof { + expected = append(expected, "EOF") + } + p.addErrAt(errors.New("no match found, expected: "+listJoin(expected, ", ", "or")), p.maxFailPos, expected) + } + + return nil, p.errs.err() + } + return val, p.errs.err() +} + +func listJoin(list []string, sep string, lastSep string) string { + switch len(list) { + case 0: + return "" + case 1: + return list[0] + default: + return fmt.Sprintf("%s %s %s", strings.Join(list[:len(list)-1], sep), lastSep, list[len(list)-1]) + } +} + +func (p *parser) parseRule(rule *rule) (interface{}, bool) { + if p.debug { + defer p.out(p.in("parseRule " + rule.name)) + } + + if p.memoize { + res, ok := p.getMemoized(rule) + if ok { + p.restore(res.end) + return res.v, res.b + } + } + + start := p.pt + p.rstack = append(p.rstack, rule) + p.pushV() + val, ok := p.parseExpr(rule.expr) + p.popV() + p.rstack = p.rstack[:len(p.rstack)-1] + if ok && p.debug { + p.print(strings.Repeat(" ", p.depth)+"MATCH", string(p.sliceFrom(start))) + } + + if p.memoize { + p.setMemoized(start, rule, resultTuple{val, ok, p.pt}) + } + return val, ok +} + +func (p *parser) parseExpr(expr interface{}) (interface{}, bool) { + var pt savepoint + + if p.memoize { + res, ok := p.getMemoized(expr) + if ok { + p.restore(res.end) + return res.v, res.b + } + pt = p.pt + } + + p.ExprCnt++ + if p.ExprCnt > p.maxExprCnt { + panic(errMaxExprCnt) + } + + var val interface{} + var ok bool + switch expr := expr.(type) { + case *actionExpr: + val, ok = p.parseActionExpr(expr) + case *andCodeExpr: + val, ok = p.parseAndCodeExpr(expr) + case *andExpr: + val, ok = p.parseAndExpr(expr) + case *anyMatcher: + val, ok = p.parseAnyMatcher(expr) + case *charClassMatcher: + val, ok = p.parseCharClassMatcher(expr) + case *choiceExpr: + val, ok = p.parseChoiceExpr(expr) + case *labeledExpr: + val, ok = p.parseLabeledExpr(expr) + case *litMatcher: + val, ok = p.parseLitMatcher(expr) + case *notCodeExpr: + val, ok = p.parseNotCodeExpr(expr) + case *notExpr: + val, ok = p.parseNotExpr(expr) + case *oneOrMoreExpr: + val, ok = p.parseOneOrMoreExpr(expr) + case *recoveryExpr: + val, ok = p.parseRecoveryExpr(expr) + case *ruleRefExpr: + val, ok = p.parseRuleRefExpr(expr) + case *seqExpr: + val, ok = p.parseSeqExpr(expr) + case *stateCodeExpr: + val, ok = p.parseStateCodeExpr(expr) + case *throwExpr: + val, ok = p.parseThrowExpr(expr) + case *zeroOrMoreExpr: + val, ok = p.parseZeroOrMoreExpr(expr) + case *zeroOrOneExpr: + val, ok = p.parseZeroOrOneExpr(expr) + default: + panic(fmt.Sprintf("unknown expression type %T", expr)) + } + if p.memoize { + p.setMemoized(pt, expr, resultTuple{val, ok, p.pt}) + } + return val, ok +} + +func (p *parser) parseActionExpr(act *actionExpr) (interface{}, bool) { + if p.debug { + defer p.out(p.in("parseActionExpr")) + } + + start := p.pt + val, ok := p.parseExpr(act.expr) + if ok { + p.cur.pos = start.position + p.cur.text = p.sliceFrom(start) + state := p.cloneState() + actVal, err := act.run(p) + if err != nil { + p.addErrAt(err, start.position, []string{}) + } + p.restoreState(state) + + val = actVal + } + if ok && p.debug { + p.print(strings.Repeat(" ", p.depth)+"MATCH", string(p.sliceFrom(start))) + } + return val, ok +} + +func (p *parser) parseAndCodeExpr(and *andCodeExpr) (interface{}, bool) { + if p.debug { + defer p.out(p.in("parseAndCodeExpr")) + } + + state := p.cloneState() + + ok, err := and.run(p) + if err != nil { + p.addErr(err) + } + p.restoreState(state) + + return nil, ok +} + +func (p *parser) parseAndExpr(and *andExpr) (interface{}, bool) { + if p.debug { + defer p.out(p.in("parseAndExpr")) + } + + pt := p.pt + state := p.cloneState() + p.pushV() + _, ok := p.parseExpr(and.expr) + p.popV() + p.restoreState(state) + p.restore(pt) + + return nil, ok +} + +func (p *parser) parseAnyMatcher(any *anyMatcher) (interface{}, bool) { + if p.debug { + defer p.out(p.in("parseAnyMatcher")) + } + + if p.pt.rn == utf8.RuneError && p.pt.w == 0 { + // EOF - see utf8.DecodeRune + p.failAt(false, p.pt.position, ".") + return nil, false + } + start := p.pt + p.read() + p.failAt(true, start.position, ".") + return p.sliceFrom(start), true +} + +func (p *parser) parseCharClassMatcher(chr *charClassMatcher) (interface{}, bool) { + if p.debug { + defer p.out(p.in("parseCharClassMatcher")) + } + + cur := p.pt.rn + start := p.pt + + // can't match EOF + if cur == utf8.RuneError && p.pt.w == 0 { // see utf8.DecodeRune + p.failAt(false, start.position, chr.val) + return nil, false + } + + if chr.ignoreCase { + cur = unicode.ToLower(cur) + } + + // try to match in the list of available chars + for _, rn := range chr.chars { + if rn == cur { + if chr.inverted { + p.failAt(false, start.position, chr.val) + return nil, false + } + p.read() + p.failAt(true, start.position, chr.val) + return p.sliceFrom(start), true + } + } + + // try to match in the list of ranges + for i := 0; i < len(chr.ranges); i += 2 { + if cur >= chr.ranges[i] && cur <= chr.ranges[i+1] { + if chr.inverted { + p.failAt(false, start.position, chr.val) + return nil, false + } + p.read() + p.failAt(true, start.position, chr.val) + return p.sliceFrom(start), true + } + } + + // try to match in the list of Unicode classes + for _, cl := range chr.classes { + if unicode.Is(cl, cur) { + if chr.inverted { + p.failAt(false, start.position, chr.val) + return nil, false + } + p.read() + p.failAt(true, start.position, chr.val) + return p.sliceFrom(start), true + } + } + + if chr.inverted { + p.read() + p.failAt(true, start.position, chr.val) + return p.sliceFrom(start), true + } + p.failAt(false, start.position, chr.val) + return nil, false +} + +func (p *parser) incChoiceAltCnt(ch *choiceExpr, altI int) { + choiceIdent := fmt.Sprintf("%s %d:%d", p.rstack[len(p.rstack)-1].name, ch.pos.line, ch.pos.col) + m := p.ChoiceAltCnt[choiceIdent] + if m == nil { + m = make(map[string]int) + p.ChoiceAltCnt[choiceIdent] = m + } + // We increment altI by 1, so the keys do not start at 0 + alt := strconv.Itoa(altI + 1) + if altI == choiceNoMatch { + alt = p.choiceNoMatch + } + m[alt]++ +} + +func (p *parser) parseChoiceExpr(ch *choiceExpr) (interface{}, bool) { + if p.debug { + defer p.out(p.in("parseChoiceExpr")) + } + + for altI, alt := range ch.alternatives { + // dummy assignment to prevent compile error if optimized + _ = altI + + state := p.cloneState() + + p.pushV() + val, ok := p.parseExpr(alt) + p.popV() + if ok { + p.incChoiceAltCnt(ch, altI) + return val, ok + } + p.restoreState(state) + } + p.incChoiceAltCnt(ch, choiceNoMatch) + return nil, false +} + +func (p *parser) parseLabeledExpr(lab *labeledExpr) (interface{}, bool) { + if p.debug { + defer p.out(p.in("parseLabeledExpr")) + } + + p.pushV() + val, ok := p.parseExpr(lab.expr) + p.popV() + if ok && lab.label != "" { + m := p.vstack[len(p.vstack)-1] + m[lab.label] = val + } + return val, ok +} + +func (p *parser) parseLitMatcher(lit *litMatcher) (interface{}, bool) { + if p.debug { + defer p.out(p.in("parseLitMatcher")) + } + + ignoreCase := "" + if lit.ignoreCase { + ignoreCase = "i" + } + val := fmt.Sprintf("%q%s", lit.val, ignoreCase) + start := p.pt + for _, want := range lit.val { + cur := p.pt.rn + if lit.ignoreCase { + cur = unicode.ToLower(cur) + } + if cur != want { + p.failAt(false, start.position, val) + p.restore(start) + return nil, false + } + p.read() + } + p.failAt(true, start.position, val) + return p.sliceFrom(start), true +} + +func (p *parser) parseNotCodeExpr(not *notCodeExpr) (interface{}, bool) { + if p.debug { + defer p.out(p.in("parseNotCodeExpr")) + } + + state := p.cloneState() + + ok, err := not.run(p) + if err != nil { + p.addErr(err) + } + p.restoreState(state) + + return nil, !ok +} + +func (p *parser) parseNotExpr(not *notExpr) (interface{}, bool) { + if p.debug { + defer p.out(p.in("parseNotExpr")) + } + + pt := p.pt + state := p.cloneState() + p.pushV() + p.maxFailInvertExpected = !p.maxFailInvertExpected + _, ok := p.parseExpr(not.expr) + p.maxFailInvertExpected = !p.maxFailInvertExpected + p.popV() + p.restoreState(state) + p.restore(pt) + + return nil, !ok +} + +func (p *parser) parseOneOrMoreExpr(expr *oneOrMoreExpr) (interface{}, bool) { + if p.debug { + defer p.out(p.in("parseOneOrMoreExpr")) + } + + var vals []interface{} + + for { + p.pushV() + val, ok := p.parseExpr(expr.expr) + p.popV() + if !ok { + if len(vals) == 0 { + // did not match once, no match + return nil, false + } + return vals, true + } + vals = append(vals, val) + } +} + +func (p *parser) parseRecoveryExpr(recover *recoveryExpr) (interface{}, bool) { + if p.debug { + defer p.out(p.in("parseRecoveryExpr (" + strings.Join(recover.failureLabel, ",") + ")")) + } + + p.pushRecovery(recover.failureLabel, recover.recoverExpr) + val, ok := p.parseExpr(recover.expr) + p.popRecovery() + + return val, ok +} + +func (p *parser) parseRuleRefExpr(ref *ruleRefExpr) (interface{}, bool) { + if p.debug { + defer p.out(p.in("parseRuleRefExpr " + ref.name)) + } + + if ref.name == "" { + panic(fmt.Sprintf("%s: invalid rule: missing name", ref.pos)) + } + + rule := p.rules[ref.name] + if rule == nil { + p.addErr(fmt.Errorf("undefined rule: %s", ref.name)) + return nil, false + } + return p.parseRule(rule) +} + +func (p *parser) parseSeqExpr(seq *seqExpr) (interface{}, bool) { + if p.debug { + defer p.out(p.in("parseSeqExpr")) + } + + vals := make([]interface{}, 0, len(seq.exprs)) + + pt := p.pt + state := p.cloneState() + for _, expr := range seq.exprs { + val, ok := p.parseExpr(expr) + if !ok { + p.restoreState(state) + p.restore(pt) + return nil, false + } + vals = append(vals, val) + } + return vals, true +} + +func (p *parser) parseStateCodeExpr(state *stateCodeExpr) (interface{}, bool) { + if p.debug { + defer p.out(p.in("parseStateCodeExpr")) + } + + err := state.run(p) + if err != nil { + p.addErr(err) + } + return nil, true +} + +func (p *parser) parseThrowExpr(expr *throwExpr) (interface{}, bool) { + if p.debug { + defer p.out(p.in("parseThrowExpr")) + } + + for i := len(p.recoveryStack) - 1; i >= 0; i-- { + if recoverExpr, ok := p.recoveryStack[i][expr.label]; ok { + if val, ok := p.parseExpr(recoverExpr); ok { + return val, ok + } + } + } + + return nil, false +} + +func (p *parser) parseZeroOrMoreExpr(expr *zeroOrMoreExpr) (interface{}, bool) { + if p.debug { + defer p.out(p.in("parseZeroOrMoreExpr")) + } + + var vals []interface{} + + for { + p.pushV() + val, ok := p.parseExpr(expr.expr) + p.popV() + if !ok { + return vals, true + } + vals = append(vals, val) + } +} + +func (p *parser) parseZeroOrOneExpr(expr *zeroOrOneExpr) (interface{}, bool) { + if p.debug { + defer p.out(p.in("parseZeroOrOneExpr")) + } + + p.pushV() + val, _ := p.parseExpr(expr.expr) + p.popV() + // whether it matched or not, consider it a match + return val, true +} diff --git a/vendor/github.com/open-policy-agent/opa/ast/parser_ext.go b/vendor/github.com/open-policy-agent/opa/ast/parser_ext.go new file mode 100644 index 000000000..03bd20f4d --- /dev/null +++ b/vendor/github.com/open-policy-agent/opa/ast/parser_ext.go @@ -0,0 +1,838 @@ +// Copyright 2016 The OPA Authors. All rights reserved. +// Use of this source code is governed by an Apache2 +// license that can be found in the LICENSE file. + +// This file contains extra functions for parsing Rego. +// Most of the parsing is handled by the auto-generated code in +// parser.go, however, there are additional utilities that are +// helpful for dealing with Rego source inputs (e.g., REPL +// statements, source files, etc.) + +package ast + +import ( + "fmt" + "sort" + "strings" + "unicode" + + "github.com/pkg/errors" +) + +// MustParseBody returns a parsed body. +// If an error occurs during parsing, panic. +func MustParseBody(input string) Body { + parsed, err := ParseBody(input) + if err != nil { + panic(err) + } + return parsed +} + +// MustParseExpr returns a parsed expression. +// If an error occurs during parsing, panic. +func MustParseExpr(input string) *Expr { + parsed, err := ParseExpr(input) + if err != nil { + panic(err) + } + return parsed +} + +// MustParseImports returns a slice of imports. +// If an error occurs during parsing, panic. +func MustParseImports(input string) []*Import { + parsed, err := ParseImports(input) + if err != nil { + panic(err) + } + return parsed +} + +// MustParseModule returns a parsed module. +// If an error occurs during parsing, panic. +func MustParseModule(input string) *Module { + parsed, err := ParseModule("", input) + if err != nil { + panic(err) + } + return parsed +} + +// MustParsePackage returns a Package. +// If an error occurs during parsing, panic. +func MustParsePackage(input string) *Package { + parsed, err := ParsePackage(input) + if err != nil { + panic(err) + } + return parsed +} + +// MustParseStatements returns a slice of parsed statements. +// If an error occurs during parsing, panic. +func MustParseStatements(input string) []Statement { + parsed, _, err := ParseStatements("", input) + if err != nil { + panic(err) + } + return parsed +} + +// MustParseStatement returns exactly one statement. +// If an error occurs during parsing, panic. +func MustParseStatement(input string) Statement { + parsed, err := ParseStatement(input) + if err != nil { + panic(err) + } + return parsed +} + +// MustParseRef returns a parsed reference. +// If an error occurs during parsing, panic. +func MustParseRef(input string) Ref { + parsed, err := ParseRef(input) + if err != nil { + panic(err) + } + return parsed +} + +// MustParseRule returns a parsed rule. +// If an error occurs during parsing, panic. +func MustParseRule(input string) *Rule { + parsed, err := ParseRule(input) + if err != nil { + panic(err) + } + return parsed +} + +// MustParseTerm returns a parsed term. +// If an error occurs during parsing, panic. +func MustParseTerm(input string) *Term { + parsed, err := ParseTerm(input) + if err != nil { + panic(err) + } + return parsed +} + +// ParseRuleFromBody returns a rule if the body can be interpreted as a rule +// definition. Otherwise, an error is returned. +func ParseRuleFromBody(module *Module, body Body) (*Rule, error) { + + if len(body) != 1 { + return nil, fmt.Errorf("multiple expressions cannot be used for rule head") + } + + return ParseRuleFromExpr(module, body[0]) +} + +// ParseRuleFromExpr returns a rule if the expression can be interpreted as a +// rule definition. +func ParseRuleFromExpr(module *Module, expr *Expr) (*Rule, error) { + + if len(expr.With) > 0 { + return nil, fmt.Errorf("expressions using with keyword cannot be used for rule head") + } + + if expr.Negated { + return nil, fmt.Errorf("negated expressions cannot be used for rule head") + } + + if _, ok := expr.Terms.(*SomeDecl); ok { + return nil, errors.New("some declarations cannot be used for rule head") + } + + if term, ok := expr.Terms.(*Term); ok { + switch v := term.Value.(type) { + case Ref: + return ParsePartialSetDocRuleFromTerm(module, term) + default: + return nil, fmt.Errorf("%v cannot be used for rule name", TypeName(v)) + } + } + + if _, ok := expr.Terms.([]*Term); !ok { + // This is a defensive check in case other kinds of expression terms are + // introduced in the future. + return nil, errors.New("expression cannot be used for rule head") + } + + if expr.IsAssignment() { + + lhs, rhs := expr.Operand(0), expr.Operand(1) + rule, err := ParseCompleteDocRuleFromAssignmentExpr(module, lhs, rhs) + + if err == nil { + return rule, nil + } else if _, ok := lhs.Value.(Call); ok { + return nil, errFunctionAssignOperator + } else if _, ok := lhs.Value.(Ref); ok { + return nil, errPartialRuleAssignOperator + } + + return nil, errTermAssignOperator(lhs.Value) + } + + if expr.IsEquality() { + + lhs, rhs := expr.Operand(0), expr.Operand(1) + rule, err := ParseCompleteDocRuleFromEqExpr(module, lhs, rhs) + + if err == nil { + return rule, nil + } + + rule, err = ParseRuleFromCallEqExpr(module, lhs, rhs) + if err == nil { + return rule, nil + } + + return ParsePartialObjectDocRuleFromEqExpr(module, lhs, rhs) + } + + if _, ok := BuiltinMap[expr.Operator().String()]; ok { + return nil, fmt.Errorf("rule name conflicts with built-in function") + } + + return ParseRuleFromCallExpr(module, expr.Terms.([]*Term)) +} + +// ParseCompleteDocRuleFromAssignmentExpr returns a rule if the expression can +// be interpreted as a complete document definition declared with the assignment +// operator. +func ParseCompleteDocRuleFromAssignmentExpr(module *Module, lhs, rhs *Term) (*Rule, error) { + + rule, err := ParseCompleteDocRuleFromEqExpr(module, lhs, rhs) + if err != nil { + return nil, err + } + + rule.Head.Assign = true + + return rule, nil +} + +// ParseCompleteDocRuleFromEqExpr returns a rule if the expression can be +// interpreted as a complete document definition. +func ParseCompleteDocRuleFromEqExpr(module *Module, lhs, rhs *Term) (*Rule, error) { + + var name Var + + if RootDocumentRefs.Contains(lhs) { + name = lhs.Value.(Ref)[0].Value.(Var) + } else if v, ok := lhs.Value.(Var); ok { + name = v + } else { + return nil, fmt.Errorf("%v cannot be used for rule name", TypeName(lhs.Value)) + } + + rule := &Rule{ + Location: rhs.Location, + Head: &Head{ + Location: rhs.Location, + Name: name, + Value: rhs, + }, + Body: NewBody( + NewExpr(BooleanTerm(true).SetLocation(rhs.Location)).SetLocation(rhs.Location), + ), + Module: module, + } + + return rule, nil +} + +// ParsePartialObjectDocRuleFromEqExpr returns a rule if the expression can be +// interpreted as a partial object document definition. +func ParsePartialObjectDocRuleFromEqExpr(module *Module, lhs, rhs *Term) (*Rule, error) { + + ref, ok := lhs.Value.(Ref) + if !ok || len(ref) != 2 { + return nil, fmt.Errorf("%v cannot be used for rule name", TypeName(lhs.Value)) + } + + name := ref[0].Value.(Var) + key := ref[1] + + rule := &Rule{ + Location: rhs.Location, + Head: &Head{ + Location: rhs.Location, + Name: name, + Key: key, + Value: rhs, + }, + Body: NewBody( + NewExpr(BooleanTerm(true).SetLocation(rhs.Location)).SetLocation(rhs.Location), + ), + Module: module, + } + + return rule, nil +} + +// ParsePartialSetDocRuleFromTerm returns a rule if the term can be interpreted +// as a partial set document definition. +func ParsePartialSetDocRuleFromTerm(module *Module, term *Term) (*Rule, error) { + + ref, ok := term.Value.(Ref) + if !ok { + return nil, fmt.Errorf("%vs cannot be used for rule head", TypeName(term.Value)) + } + + if len(ref) != 2 { + return nil, fmt.Errorf("refs cannot be used for rule") + } + + rule := &Rule{ + Location: term.Location, + Head: &Head{ + Location: term.Location, + Name: ref[0].Value.(Var), + Key: ref[1], + }, + Body: NewBody( + NewExpr(BooleanTerm(true).SetLocation(term.Location)).SetLocation(term.Location), + ), + Module: module, + } + + return rule, nil +} + +// ParseRuleFromCallEqExpr returns a rule if the term can be interpreted as a +// function definition (e.g., f(x) = y => f(x) = y { true }). +func ParseRuleFromCallEqExpr(module *Module, lhs, rhs *Term) (*Rule, error) { + + call, ok := lhs.Value.(Call) + if !ok { + return nil, fmt.Errorf("must be call") + } + + rule := &Rule{ + Location: lhs.Location, + Head: &Head{ + Location: lhs.Location, + Name: call[0].Value.(Ref)[0].Value.(Var), + Args: Args(call[1:]), + Value: rhs, + }, + Body: NewBody(NewExpr(BooleanTerm(true).SetLocation(rhs.Location)).SetLocation(rhs.Location)), + Module: module, + } + + return rule, nil +} + +// ParseRuleFromCallExpr returns a rule if the terms can be interpreted as a +// function returning true or some value (e.g., f(x) => f(x) = true { true }). +func ParseRuleFromCallExpr(module *Module, terms []*Term) (*Rule, error) { + + if len(terms) <= 1 { + return nil, fmt.Errorf("rule argument list must take at least one argument") + } + + loc := terms[0].Location + args := terms[1:] + value := BooleanTerm(true).SetLocation(loc) + + rule := &Rule{ + Location: loc, + Head: &Head{ + Location: loc, + Name: Var(terms[0].String()), + Args: args, + Value: value, + }, + Module: module, + Body: NewBody(NewExpr(BooleanTerm(true).SetLocation(loc)).SetLocation(loc)), + } + return rule, nil +} + +// ParseImports returns a slice of Import objects. +func ParseImports(input string) ([]*Import, error) { + stmts, _, err := ParseStatements("", input) + if err != nil { + return nil, err + } + result := []*Import{} + for _, stmt := range stmts { + if imp, ok := stmt.(*Import); ok { + result = append(result, imp) + } else { + return nil, fmt.Errorf("expected import but got %T", stmt) + } + } + return result, nil +} + +// ParseModule returns a parsed Module object. +// For details on Module objects and their fields, see policy.go. +// Empty input will return nil, nil. +func ParseModule(filename, input string) (*Module, error) { + stmts, comments, err := ParseStatements(filename, input) + if err != nil { + return nil, err + } + return parseModule(filename, stmts, comments) +} + +// ParseBody returns exactly one body. +// If multiple bodies are parsed, an error is returned. +func ParseBody(input string) (Body, error) { + stmts, _, err := ParseStatements("", input) + if err != nil { + return nil, err + } + + result := Body{} + + for _, stmt := range stmts { + switch stmt := stmt.(type) { + case Body: + result = append(result, stmt...) + case *Comment: + // skip + default: + return nil, fmt.Errorf("expected body but got %T", stmt) + } + } + + setExprIndices(result) + + return result, nil +} + +// ParseExpr returns exactly one expression. +// If multiple expressions are parsed, an error is returned. +func ParseExpr(input string) (*Expr, error) { + body, err := ParseBody(input) + if err != nil { + return nil, errors.Wrap(err, "failed to parse expression") + } + if len(body) != 1 { + return nil, fmt.Errorf("expected exactly one expression but got: %v", body) + } + return body[0], nil +} + +// ParsePackage returns exactly one Package. +// If multiple statements are parsed, an error is returned. +func ParsePackage(input string) (*Package, error) { + stmt, err := ParseStatement(input) + if err != nil { + return nil, err + } + pkg, ok := stmt.(*Package) + if !ok { + return nil, fmt.Errorf("expected package but got %T", stmt) + } + return pkg, nil +} + +// ParseTerm returns exactly one term. +// If multiple terms are parsed, an error is returned. +func ParseTerm(input string) (*Term, error) { + body, err := ParseBody(input) + if err != nil { + return nil, errors.Wrap(err, "failed to parse term") + } + if len(body) != 1 { + return nil, fmt.Errorf("expected exactly one term but got: %v", body) + } + term, ok := body[0].Terms.(*Term) + if !ok { + return nil, fmt.Errorf("expected term but got %v", body[0].Terms) + } + return term, nil +} + +// ParseRef returns exactly one reference. +func ParseRef(input string) (Ref, error) { + term, err := ParseTerm(input) + if err != nil { + return nil, errors.Wrap(err, "failed to parse ref") + } + ref, ok := term.Value.(Ref) + if !ok { + return nil, fmt.Errorf("expected ref but got %v", term) + } + return ref, nil +} + +// ParseRule returns exactly one rule. +// If multiple rules are parsed, an error is returned. +func ParseRule(input string) (*Rule, error) { + stmts, _, err := ParseStatements("", input) + if err != nil { + return nil, err + } + if len(stmts) != 1 { + return nil, fmt.Errorf("expected exactly one statement (rule)") + } + rule, ok := stmts[0].(*Rule) + if !ok { + return nil, fmt.Errorf("expected rule but got %T", stmts[0]) + } + return rule, nil +} + +// ParseStatement returns exactly one statement. +// A statement might be a term, expression, rule, etc. Regardless, +// this function expects *exactly* one statement. If multiple +// statements are parsed, an error is returned. +func ParseStatement(input string) (Statement, error) { + stmts, _, err := ParseStatements("", input) + if err != nil { + return nil, err + } + if len(stmts) != 1 { + return nil, fmt.Errorf("expected exactly one statement") + } + return stmts[0], nil +} + +// CommentsOption returns a parser option to initialize the comments store within +// the parser. +func CommentsOption() Option { + return GlobalStore(commentsKey, map[commentKey]*Comment{}) +} + +type commentKey struct { + File string + Row int + Col int +} + +func (a commentKey) Compare(other commentKey) int { + if a.File < other.File { + return -1 + } else if a.File > other.File { + return 1 + } else if a.Row < other.Row { + return -1 + } else if a.Row > other.Row { + return 1 + } else if a.Col < other.Col { + return -1 + } else if a.Col > other.Col { + return 1 + } + return 0 +} + +// ParseStatements returns a slice of parsed statements. +// This is the default return value from the parser. +func ParseStatements(filename, input string) ([]Statement, []*Comment, error) { + + bs := []byte(input) + + parsed, err := Parse(filename, bs, GlobalStore(filenameKey, filename), CommentsOption()) + if err != nil { + return nil, nil, formatParserErrors(filename, bs, err) + } + + var comments []*Comment + var sl []interface{} + if p, ok := parsed.(program); ok { + sl = p.buf + commentMap := p.comments.(map[commentKey]*Comment) + commentKeys := []commentKey{} + for k := range commentMap { + commentKeys = append(commentKeys, k) + } + sort.Slice(commentKeys, func(i, j int) bool { + return commentKeys[i].Compare(commentKeys[j]) < 0 + }) + for _, k := range commentKeys { + comments = append(comments, commentMap[k]) + } + } else { + sl = parsed.([]interface{}) + } + stmts := make([]Statement, 0, len(sl)) + + for _, x := range sl { + if rules, ok := x.([]*Rule); ok { + for _, rule := range rules { + stmts = append(stmts, rule) + } + } else { + // Unchecked cast should be safe. A panic indicates grammar is + // out-of-sync. + stmts = append(stmts, x.(Statement)) + } + } + + return stmts, comments, postProcess(filename, stmts) +} + +func formatParserErrors(filename string, bs []byte, err error) error { + // Errors returned by the parser are always of type errList and the errList + // always contains *parserError. + // https://godoc.org/github.com/mna/pigeon#hdr-Error_reporting. + errs := err.(errList) + r := make(Errors, len(errs)) + for i, e := range errs { + r[i] = formatParserError(filename, bs, e.(*parserError)) + } + return r +} + +func formatParserError(filename string, bs []byte, e *parserError) *Error { + loc := NewLocation(nil, filename, e.pos.line, e.pos.col) + inner := e.Inner.Error() + idx := strings.Index(inner, "no match found") + if idx >= 0 { + // Match errors end with "no match found, expected: ...". We do not want to + // include ", expected: ..." as it does not provide any value, so truncate the + // string here. + inner = inner[:idx+14] + } + err := NewError(ParseErr, loc, inner) + err.Details = newParserErrorDetail(bs, e.pos) + return err +} + +func parseModule(filename string, stmts []Statement, comments []*Comment) (*Module, error) { + + if len(stmts) == 0 { + return nil, NewError(ParseErr, &Location{File: filename}, "empty module") + } + + var errs Errors + + _package, ok := stmts[0].(*Package) + if !ok { + loc := stmts[0].(Statement).Loc() + errs = append(errs, NewError(ParseErr, loc, "package expected")) + } + + mod := &Module{ + Package: _package, + } + + // The comments slice only holds comments that were not their own statements. + mod.Comments = append(mod.Comments, comments...) + + for _, stmt := range stmts[1:] { + switch stmt := stmt.(type) { + case *Import: + mod.Imports = append(mod.Imports, stmt) + case *Rule: + setRuleModule(stmt, mod) + mod.Rules = append(mod.Rules, stmt) + case Body: + rule, err := ParseRuleFromBody(mod, stmt) + if err != nil { + errs = append(errs, NewError(ParseErr, stmt[0].Location, err.Error())) + } else { + mod.Rules = append(mod.Rules, rule) + } + case *Package: + errs = append(errs, NewError(ParseErr, stmt.Loc(), "unexpected package")) + case *Comment: // Ignore comments, they're handled above. + default: + panic("illegal value") // Indicates grammar is out-of-sync with code. + } + } + + if len(errs) == 0 { + return mod, nil + } + + return nil, errs +} + +func postProcess(filename string, stmts []Statement) error { + + if err := mangleDataVars(stmts); err != nil { + return err + } + + if err := mangleInputVars(stmts); err != nil { + return err + } + + mangleWildcards(stmts) + mangleExprIndices(stmts) + + return nil +} + +func mangleDataVars(stmts []Statement) error { + for i := range stmts { + vt := newVarToRefTransformer(DefaultRootDocument.Value.(Var), DefaultRootRef.Copy()) + stmt, err := Transform(vt, stmts[i]) + if err != nil { + return err + } + stmts[i] = stmt.(Statement) + } + return nil +} + +func mangleInputVars(stmts []Statement) error { + for i := range stmts { + vt := newVarToRefTransformer(InputRootDocument.Value.(Var), InputRootRef.Copy()) + stmt, err := Transform(vt, stmts[i]) + if err != nil { + return err + } + stmts[i] = stmt.(Statement) + } + return nil +} + +func mangleExprIndices(stmts []Statement) { + for _, stmt := range stmts { + setExprIndices(stmt) + } +} + +func setExprIndices(x interface{}) { + WalkBodies(x, func(b Body) bool { + for i, expr := range b { + expr.Index = i + } + return false + }) +} + +func mangleWildcards(stmts []Statement) { + m := &wildcardMangler{} + for i := range stmts { + stmt, _ := Transform(m, stmts[i]) + stmts[i] = stmt.(Statement) + } +} + +type wildcardMangler struct { + c int +} + +func (m *wildcardMangler) Transform(x interface{}) (interface{}, error) { + if term, ok := x.(Var); ok { + if term.Equal(Wildcard.Value) { + name := fmt.Sprintf("%s%d", WildcardPrefix, m.c) + m.c++ + return Var(name), nil + } + } + return x, nil +} + +func setRuleModule(rule *Rule, module *Module) { + rule.Module = module + if rule.Else != nil { + setRuleModule(rule.Else, module) + } +} + +type varToRefTransformer struct { + orig Var + target Ref + // skip set to true to avoid recursively processing the result of + // transformation. + skip bool +} + +func newVarToRefTransformer(orig Var, target Ref) *varToRefTransformer { + return &varToRefTransformer{ + orig: orig, + target: target, + skip: false, + } +} + +func (vt *varToRefTransformer) Transform(x interface{}) (interface{}, error) { + if vt.skip { + vt.skip = false + return x, nil + } + switch x := x.(type) { + case *Head: + // The next AST node will be the rule name (which should not be + // transformed). + vt.skip = true + case Ref: + // The next AST node will be the ref head (which should not be + // transformed). + vt.skip = true + case Var: + if x.Equal(vt.orig) { + vt.skip = true + return vt.target, nil + } + } + return x, nil +} + +// ParserErrorDetail holds additional details for parser errors. +type ParserErrorDetail struct { + Line string `json:"line"` + Idx int `json:"idx"` +} + +func newParserErrorDetail(bs []byte, pos position) *ParserErrorDetail { + + offset := pos.offset + + // Find first non-space character at or before offset position. + if offset >= len(bs) { + offset = len(bs) - 1 + } else if offset < 0 { + offset = 0 + } + + for offset > 0 && unicode.IsSpace(rune(bs[offset])) { + offset-- + } + + // Find beginning of line containing offset. + begin := offset + + for begin > 0 && !isNewLineChar(bs[begin]) { + begin-- + } + + if isNewLineChar(bs[begin]) { + begin++ + } + + // Find end of line containing offset. + end := offset + + for end < len(bs) && !isNewLineChar(bs[end]) { + end++ + } + + if begin > end { + begin = end + } + + // Extract line and compute index of offset byte in line. + line := bs[begin:end] + index := offset - begin + + return &ParserErrorDetail{ + Line: string(line), + Idx: index, + } +} + +// Lines returns the pretty formatted line output for the error details. +func (d ParserErrorDetail) Lines() []string { + line := strings.TrimLeft(d.Line, "\t") // remove leading tabs + tabCount := len(d.Line) - len(line) + return []string{line, strings.Repeat(" ", d.Idx-tabCount) + "^"} +} + +func isNewLineChar(b byte) bool { + return b == '\r' || b == '\n' +} diff --git a/vendor/github.com/open-policy-agent/opa/ast/parser_internal.go b/vendor/github.com/open-policy-agent/opa/ast/parser_internal.go new file mode 100644 index 000000000..f4e32688d --- /dev/null +++ b/vendor/github.com/open-policy-agent/opa/ast/parser_internal.go @@ -0,0 +1,620 @@ +// Copyright 2018 The OPA Authors. All rights reserved. +// Use of this source code is governed by an Apache2 +// license that can be found in the LICENSE file.op + +package ast + +import ( + "bytes" + "encoding/json" + "fmt" + "math/big" +) + +const ( + // commentsKey is the global map key for the comments slice. + commentsKey = "comments" + + // filenameKey is the global map key for the filename. + filenameKey = "filename" +) + +type program struct { + buf []interface{} + comments interface{} +} + +type ruleExt struct { + loc *Location + term *Term + body Body +} + +// currentLocation converts the parser context to a Location object. +func currentLocation(c *current) *Location { + return NewLocation(c.text, c.globalStore[filenameKey].(string), c.pos.line, c.pos.col) +} + +func makeProgram(c *current, vals interface{}) (interface{}, error) { + var buf []interface{} + if vals == nil { + return buf, nil + } + ifaceSlice := vals.([]interface{}) + head := ifaceSlice[0] + buf = append(buf, head) + for _, tail := range ifaceSlice[1].([]interface{}) { + stmt := tail.([]interface{})[1] + buf = append(buf, stmt) + } + return program{buf, c.globalStore[commentsKey]}, nil +} + +func makePackage(loc *Location, value interface{}) (interface{}, error) { + // All packages are implicitly declared under the default root document. + term := value.(*Term) + path := Ref{DefaultRootDocument.Copy().SetLocation(term.Location)} + switch v := term.Value.(type) { + case Ref: + // Convert head of package Ref to String because it will be prefixed + // with the root document variable. + head := StringTerm(string(v[0].Value.(Var))).SetLocation(v[0].Location) + tail := v[1:] + if !tail.IsGround() { + return nil, fmt.Errorf("package name cannot contain variables: %v", v) + } + + // We do not allow non-string values in package names. + // Because documents are typically represented as JSON, non-string keys are + // not allowed for now. + // TODO(tsandall): consider special syntax for namespacing under arrays. + for _, p := range tail { + _, ok := p.Value.(String) + if !ok { + return nil, fmt.Errorf("package name cannot contain non-string values: %v", v) + } + } + path = append(path, head) + path = append(path, tail...) + case Var: + s := StringTerm(string(v)).SetLocation(term.Location) + path = append(path, s) + } + pkg := &Package{Location: loc, Path: path} + return pkg, nil +} + +func makeImport(loc *Location, path, alias interface{}) (interface{}, error) { + imp := &Import{} + imp.Location = loc + imp.Path = path.(*Term) + if err := IsValidImportPath(imp.Path.Value); err != nil { + return nil, err + } + if alias == nil { + return imp, nil + } + aliasSlice := alias.([]interface{}) + // Import definition above describes the "alias" slice. We only care about the "Var" element. + imp.Alias = aliasSlice[3].(*Term).Value.(Var) + return imp, nil +} + +func makeDefaultRule(loc *Location, name, operator, value interface{}) (interface{}, error) { + + if string(operator.([]uint8)) == Assign.Infix { + return nil, fmt.Errorf("default rules must use = operator (not := operator)") + } + + term := value.(*Term) + var err error + + vis := NewGenericVisitor(func(x interface{}) bool { + if err != nil { + return true + } + switch x.(type) { + case *ArrayComprehension, *ObjectComprehension, *SetComprehension: // skip closures + return true + case Ref, Var: + err = fmt.Errorf("default rule value cannot contain %v", TypeName(x)) + return true + } + return false + }) + + vis.Walk(term) + + if err != nil { + return nil, err + } + + body := NewBody(NewExpr(BooleanTerm(true).SetLocation(loc))) + + rule := &Rule{ + Location: loc, + Default: true, + Head: &Head{ + Location: loc, + Name: name.(*Term).Value.(Var), + Value: value.(*Term), + }, + Body: body, + } + rule.Body[0].Location = loc + + return []*Rule{rule}, nil +} + +func makeRule(loc *Location, head, rest interface{}) (interface{}, error) { + + if head == nil { + return nil, nil + } + + sl := rest.([]interface{}) + + rules := []*Rule{ + { + Location: loc, + Head: head.(*Head), + Body: sl[0].(Body), + }, + } + + var ordered bool + prev := rules[0] + + for i, elem := range sl[1].([]interface{}) { + + next := elem.([]interface{}) + re := next[1].(ruleExt) + + if rules[0].Head.Assign { + return nil, errElseAssignOperator + } + + if re.term == nil { + if ordered { + return nil, fmt.Errorf("expected 'else' keyword") + } + rules = append(rules, &Rule{ + Location: re.loc, + Head: prev.Head.Copy(), + Body: re.body, + }) + } else { + if (rules[0].Head.DocKind() != CompleteDoc) || (i != 0 && !ordered) { + return nil, fmt.Errorf("unexpected 'else' keyword") + } + ordered = true + curr := &Rule{ + Location: re.loc, + Head: &Head{ + Name: prev.Head.Name, + Args: prev.Head.Args.Copy(), + Value: re.term, + Location: re.term.Location, + }, + Body: re.body, + } + prev.Else = curr + prev = curr + } + } + + return rules, nil +} + +func makeRuleHead(loc *Location, name, args, key, value interface{}) (interface{}, error) { + + head := &Head{} + + head.Location = loc + head.Name = name.(*Term).Value.(Var) + + if args != nil && key != nil { + return nil, fmt.Errorf("partial rules cannot take arguments") + } + + if args != nil { + argSlice := args.([]interface{}) + head.Args = argSlice[3].(Args) + } + + if key != nil { + keySlice := key.([]interface{}) + // Head definition above describes the "key" slice. We care about the "Term" element. + head.Key = keySlice[3].(*Term) + } + + if value != nil { + valueSlice := value.([]interface{}) + operator := string(valueSlice[1].([]uint8)) + + if operator == Assign.Infix { + if head.Key != nil { + return nil, errPartialRuleAssignOperator + } else if len(head.Args) > 0 { + return nil, errFunctionAssignOperator + } + head.Assign = true + } + + // Head definition above describes the "value" slice. We care about the "Term" element. + head.Value = valueSlice[len(valueSlice)-1].(*Term) + } + + if key == nil && value == nil { + head.Value = BooleanTerm(true).SetLocation(head.Location) + } + + if key != nil && value != nil { + switch head.Key.Value.(type) { + case Var, String, Ref: // nop + default: + return nil, fmt.Errorf("object key must be string, var, or ref, not %v", TypeName(head.Key.Value)) + } + } + + return head, nil +} + +func makeArgs(list interface{}) (interface{}, error) { + termSlice := list.([]*Term) + args := make(Args, len(termSlice)) + for i := 0; i < len(args); i++ { + args[i] = termSlice[i] + } + return args, nil +} + +func makeRuleExt(loc *Location, val, b interface{}) (interface{}, error) { + bs := b.([]interface{}) + body := bs[1].(Body) + + if val == nil { + term := BooleanTerm(true) + term.Location = loc + return ruleExt{term.Location, term, body}, nil + } + + vs := val.([]interface{}) + t := vs[3].(*Term) + return ruleExt{loc, t, body}, nil +} + +func makeLiteral(negated, value, with interface{}) (interface{}, error) { + + expr := value.(*Expr) + + expr.Negated = negated.(bool) + + if with != nil { + expr.With = with.([]*With) + } + + return expr, nil +} + +func makeLiteralExpr(loc *Location, lhs, rest interface{}) (interface{}, error) { + + if rest == nil { + if call, ok := lhs.(*Term).Value.(Call); ok { + return NewExpr([]*Term(call)).SetLocation(loc), nil + } + return NewExpr(lhs).SetLocation(loc), nil + } + + termSlice := rest.([]interface{}) + terms := []*Term{ + termSlice[1].(*Term), + lhs.(*Term), + termSlice[3].(*Term), + } + + expr := NewExpr(terms).SetLocation(loc) + + return expr, nil +} + +func makeSomeDeclLiteral(loc *Location, sl interface{}) (interface{}, error) { + symbols := sl.([]*Term) + return NewExpr(&SomeDecl{Location: loc, Symbols: symbols}).SetLocation(loc), nil +} + +func makeSomeDeclSymbols(head interface{}, rest interface{}) (interface{}, error) { + + var symbols []*Term + + symbols = append(symbols, head.(*Term)) + + if sl1, ok := rest.([]interface{}); ok { + for i := range sl1 { + if sl2, ok := sl1[i].([]interface{}); ok { + symbols = append(symbols, sl2[3].(*Term)) + } + } + } + + return symbols, nil +} + +func makeWithKeywordList(head, tail interface{}) (interface{}, error) { + var withs []*With + + if head == nil { + return withs, nil + } + + sl := tail.([]interface{}) + + withs = make([]*With, 0, len(sl)+1) + withs = append(withs, head.(*With)) + + for i := range sl { + withSlice := sl[i].([]interface{}) + withs = append(withs, withSlice[1].(*With)) + } + + return withs, nil +} + +func makeWithKeyword(loc *Location, target, value interface{}) (interface{}, error) { + w := &With{ + Target: target.(*Term), + Value: value.(*Term), + } + return w.SetLocation(loc), nil +} + +func makeExprTerm(loc *Location, lhs, rest interface{}) (interface{}, error) { + + if rest == nil { + return lhs, nil + } + + sl := rest.([]interface{}) + + if len(sl) == 0 { + return lhs, nil + } + + for i := range sl { + termSlice := sl[i].([]interface{}) + call := Call{ + termSlice[1].(*Term), + lhs.(*Term), + termSlice[3].(*Term), + } + lhs = NewTerm(call).SetLocation(loc) + } + + return lhs, nil +} + +func makeCall(loc *Location, operator, args interface{}) (interface{}, error) { + + termSlice := args.([]*Term) + termOperator := operator.(*Term) + + call := make(Call, len(termSlice)+1) + + if _, ok := termOperator.Value.(Var); ok { + termOperator = RefTerm(termOperator).SetLocation(loc) + } + + call[0] = termOperator + + for i := 1; i < len(call); i++ { + call[i] = termSlice[i-1] + } + + return NewTerm(call).SetLocation(loc), nil +} + +func makeBraceEnclosedBody(loc *Location, body interface{}) (interface{}, error) { + if body != nil { + return body, nil + } + return NewBody(NewExpr(ObjectTerm().SetLocation(loc)).SetLocation(loc)), nil +} + +func makeBody(head, tail interface{}, pos int) (interface{}, error) { + + sl := tail.([]interface{}) + body := make(Body, len(sl)+1) + body[0] = head.(*Expr) + + for i := 1; i < len(body); i++ { + body[i] = sl[i-1].([]interface{})[pos].(*Expr) + } + + return body, nil +} + +func makeExprTermList(head, tail interface{}) (interface{}, error) { + + var terms []*Term + + if head == nil { + return terms, nil + } + + sl := tail.([]interface{}) + + terms = make([]*Term, 0, len(sl)+1) + terms = append(terms, head.(*Term)) + + for i := range sl { + termSlice := sl[i].([]interface{}) + terms = append(terms, termSlice[3].(*Term)) + } + + return terms, nil +} + +func makeExprTermPairList(head, tail interface{}) (interface{}, error) { + + var terms [][2]*Term + + if head == nil { + return terms, nil + } + + sl := tail.([]interface{}) + + terms = make([][2]*Term, 0, len(sl)+1) + terms = append(terms, head.([2]*Term)) + + for i := range sl { + termSlice := sl[i].([]interface{}) + terms = append(terms, termSlice[3].([2]*Term)) + } + + return terms, nil +} + +func makeExprTermPair(key, value interface{}) (interface{}, error) { + return [2]*Term{key.(*Term), value.(*Term)}, nil +} + +func makeInfixOperator(loc *Location, text []byte) (interface{}, error) { + op := string(text) + for _, b := range Builtins { + if string(b.Infix) == op { + op = string(b.Name) + } + } + operator := RefTerm(VarTerm(op).SetLocation(loc)).SetLocation(loc) + return operator, nil +} + +func makeArray(loc *Location, list interface{}) (interface{}, error) { + termSlice := list.([]*Term) + return ArrayTerm(termSlice...).SetLocation(loc), nil +} + +func makeObject(loc *Location, list interface{}) (interface{}, error) { + termPairSlice := list.([][2]*Term) + return ObjectTerm(termPairSlice...).SetLocation(loc), nil +} + +func makeSet(loc *Location, list interface{}) (interface{}, error) { + termSlice := list.([]*Term) + return SetTerm(termSlice...).SetLocation(loc), nil +} + +func makeArrayComprehension(loc *Location, head, body interface{}) (interface{}, error) { + return ArrayComprehensionTerm(head.(*Term), body.(Body)).SetLocation(loc), nil +} + +func makeSetComprehension(loc *Location, head, body interface{}) (interface{}, error) { + return SetComprehensionTerm(head.(*Term), body.(Body)).SetLocation(loc), nil +} + +func makeObjectComprehension(loc *Location, head, body interface{}) (interface{}, error) { + pair := head.([2]*Term) + return ObjectComprehensionTerm(pair[0], pair[1], body.(Body)).SetLocation(loc), nil +} + +func makeRef(loc *Location, head, rest interface{}) (interface{}, error) { + + headTerm := head.(*Term) + ifaceSlice := rest.([]interface{}) + + if len(ifaceSlice) == 0 { + return headTerm, nil + } + + ref := make(Ref, len(ifaceSlice)+1) + ref[0] = headTerm + + for i := 1; i < len(ref); i++ { + ref[i] = ifaceSlice[i-1].(*Term) + } + + return NewTerm(ref).SetLocation(loc), nil +} + +func makeRefOperandDot(loc *Location, val interface{}) (interface{}, error) { + return StringTerm(string(val.(*Term).Value.(Var))).SetLocation(loc), nil +} + +func makeVar(loc *Location, text interface{}) (interface{}, error) { + str := string(text.([]byte)) + return VarTerm(str).SetLocation(loc), nil +} + +func makeNumber(loc *Location, text interface{}) (interface{}, error) { + f, ok := new(big.Float).SetString(string(text.([]byte))) + if !ok { + // This indicates the grammar is out-of-sync with what the string + // representation of floating point numbers. This should not be + // possible. + panic("illegal value") + } + + // Put limit on size of exponent to prevent non-linear cost of String() + // function on big.Float from causing denial of service: https://github.com/golang/go/issues/11068 + // + // n == sign * mantissa * 2^exp + // 0.5 <= mantissa < 1.0 + // + // The limit is arbitrary. + exp := f.MantExp(nil) + if exp > 1e5 || exp < -1e5 { + return nil, fmt.Errorf("number too big") + } + + return NumberTerm(json.Number(f.String())).SetLocation(loc), nil +} + +func makeString(loc *Location, text interface{}) (interface{}, error) { + var v string + err := json.Unmarshal(text.([]byte), &v) + return StringTerm(v).SetLocation(loc), err +} + +func makeRawString(loc *Location, text interface{}) (interface{}, error) { + s := string(text.([]byte)) + s = s[1 : len(s)-1] // Trim surrounding quotes. + return StringTerm(s).SetLocation(loc), nil +} + +func makeNonterminatedString(loc *Location, s string) (interface{}, error) { + return StringTerm(s).SetLocation(loc), fmt.Errorf("found non-terminated string literal") +} + +func makeBool(loc *Location, text interface{}) (interface{}, error) { + var term *Term + if string(text.([]byte)) == "true" { + term = BooleanTerm(true) + } else { + term = BooleanTerm(false) + } + return term.SetLocation(loc), nil +} + +func makeNull(loc *Location) (interface{}, error) { + return NullTerm().SetLocation(loc), nil +} + +func makeComments(c *current, text interface{}) (interface{}, error) { + + var buf bytes.Buffer + for _, x := range text.([]interface{}) { + buf.Write(x.([]byte)) + } + + comment := NewComment(buf.Bytes()) + comment.Location = currentLocation(c) + comments := c.globalStore[commentsKey].(map[commentKey]*Comment) + key := commentKey{ + File: comment.Location.File, + Row: comment.Location.Row, + Col: comment.Location.Col, + } + comments[key] = comment + return comment, nil +} diff --git a/vendor/github.com/open-policy-agent/opa/ast/policy.go b/vendor/github.com/open-policy-agent/opa/ast/policy.go new file mode 100644 index 000000000..d66f7f06f --- /dev/null +++ b/vendor/github.com/open-policy-agent/opa/ast/policy.go @@ -0,0 +1,1352 @@ +// Copyright 2016 The OPA Authors. All rights reserved. +// Use of this source code is governed by an Apache2 +// license that can be found in the LICENSE file. + +package ast + +import ( + "bytes" + "encoding/json" + "fmt" + "math/rand" + "strings" + "time" + + "github.com/open-policy-agent/opa/util" +) + +// Initialize seed for term hashing. This is intentionally placed before the +// root document sets are constructed to ensure they use the same hash seed as +// subsequent lookups. If the hash seeds are out of sync, lookups will fail. +var hashSeed = rand.New(rand.NewSource(time.Now().UnixNano())) +var hashSeed0 = (uint64(hashSeed.Uint32()) << 32) | uint64(hashSeed.Uint32()) +var hashSeed1 = (uint64(hashSeed.Uint32()) << 32) | uint64(hashSeed.Uint32()) + +// DefaultRootDocument is the default root document. +// +// All package directives inside source files are implicitly prefixed with the +// DefaultRootDocument value. +var DefaultRootDocument = VarTerm("data") + +// InputRootDocument names the document containing query arguments. +var InputRootDocument = VarTerm("input") + +// RootDocumentNames contains the names of top-level documents that can be +// referred to in modules and queries. +var RootDocumentNames = NewSet( + DefaultRootDocument, + InputRootDocument, +) + +// DefaultRootRef is a reference to the root of the default document. +// +// All refs to data in the policy engine's storage layer are prefixed with this ref. +var DefaultRootRef = Ref{DefaultRootDocument} + +// InputRootRef is a reference to the root of the input document. +// +// All refs to query arguments are prefixed with this ref. +var InputRootRef = Ref{InputRootDocument} + +// RootDocumentRefs contains the prefixes of top-level documents that all +// non-local references start with. +var RootDocumentRefs = NewSet( + NewTerm(DefaultRootRef), + NewTerm(InputRootRef), +) + +// SystemDocumentKey is the name of the top-level key that identifies the system +// document. +var SystemDocumentKey = String("system") + +// ReservedVars is the set of names that refer to implicitly ground vars. +var ReservedVars = NewVarSet( + DefaultRootDocument.Value.(Var), + InputRootDocument.Value.(Var), +) + +// Wildcard represents the wildcard variable as defined in the language. +var Wildcard = &Term{Value: Var("_")} + +// WildcardPrefix is the special character that all wildcard variables are +// prefixed with when the statement they are contained in is parsed. +var WildcardPrefix = "$" + +// Keywords contains strings that map to language keywords. +var Keywords = [...]string{ + "not", + "package", + "import", + "as", + "default", + "else", + "with", + "null", + "true", + "false", + "some", +} + +// IsKeyword returns true if s is a language keyword. +func IsKeyword(s string) bool { + for _, x := range Keywords { + if x == s { + return true + } + } + return false +} + +type ( + // Node represents a node in an AST. Nodes may be statements in a policy module + // or elements of an ad-hoc query, expression, etc. + Node interface { + fmt.Stringer + Loc() *Location + SetLoc(*Location) + } + + // Statement represents a single statement in a policy module. + Statement interface { + Node + } +) + +type ( + + // Module represents a collection of policies (defined by rules) + // within a namespace (defined by the package) and optional + // dependencies on external documents (defined by imports). + Module struct { + Package *Package `json:"package"` + Imports []*Import `json:"imports,omitempty"` + Rules []*Rule `json:"rules,omitempty"` + Comments []*Comment `json:"comments,omitempty"` + } + + // Comment contains the raw text from the comment in the definition. + Comment struct { + Text []byte + Location *Location + } + + // Package represents the namespace of the documents produced + // by rules inside the module. + Package struct { + Location *Location `json:"-"` + Path Ref `json:"path"` + } + + // Import represents a dependency on a document outside of the policy + // namespace. Imports are optional. + Import struct { + Location *Location `json:"-"` + Path *Term `json:"path"` + Alias Var `json:"alias,omitempty"` + } + + // Rule represents a rule as defined in the language. Rules define the + // content of documents that represent policy decisions. + Rule struct { + Location *Location `json:"-"` + Default bool `json:"default,omitempty"` + Head *Head `json:"head"` + Body Body `json:"body"` + Else *Rule `json:"else,omitempty"` + + // Module is a pointer to the module containing this rule. If the rule + // was NOT created while parsing/constructing a module, this should be + // left unset. The pointer is not included in any standard operations + // on the rule (e.g., printing, comparison, visiting, etc.) + Module *Module `json:"-"` + } + + // Head represents the head of a rule. + Head struct { + Location *Location `json:"-"` + Name Var `json:"name"` + Args Args `json:"args,omitempty"` + Key *Term `json:"key,omitempty"` + Value *Term `json:"value,omitempty"` + Assign bool `json:"assign,omitempty"` + } + + // Args represents zero or more arguments to a rule. + Args []*Term + + // Body represents one or more expressions contained inside a rule or user + // function. + Body []*Expr + + // Expr represents a single expression contained inside the body of a rule. + Expr struct { + Location *Location `json:"-"` + Generated bool `json:"generated,omitempty"` + Index int `json:"index"` + Negated bool `json:"negated,omitempty"` + Terms interface{} `json:"terms"` + With []*With `json:"with,omitempty"` + } + + // SomeDecl represents a variable declaration statement. The symbols are variables. + SomeDecl struct { + Location *Location `json:"-"` + Symbols []*Term `json:"symbols"` + } + + // With represents a modifier on an expression. + With struct { + Location *Location `json:"-"` + Target *Term `json:"target"` + Value *Term `json:"value"` + } +) + +// Compare returns an integer indicating whether mod is less than, equal to, +// or greater than other. +func (mod *Module) Compare(other *Module) int { + if mod == nil { + if other == nil { + return 0 + } + return -1 + } else if other == nil { + return 1 + } + if cmp := mod.Package.Compare(other.Package); cmp != 0 { + return cmp + } + if cmp := importsCompare(mod.Imports, other.Imports); cmp != 0 { + return cmp + } + return rulesCompare(mod.Rules, other.Rules) +} + +// Copy returns a deep copy of mod. +func (mod *Module) Copy() *Module { + cpy := *mod + cpy.Rules = make([]*Rule, len(mod.Rules)) + for i := range mod.Rules { + cpy.Rules[i] = mod.Rules[i].Copy() + } + cpy.Imports = make([]*Import, len(mod.Imports)) + for i := range mod.Imports { + cpy.Imports[i] = mod.Imports[i].Copy() + } + cpy.Package = mod.Package.Copy() + return &cpy +} + +// Equal returns true if mod equals other. +func (mod *Module) Equal(other *Module) bool { + return mod.Compare(other) == 0 +} + +func (mod *Module) String() string { + buf := []string{} + buf = append(buf, mod.Package.String()) + if len(mod.Imports) > 0 { + buf = append(buf, "") + for _, imp := range mod.Imports { + buf = append(buf, imp.String()) + } + } + if len(mod.Rules) > 0 { + buf = append(buf, "") + for _, rule := range mod.Rules { + buf = append(buf, rule.String()) + } + } + return strings.Join(buf, "\n") +} + +// RuleSet returns a RuleSet containing named rules in the mod. +func (mod *Module) RuleSet(name Var) RuleSet { + rs := NewRuleSet() + for _, rule := range mod.Rules { + if rule.Head.Name.Equal(name) { + rs.Add(rule) + } + } + return rs +} + +// UnmarshalJSON parses bs and stores the result in mod. The rules in the module +// will have their module pointer set to mod. +func (mod *Module) UnmarshalJSON(bs []byte) error { + + // Declare a new type and use a type conversion to avoid recursively calling + // Module#UnmarshalJSON. + type module Module + + if err := util.UnmarshalJSON(bs, (*module)(mod)); err != nil { + return err + } + + WalkRules(mod, func(rule *Rule) bool { + rule.Module = mod + return false + }) + + return nil +} + +// NewComment returns a new Comment object. +func NewComment(text []byte) *Comment { + return &Comment{ + Text: text, + } +} + +// Loc returns the location of the comment in the definition. +func (c *Comment) Loc() *Location { + if c == nil { + return nil + } + return c.Location +} + +// SetLoc sets the location on c. +func (c *Comment) SetLoc(loc *Location) { + c.Location = loc +} + +func (c *Comment) String() string { + return "#" + string(c.Text) +} + +// Equal returns true if this comment equals the other comment. +// Unlike other equality checks on AST nodes, comment equality +// depends on location. +func (c *Comment) Equal(other *Comment) bool { + return c.Location.Equal(other.Location) && bytes.Equal(c.Text, other.Text) +} + +// Compare returns an integer indicating whether pkg is less than, equal to, +// or greater than other. +func (pkg *Package) Compare(other *Package) int { + return Compare(pkg.Path, other.Path) +} + +// Copy returns a deep copy of pkg. +func (pkg *Package) Copy() *Package { + cpy := *pkg + cpy.Path = pkg.Path.Copy() + return &cpy +} + +// Equal returns true if pkg is equal to other. +func (pkg *Package) Equal(other *Package) bool { + return pkg.Compare(other) == 0 +} + +// Loc returns the location of the Package in the definition. +func (pkg *Package) Loc() *Location { + if pkg == nil { + return nil + } + return pkg.Location +} + +// SetLoc sets the location on pkg. +func (pkg *Package) SetLoc(loc *Location) { + pkg.Location = loc +} + +func (pkg *Package) String() string { + if pkg == nil { + return "" + } else if len(pkg.Path) <= 1 { + return fmt.Sprintf("package ", pkg.Path) + } + // Omit head as all packages have the DefaultRootDocument prepended at parse time. + path := make(Ref, len(pkg.Path)-1) + path[0] = VarTerm(string(pkg.Path[1].Value.(String))) + copy(path[1:], pkg.Path[2:]) + return fmt.Sprintf("package %v", path) +} + +// IsValidImportPath returns an error indicating if the import path is invalid. +// If the import path is invalid, err is nil. +func IsValidImportPath(v Value) (err error) { + switch v := v.(type) { + case Var: + if !v.Equal(DefaultRootDocument.Value) && !v.Equal(InputRootDocument.Value) { + return fmt.Errorf("invalid path %v: path must begin with input or data", v) + } + case Ref: + if err := IsValidImportPath(v[0].Value); err != nil { + return fmt.Errorf("invalid path %v: path must begin with input or data", v) + } + for _, e := range v[1:] { + if _, ok := e.Value.(String); !ok { + return fmt.Errorf("invalid path %v: path elements must be strings", v) + } + } + default: + return fmt.Errorf("invalid path %v: path must be ref or var", v) + } + return nil +} + +// Compare returns an integer indicating whether imp is less than, equal to, +// or greater than other. +func (imp *Import) Compare(other *Import) int { + if imp == nil { + if other == nil { + return 0 + } + return -1 + } else if other == nil { + return 1 + } + if cmp := Compare(imp.Path, other.Path); cmp != 0 { + return cmp + } + return Compare(imp.Alias, other.Alias) +} + +// Copy returns a deep copy of imp. +func (imp *Import) Copy() *Import { + cpy := *imp + cpy.Path = imp.Path.Copy() + return &cpy +} + +// Equal returns true if imp is equal to other. +func (imp *Import) Equal(other *Import) bool { + return imp.Compare(other) == 0 +} + +// Loc returns the location of the Import in the definition. +func (imp *Import) Loc() *Location { + if imp == nil { + return nil + } + return imp.Location +} + +// SetLoc sets the location on imp. +func (imp *Import) SetLoc(loc *Location) { + imp.Location = loc +} + +// Name returns the variable that is used to refer to the imported virtual +// document. This is the alias if defined otherwise the last element in the +// path. +func (imp *Import) Name() Var { + if len(imp.Alias) != 0 { + return imp.Alias + } + switch v := imp.Path.Value.(type) { + case Var: + return v + case Ref: + if len(v) == 1 { + return v[0].Value.(Var) + } + return Var(v[len(v)-1].Value.(String)) + } + panic("illegal import") +} + +func (imp *Import) String() string { + buf := []string{"import", imp.Path.String()} + if len(imp.Alias) > 0 { + buf = append(buf, "as "+imp.Alias.String()) + } + return strings.Join(buf, " ") +} + +// Compare returns an integer indicating whether rule is less than, equal to, +// or greater than other. +func (rule *Rule) Compare(other *Rule) int { + if rule == nil { + if other == nil { + return 0 + } + return -1 + } else if other == nil { + return 1 + } + if cmp := rule.Head.Compare(other.Head); cmp != 0 { + return cmp + } + if cmp := util.Compare(rule.Default, other.Default); cmp != 0 { + return cmp + } + if cmp := rule.Body.Compare(other.Body); cmp != 0 { + return cmp + } + return rule.Else.Compare(other.Else) +} + +// Copy returns a deep copy of rule. +func (rule *Rule) Copy() *Rule { + cpy := *rule + cpy.Head = rule.Head.Copy() + cpy.Body = rule.Body.Copy() + if cpy.Else != nil { + cpy.Else = rule.Else.Copy() + } + return &cpy +} + +// Equal returns true if rule is equal to other. +func (rule *Rule) Equal(other *Rule) bool { + return rule.Compare(other) == 0 +} + +// Loc returns the location of the Rule in the definition. +func (rule *Rule) Loc() *Location { + if rule == nil { + return nil + } + return rule.Location +} + +// SetLoc sets the location on rule. +func (rule *Rule) SetLoc(loc *Location) { + rule.Location = loc +} + +// Path returns a ref referring to the document produced by this rule. If rule +// is not contained in a module, this function panics. +func (rule *Rule) Path() Ref { + if rule.Module == nil { + panic("assertion failed") + } + return rule.Module.Package.Path.Append(StringTerm(string(rule.Head.Name))) +} + +func (rule *Rule) String() string { + buf := []string{} + if rule.Default { + buf = append(buf, "default") + } + buf = append(buf, rule.Head.String()) + if !rule.Default { + buf = append(buf, "{") + buf = append(buf, rule.Body.String()) + buf = append(buf, "}") + } + if rule.Else != nil { + buf = append(buf, rule.Else.elseString()) + } + return strings.Join(buf, " ") +} + +func (rule *Rule) elseString() string { + var buf []string + + buf = append(buf, "else") + + value := rule.Head.Value + if value != nil { + buf = append(buf, "=") + buf = append(buf, value.String()) + } + + buf = append(buf, "{") + buf = append(buf, rule.Body.String()) + buf = append(buf, "}") + + if rule.Else != nil { + buf = append(buf, rule.Else.elseString()) + } + + return strings.Join(buf, " ") +} + +// NewHead returns a new Head object. If args are provided, the first will be +// used for the key and the second will be used for the value. +func NewHead(name Var, args ...*Term) *Head { + head := &Head{ + Name: name, + } + if len(args) == 0 { + return head + } + head.Key = args[0] + if len(args) == 1 { + return head + } + head.Value = args[1] + return head +} + +// DocKind represents the collection of document types that can be produced by rules. +type DocKind int + +const ( + // CompleteDoc represents a document that is completely defined by the rule. + CompleteDoc = iota + + // PartialSetDoc represents a set document that is partially defined by the rule. + PartialSetDoc = iota + + // PartialObjectDoc represents an object document that is partially defined by the rule. + PartialObjectDoc = iota +) + +// DocKind returns the type of document produced by this rule. +func (head *Head) DocKind() DocKind { + if head.Key != nil { + if head.Value != nil { + return PartialObjectDoc + } + return PartialSetDoc + } + return CompleteDoc +} + +// Compare returns an integer indicating whether head is less than, equal to, +// or greater than other. +func (head *Head) Compare(other *Head) int { + if head == nil { + if other == nil { + return 0 + } + return -1 + } else if other == nil { + return 1 + } + if head.Assign && !other.Assign { + return -1 + } else if !head.Assign && other.Assign { + return 1 + } + if cmp := Compare(head.Args, other.Args); cmp != 0 { + return cmp + } + if cmp := Compare(head.Name, other.Name); cmp != 0 { + return cmp + } + if cmp := Compare(head.Key, other.Key); cmp != 0 { + return cmp + } + return Compare(head.Value, other.Value) +} + +// Copy returns a deep copy of head. +func (head *Head) Copy() *Head { + cpy := *head + cpy.Args = head.Args.Copy() + cpy.Key = head.Key.Copy() + cpy.Value = head.Value.Copy() + return &cpy +} + +// Equal returns true if this head equals other. +func (head *Head) Equal(other *Head) bool { + return head.Compare(other) == 0 +} + +func (head *Head) String() string { + var buf []string + if len(head.Args) != 0 { + buf = append(buf, head.Name.String()+head.Args.String()) + } else if head.Key != nil { + buf = append(buf, head.Name.String()+"["+head.Key.String()+"]") + } else { + buf = append(buf, head.Name.String()) + } + if head.Value != nil { + if head.Assign { + buf = append(buf, ":=") + } else { + buf = append(buf, "=") + } + buf = append(buf, head.Value.String()) + } + return strings.Join(buf, " ") +} + +// Vars returns a set of vars found in the head. +func (head *Head) Vars() VarSet { + vis := &VarVisitor{vars: VarSet{}} + // TODO: improve test coverage for this. + if head.Args != nil { + vis.Walk(head.Args) + } + if head.Key != nil { + vis.Walk(head.Key) + } + if head.Value != nil { + vis.Walk(head.Value) + } + return vis.vars +} + +// Loc returns the Location of head. +func (head *Head) Loc() *Location { + if head == nil { + return nil + } + return head.Location +} + +// SetLoc sets the location on head. +func (head *Head) SetLoc(loc *Location) { + head.Location = loc +} + +// Copy returns a deep copy of a. +func (a Args) Copy() Args { + cpy := Args{} + for _, t := range a { + cpy = append(cpy, t.Copy()) + } + return cpy +} + +func (a Args) String() string { + var buf []string + for _, t := range a { + buf = append(buf, t.String()) + } + return "(" + strings.Join(buf, ", ") + ")" +} + +// Loc returns the Location of a. +func (a Args) Loc() *Location { + if len(a) == 0 { + return nil + } + return a[0].Location +} + +// SetLoc sets the location on a. +func (a Args) SetLoc(loc *Location) { + if len(a) != 0 { + a[0].SetLocation(loc) + } +} + +// Vars returns a set of vars that appear in a. +func (a Args) Vars() VarSet { + vis := &VarVisitor{vars: VarSet{}} + vis.Walk(a) + return vis.vars +} + +// NewBody returns a new Body containing the given expressions. The indices of +// the immediate expressions will be reset. +func NewBody(exprs ...*Expr) Body { + for i, expr := range exprs { + expr.Index = i + } + return Body(exprs) +} + +// MarshalJSON returns JSON encoded bytes representing body. +func (body Body) MarshalJSON() ([]byte, error) { + // Serialize empty Body to empty array. This handles both the empty case and the + // nil case (whereas by default the result would be null if body was nil.) + if len(body) == 0 { + return []byte(`[]`), nil + } + return json.Marshal([]*Expr(body)) +} + +// Append adds the expr to the body and updates the expr's index accordingly. +func (body *Body) Append(expr *Expr) { + n := len(*body) + expr.Index = n + *body = append(*body, expr) +} + +// Set sets the expr in the body at the specified position and updates the +// expr's index accordingly. +func (body Body) Set(expr *Expr, pos int) { + body[pos] = expr + expr.Index = pos +} + +// Compare returns an integer indicating whether body is less than, equal to, +// or greater than other. +// +// If body is a subset of other, it is considered less than (and vice versa). +func (body Body) Compare(other Body) int { + minLen := len(body) + if len(other) < minLen { + minLen = len(other) + } + for i := 0; i < minLen; i++ { + if cmp := body[i].Compare(other[i]); cmp != 0 { + return cmp + } + } + if len(body) < len(other) { + return -1 + } + if len(other) < len(body) { + return 1 + } + return 0 +} + +// Copy returns a deep copy of body. +func (body Body) Copy() Body { + cpy := make(Body, len(body)) + for i := range body { + cpy[i] = body[i].Copy() + } + return cpy +} + +// Contains returns true if this body contains the given expression. +func (body Body) Contains(x *Expr) bool { + for _, e := range body { + if e.Equal(x) { + return true + } + } + return false +} + +// Equal returns true if this Body is equal to the other Body. +func (body Body) Equal(other Body) bool { + return body.Compare(other) == 0 +} + +// Hash returns the hash code for the Body. +func (body Body) Hash() int { + s := 0 + for _, e := range body { + s += e.Hash() + } + return s +} + +// IsGround returns true if all of the expressions in the Body are ground. +func (body Body) IsGround() bool { + for _, e := range body { + if !e.IsGround() { + return false + } + } + return true +} + +// Loc returns the location of the Body in the definition. +func (body Body) Loc() *Location { + if len(body) == 0 { + return nil + } + return body[0].Location +} + +// SetLoc sets the location on body. +func (body Body) SetLoc(loc *Location) { + if len(body) != 0 { + body[0].SetLocation(loc) + } +} + +func (body Body) String() string { + var buf []string + for _, v := range body { + buf = append(buf, v.String()) + } + return strings.Join(buf, "; ") +} + +// Vars returns a VarSet containing variables in body. The params can be set to +// control which vars are included. +func (body Body) Vars(params VarVisitorParams) VarSet { + vis := NewVarVisitor().WithParams(params) + vis.Walk(body) + return vis.Vars() +} + +// NewExpr returns a new Expr object. +func NewExpr(terms interface{}) *Expr { + return &Expr{ + Negated: false, + Terms: terms, + Index: 0, + With: nil, + } +} + +// Complement returns a copy of this expression with the negation flag flipped. +func (expr *Expr) Complement() *Expr { + cpy := *expr + cpy.Negated = !cpy.Negated + return &cpy +} + +// Equal returns true if this Expr equals the other Expr. +func (expr *Expr) Equal(other *Expr) bool { + return expr.Compare(other) == 0 +} + +// Compare returns an integer indicating whether expr is less than, equal to, +// or greater than other. +// +// Expressions are compared as follows: +// +// 1. Declarations are always less than other expressions. +// 2. Preceding expression (by Index) is always less than the other expression. +// 3. Non-negated expressions are always less than than negated expressions. +// 4. Single term expressions are always less than built-in expressions. +// +// Otherwise, the expression terms are compared normally. If both expressions +// have the same terms, the modifiers are compared. +func (expr *Expr) Compare(other *Expr) int { + + if expr == nil { + if other == nil { + return 0 + } + return -1 + } else if other == nil { + return 1 + } + + o1 := expr.sortOrder() + o2 := other.sortOrder() + if o1 < o2 { + return -1 + } else if o2 < o1 { + return 1 + } + + switch { + case expr.Index < other.Index: + return -1 + case expr.Index > other.Index: + return 1 + } + + switch { + case expr.Negated && !other.Negated: + return 1 + case !expr.Negated && other.Negated: + return -1 + } + + switch t := expr.Terms.(type) { + case *Term: + if cmp := Compare(t.Value, other.Terms.(*Term).Value); cmp != 0 { + return cmp + } + case []*Term: + if cmp := termSliceCompare(t, other.Terms.([]*Term)); cmp != 0 { + return cmp + } + case *SomeDecl: + if cmp := Compare(t, other.Terms.(*SomeDecl)); cmp != 0 { + return cmp + } + } + + return withSliceCompare(expr.With, other.With) +} + +func (expr *Expr) sortOrder() int { + switch expr.Terms.(type) { + case *SomeDecl: + return 0 + case *Term: + return 1 + case []*Term: + return 2 + } + return -1 +} + +// Copy returns a deep copy of expr. +func (expr *Expr) Copy() *Expr { + + cpy := *expr + + switch ts := expr.Terms.(type) { + case *SomeDecl: + cpy.Terms = ts.Copy() + case []*Term: + cpyTs := make([]*Term, len(ts)) + for i := range ts { + cpyTs[i] = ts[i].Copy() + } + cpy.Terms = cpyTs + case *Term: + cpy.Terms = ts.Copy() + } + + cpy.With = make([]*With, len(expr.With)) + for i := range expr.With { + cpy.With[i] = expr.With[i].Copy() + } + + return &cpy +} + +// Hash returns the hash code of the Expr. +func (expr *Expr) Hash() int { + s := expr.Index + switch ts := expr.Terms.(type) { + case *SomeDecl: + s += ts.Hash() + case []*Term: + for _, t := range ts { + s += t.Value.Hash() + } + case *Term: + s += ts.Value.Hash() + } + if expr.Negated { + s++ + } + for _, w := range expr.With { + s += w.Hash() + } + return s +} + +// IncludeWith returns a copy of expr with the with modifier appended. +func (expr *Expr) IncludeWith(target *Term, value *Term) *Expr { + cpy := *expr + cpy.With = append(cpy.With, &With{Target: target, Value: value}) + return &cpy +} + +// NoWith returns a copy of expr where the with modifier has been removed. +func (expr *Expr) NoWith() *Expr { + cpy := *expr + cpy.With = nil + return &cpy +} + +// IsEquality returns true if this is an equality expression. +func (expr *Expr) IsEquality() bool { + return isglobalbuiltin(expr, Var(Equality.Name)) +} + +// IsAssignment returns true if this an assignment expression. +func (expr *Expr) IsAssignment() bool { + return isglobalbuiltin(expr, Var(Assign.Name)) +} + +// IsCall returns true if this expression calls a function. +func (expr *Expr) IsCall() bool { + _, ok := expr.Terms.([]*Term) + return ok +} + +// Operator returns the name of the function or built-in this expression refers +// to. If this expression is not a function call, returns nil. +func (expr *Expr) Operator() Ref { + terms, ok := expr.Terms.([]*Term) + if !ok || len(terms) == 0 { + return nil + } + return terms[0].Value.(Ref) +} + +// Operand returns the term at the zero-based pos. If the expr does not include +// at least pos+1 terms, this function returns nil. +func (expr *Expr) Operand(pos int) *Term { + terms, ok := expr.Terms.([]*Term) + if !ok { + return nil + } + idx := pos + 1 + if idx < len(terms) { + return terms[idx] + } + return nil +} + +// Operands returns the built-in function operands. +func (expr *Expr) Operands() []*Term { + terms, ok := expr.Terms.([]*Term) + if !ok { + return nil + } + return terms[1:] +} + +// IsGround returns true if all of the expression terms are ground. +func (expr *Expr) IsGround() bool { + switch ts := expr.Terms.(type) { + case []*Term: + for _, t := range ts[1:] { + if !t.IsGround() { + return false + } + } + case *Term: + return ts.IsGround() + } + return true +} + +// SetOperator sets the expr's operator and returns the expr itself. If expr is +// not a call expr, this function will panic. +func (expr *Expr) SetOperator(term *Term) *Expr { + expr.Terms.([]*Term)[0] = term + return expr +} + +// SetLocation sets the expr's location and returns the expr itself. +func (expr *Expr) SetLocation(loc *Location) *Expr { + expr.Location = loc + return expr +} + +// Loc returns the Location of expr. +func (expr *Expr) Loc() *Location { + if expr == nil { + return nil + } + return expr.Location +} + +// SetLoc sets the location on expr. +func (expr *Expr) SetLoc(loc *Location) { + expr.SetLocation(loc) +} + +func (expr *Expr) String() string { + var buf []string + if expr.Negated { + buf = append(buf, "not") + } + switch t := expr.Terms.(type) { + case []*Term: + if expr.IsEquality() && validEqAssignArgCount(expr) { + buf = append(buf, fmt.Sprintf("%v %v %v", t[1], Equality.Infix, t[2])) + } else { + buf = append(buf, Call(t).String()) + } + case *Term: + buf = append(buf, t.String()) + case *SomeDecl: + buf = append(buf, t.String()) + } + + for i := range expr.With { + buf = append(buf, expr.With[i].String()) + } + + return strings.Join(buf, " ") +} + +// UnmarshalJSON parses the byte array and stores the result in expr. +func (expr *Expr) UnmarshalJSON(bs []byte) error { + v := map[string]interface{}{} + if err := util.UnmarshalJSON(bs, &v); err != nil { + return err + } + return unmarshalExpr(expr, v) +} + +// Vars returns a VarSet containing variables in expr. The params can be set to +// control which vars are included. +func (expr *Expr) Vars(params VarVisitorParams) VarSet { + vis := NewVarVisitor().WithParams(params) + vis.Walk(expr) + return vis.Vars() +} + +// NewBuiltinExpr creates a new Expr object with the supplied terms. +// The builtin operator must be the first term. +func NewBuiltinExpr(terms ...*Term) *Expr { + return &Expr{Terms: terms} +} + +func (d *SomeDecl) String() string { + buf := make([]string, len(d.Symbols)) + for i := range buf { + buf[i] = d.Symbols[i].String() + } + return "some " + strings.Join(buf, ", ") +} + +// SetLoc sets the Location on d. +func (d *SomeDecl) SetLoc(loc *Location) { + d.Location = loc +} + +// Loc returns the Location of d. +func (d *SomeDecl) Loc() *Location { + return d.Location +} + +// Copy returns a deep copy of d. +func (d *SomeDecl) Copy() *SomeDecl { + cpy := *d + cpy.Symbols = termSliceCopy(d.Symbols) + return &cpy +} + +// Compare returns an integer indicating whether d is less than, equal to, or +// greater than other. +func (d *SomeDecl) Compare(other *SomeDecl) int { + return termSliceCompare(d.Symbols, other.Symbols) +} + +// Hash returns a hash code of d. +func (d *SomeDecl) Hash() int { + return termSliceHash(d.Symbols) +} + +func (w *With) String() string { + return "with " + w.Target.String() + " as " + w.Value.String() +} + +// Equal returns true if this With is equals the other With. +func (w *With) Equal(other *With) bool { + return Compare(w, other) == 0 +} + +// Compare returns an integer indicating whether w is less than, equal to, or +// greater than other. +func (w *With) Compare(other *With) int { + if w == nil { + if other == nil { + return 0 + } + return -1 + } else if other == nil { + return 1 + } + if cmp := Compare(w.Target, other.Target); cmp != 0 { + return cmp + } + return Compare(w.Value, other.Value) +} + +// Copy returns a deep copy of w. +func (w *With) Copy() *With { + cpy := *w + cpy.Value = w.Value.Copy() + cpy.Target = w.Target.Copy() + return &cpy +} + +// Hash returns the hash code of the With. +func (w With) Hash() int { + return w.Target.Hash() + w.Value.Hash() +} + +// SetLocation sets the location on w. +func (w *With) SetLocation(loc *Location) *With { + w.Location = loc + return w +} + +// Loc returns the Location of w. +func (w *With) Loc() *Location { + if w == nil { + return nil + } + return w.Location +} + +// SetLoc sets the location on w. +func (w *With) SetLoc(loc *Location) { + w.Location = loc +} + +// RuleSet represents a collection of rules that produce a virtual document. +type RuleSet []*Rule + +// NewRuleSet returns a new RuleSet containing the given rules. +func NewRuleSet(rules ...*Rule) RuleSet { + rs := make(RuleSet, 0, len(rules)) + for _, rule := range rules { + rs.Add(rule) + } + return rs +} + +// Add inserts the rule into rs. +func (rs *RuleSet) Add(rule *Rule) { + for _, exist := range *rs { + if exist.Equal(rule) { + return + } + } + *rs = append(*rs, rule) +} + +// Contains returns true if rs contains rule. +func (rs RuleSet) Contains(rule *Rule) bool { + for i := range rs { + if rs[i].Equal(rule) { + return true + } + } + return false +} + +// Diff returns a new RuleSet containing rules in rs that are not in other. +func (rs RuleSet) Diff(other RuleSet) RuleSet { + result := NewRuleSet() + for i := range rs { + if !other.Contains(rs[i]) { + result.Add(rs[i]) + } + } + return result +} + +// Equal returns true if rs equals other. +func (rs RuleSet) Equal(other RuleSet) bool { + return len(rs.Diff(other)) == 0 && len(other.Diff(rs)) == 0 +} + +// Merge returns a ruleset containing the union of rules from rs an other. +func (rs RuleSet) Merge(other RuleSet) RuleSet { + result := NewRuleSet() + for i := range rs { + result.Add(rs[i]) + } + for i := range other { + result.Add(other[i]) + } + return result +} + +func (rs RuleSet) String() string { + buf := make([]string, 0, len(rs)) + for _, rule := range rs { + buf = append(buf, rule.String()) + } + return "{" + strings.Join(buf, ", ") + "}" +} + +type ruleSlice []*Rule + +func (s ruleSlice) Less(i, j int) bool { return Compare(s[i], s[j]) < 0 } +func (s ruleSlice) Swap(i, j int) { x := s[i]; s[i] = s[j]; s[j] = x } +func (s ruleSlice) Len() int { return len(s) } + +// Returns true if the equality or assignment expression referred to by expr +// has a valid number of arguments. +func validEqAssignArgCount(expr *Expr) bool { + return len(expr.Operands()) == 2 +} + +// this function checks if the expr refers to a non-namespaced (global) built-in +// function like eq, gt, plus, etc. +func isglobalbuiltin(expr *Expr, name Var) bool { + terms, ok := expr.Terms.([]*Term) + if !ok { + return false + } + + // NOTE(tsandall): do not use Term#Equal or Value#Compare to avoid + // allocation here. + ref, ok := terms[0].Value.(Ref) + if !ok || len(ref) != 1 { + return false + } else if head, ok := ref[0].Value.(Var); !ok { + return false + } else { + return head.Equal(name) + } +} diff --git a/vendor/github.com/open-policy-agent/opa/ast/pretty.go b/vendor/github.com/open-policy-agent/opa/ast/pretty.go new file mode 100644 index 000000000..b4f05ad50 --- /dev/null +++ b/vendor/github.com/open-policy-agent/opa/ast/pretty.go @@ -0,0 +1,82 @@ +// Copyright 2018 The OPA Authors. All rights reserved. +// Use of this source code is governed by an Apache2 +// license that can be found in the LICENSE file. + +package ast + +import ( + "fmt" + "io" + "strings" +) + +// Pretty writes a pretty representation of the AST rooted at x to w. +// +// This is function is intended for debug purposes when inspecting ASTs. +func Pretty(w io.Writer, x interface{}) { + pp := &prettyPrinter{ + depth: -1, + w: w, + } + NewBeforeAfterVisitor(pp.Before, pp.After).Walk(x) +} + +type prettyPrinter struct { + depth int + w io.Writer +} + +func (pp *prettyPrinter) Before(x interface{}) bool { + switch x.(type) { + case *Term: + default: + pp.depth++ + } + + switch x := x.(type) { + case *Term: + return false + case Args: + if len(x) == 0 { + return false + } + pp.writeType(x) + case *Expr: + extras := []string{} + if x.Negated { + extras = append(extras, "negated") + } + extras = append(extras, fmt.Sprintf("index=%d", x.Index)) + pp.writeIndent("%v %v", TypeName(x), strings.Join(extras, " ")) + case Null, Boolean, Number, String, Var: + pp.writeValue(x) + default: + pp.writeType(x) + } + return false +} + +func (pp *prettyPrinter) After(x interface{}) { + switch x.(type) { + case *Term: + default: + pp.depth-- + } +} + +func (pp *prettyPrinter) writeValue(x interface{}) { + pp.writeIndent(fmt.Sprint(x)) +} + +func (pp *prettyPrinter) writeType(x interface{}) { + pp.writeIndent(TypeName(x)) +} + +func (pp *prettyPrinter) writeIndent(f string, a ...interface{}) { + pad := strings.Repeat(" ", pp.depth) + pp.write(pad+f, a...) +} + +func (pp *prettyPrinter) write(f string, a ...interface{}) { + fmt.Fprintf(pp.w, f+"\n", a...) +} diff --git a/vendor/github.com/open-policy-agent/opa/ast/rego.peg b/vendor/github.com/open-policy-agent/opa/ast/rego.peg new file mode 100644 index 000000000..71d04e727 --- /dev/null +++ b/vendor/github.com/open-policy-agent/opa/ast/rego.peg @@ -0,0 +1,311 @@ +{ +package ast +} + +Program <- _ vals:(Stmt (ws Stmt)*)? _ EOF { + return makeProgram(c, vals) +} + +Stmt <- val:(Package / Import / Rules / Body / Comment) { + return val, nil +} + +Package <- "package" ws val:(Ref / Var) { + return makePackage(currentLocation(c), val) +} + +Import <- "import" ws path:(Ref / Var) alias:(ws "as" ws Var)? { + return makeImport(currentLocation(c), path, alias) +} + +Rules <- DefaultRules / NormalRules + +DefaultRules <- "default" ws name:Var _ operator:( ":=" / "=" ) _ value:Term { + return makeDefaultRule(currentLocation(c), name, operator, value) +} + +NormalRules <- head:(PartialRuleHead / RuleHead) _ rest:(NonEmptyBraceEnclosedBody ( _ RuleExt)* ) { + return makeRule(currentLocation(c), head, rest) +} + +PartialRuleHead <- name:Var args:( _ "(" _ Args _ ")" _ ) value:( _ ( ":=" / "=" ) _ ExprTerm )? { + return makeRuleHead(currentLocation(c), name, args, nil, value) +} + +RuleHead <- name:Var key:( _ "[" _ ExprTerm _ "]" _ )? value:( _ ( ":=" / "=" ) _ ExprTerm )? { + return makeRuleHead(currentLocation(c), name, nil, key, value) +} + +Args <- list:ExprTermList { + return makeArgs(list) +} + +Else <- "else" value:( _ "=" _ Term )? body:( _ NonEmptyBraceEnclosedBody ) { + return makeRuleExt(currentLocation(c), value, body) +} + +RuleDup <- b:NonEmptyBraceEnclosedBody { + return ruleExt{loc: currentLocation(c), body: b.(Body)}, nil +} + +RuleExt <- Else / RuleDup + +Body <- NonWhitespaceBody / BraceEnclosedBody + +NonEmptyBraceEnclosedBody <- "{" _ val:WhitespaceBody? _ "}" { + if val == nil { + return NewBody(), fmt.Errorf("found empty body") + } + return val, nil +} + +BraceEnclosedBody <- "{" _ val:WhitespaceBody? _ "}" { + return makeBraceEnclosedBody(currentLocation(c), val) +} + +WhitespaceBody <- head:Literal tail:(WhitespaceLiteralSeparator _ Literal)* { + return makeBody(head, tail, 2) +} + +NonWhitespaceBody <- head:Literal tail:( _ NonWhitespaceLiteralSeparator _ Literal)* { + return makeBody(head, tail, 3) +} + +WhitespaceLiteralSeparator <- [ \t]* ((NonWhitespaceLiteralSeparator Comment?) / (Comment? [\r\n])) + +NonWhitespaceLiteralSeparator <- ";" + +Literal <- TermExpr / SomeDecl + +SomeDecl <- "some" ws symbols:SomeDeclList { + return makeSomeDeclLiteral(currentLocation(c), symbols) +} + +SomeDeclList <- head:Var rest:( _ ',' _ Var)* { + return makeSomeDeclSymbols(head, rest) +} + +TermExpr <- negated:NotKeyword? value:LiteralExpr with:WithKeywordList? { + return makeLiteral(negated, value, with) +} + +LiteralExpr <- lhs:ExprTerm rest:( _ LiteralExprOperator _ ExprTerm)? { + return makeLiteralExpr(currentLocation(c), lhs, rest) +} + +LiteralExprOperator <- val:( ":=" / "=" ) { + return makeInfixOperator(currentLocation(c), c.text) +} + +NotKeyword <- val:("not" ws)? { + return val != nil, nil +} + +WithKeywordList <- ws head:WithKeyword rest:( ws WithKeyword )* { + return makeWithKeywordList(head, rest) +} + +WithKeyword <- "with" ws target:ExprTerm ws "as" ws value:ExprTerm { + return makeWithKeyword(currentLocation(c), target, value) +} + +ExprTerm <- lhs:RelationExpr rest:( _ RelationOperator _ RelationExpr )* { + return makeExprTerm(currentLocation(c), lhs, rest) +} + +ExprTermPairList <- head:ExprTermPair? tail:( _ ',' _ ExprTermPair )* _ ","? { + return makeExprTermPairList(head, tail) +} + +ExprTermList <- head:ExprTerm? tail:( _ ',' _ ExprTerm )* _ ","? { + return makeExprTermList(head, tail) +} + +ExprTermPair <- key:ExprTerm _ ':' _ value:ExprTerm { + return makeExprTermPair(key, value) +} + +RelationOperator <- val:("==" / "!=" / "<=" / ">=" / ">" / "<") { + return makeInfixOperator(currentLocation(c), c.text) +} + +RelationExpr <- lhs:BitwiseOrExpr rest:( _ BitwiseOrOperator _ BitwiseOrExpr)* { + return makeExprTerm(currentLocation(c), lhs, rest) +} + +BitwiseOrOperator <- val:"|" { + return makeInfixOperator(currentLocation(c), c.text) +} + +BitwiseOrExpr <- lhs:BitwiseAndExpr rest:( _ BitwiseAndOperator _ BitwiseAndExpr)* { + return makeExprTerm(currentLocation(c), lhs, rest) +} + +BitwiseAndOperator <- val:"&" { + return makeInfixOperator(currentLocation(c), c.text) +} + +BitwiseAndExpr <- lhs:ArithExpr rest:( _ ArithOperator _ ArithExpr)* { + return makeExprTerm(currentLocation(c), lhs, rest) +} + +ArithOperator <- val:("+" / "-") { + return makeInfixOperator(currentLocation(c), c.text) +} + +ArithExpr <- lhs:FactorExpr rest:( _ FactorOperator _ FactorExpr )* { + return makeExprTerm(currentLocation(c), lhs, rest) +} + +FactorOperator <- val:("*" / "/" / "%"){ + return makeInfixOperator(currentLocation(c), c.text) +} + +FactorExpr <- ( "(" _ expr:ExprTerm _ ")" ) { + return expr, nil +} / term:Term { + return term, nil +} + +Call <- operator:(Ref / Var) "(" _ args:ExprTermList _ ")" { + return makeCall(currentLocation(c), operator, args) +} + +Term <- val:( Comprehension / Composite / Scalar / Call / Var ) refs:RefOperand* { + return makeRef(currentLocation(c), val, refs) +} + +TermPair <- key:Term _ ":" _ value:Term { + return makeExprTermPair(key, value) +} + +Comprehension <- ArrayComprehension / ObjectComprehension / SetComprehension + +ArrayComprehension <- "[" _ head:Term _ "|" _ body:WhitespaceBody _ "]" { + return makeArrayComprehension(currentLocation(c), head, body) +} + +ObjectComprehension <- "{" _ head:TermPair _ "|" _ body:WhitespaceBody _ "}" { + return makeObjectComprehension(currentLocation(c), head, body) +} + +SetComprehension <- "{" _ head:Term _ "|" _ body:WhitespaceBody _ "}" { + return makeSetComprehension(currentLocation(c), head, body) +} + +Composite <- Object / Array / Set + +Scalar <- Number / String / Bool / Null + +Object <- '{' _ list:ExprTermPairList _ '}' { + return makeObject(currentLocation(c), list) +} + +Array <- '[' _ list:ExprTermList _ ']' { + return makeArray(currentLocation(c), list) +} + +Set <- SetEmpty / SetNonEmpty + +SetEmpty <- "set(" _ ")" { + var empty []*Term + return makeSet(currentLocation(c), empty) +} + +SetNonEmpty <- '{' _ list:ExprTermList _ '}' { + return makeSet(currentLocation(c), list) +} + +Ref <- head:(Composite / Var) rest:RefOperand+ { + return makeRef(currentLocation(c), head, rest) +} + +RefOperand <- RefOperandDot / RefOperandCanonical + +RefOperandDot <- "." val:Var { + return makeRefOperandDot(currentLocation(c), val) +} + +RefOperandCanonical <- "[" val:ExprTerm "]" { + return val, nil +} + +Var <- val:VarChecked { + return val.([]interface{})[0], nil +} + +VarChecked <- val:VarUnchecked !{ + return IsKeyword(string(val.(*Term).Value.(Var))), nil +} + +VarUnchecked <- VarStart VarChar* { + return makeVar(currentLocation(c), c.text) +} + +Number <- '-'? ( Float / Integer ) { + return makeNumber(currentLocation(c), c.text) +} + +Float <- ExponentFloat / PointFloat + +ExponentFloat <- ( PointFloat / Integer ) Exponent + +PointFloat <- Integer? Fraction + +Fraction <- '.' DecimalDigit+ + +Exponent <- 'e'i [+-]? DecimalDigit+ + +Integer <- '0' / ( NonZeroDecimalDigit DecimalDigit* ) + +String <- QuotedString / RawString + +QuotedString <- '"' Char* '"' { + return makeString(currentLocation(c), c.text) +} / '"' Char* !'"' { + return makeNonterminatedString(currentLocation(c), string(c.text)) +} + +RawString <- '`' [^`]* '`' { + return makeRawString(currentLocation(c), c.text) +} + +Bool <- val:("true" / "false") !VarChar { + return makeBool(currentLocation(c), c.text) +} + +Null <- "null" !VarChar { + return makeNull(currentLocation(c)) +} + +VarStart <- AsciiLetter + +VarChar <- AsciiLetter / DecimalDigit + +AsciiLetter <- [A-Za-z_] + +Char <- ( !EscapedChar . ) / ( '\\' EscapeSequence ) + +EscapedChar <- [\x00-\x1f"\\] + +EscapeSequence <- SingleCharEscape / UnicodeEscape + +SingleCharEscape <- [ " \\ / b f n r t ] + +UnicodeEscape <- 'u' HexDigit HexDigit HexDigit HexDigit + +DecimalDigit <- [0-9] + +NonZeroDecimalDigit <- [1-9] + +HexDigit <- [0-9a-fA-F] + +ws "whitespace" <- [ \t\r\n]+ + +_ "whitespace" <- ( [ \t\r\n] / Comment )* + +Comment <- [ \t]* "#" text:[^\r\n]* { + return makeComments(c, text) +} + +EOF <- !. diff --git a/vendor/github.com/open-policy-agent/opa/ast/strings.go b/vendor/github.com/open-policy-agent/opa/ast/strings.go new file mode 100644 index 000000000..8f9928017 --- /dev/null +++ b/vendor/github.com/open-policy-agent/opa/ast/strings.go @@ -0,0 +1,15 @@ +// Copyright 2016 The OPA Authors. All rights reserved. +// Use of this source code is governed by an Apache2 +// license that can be found in the LICENSE file. + +package ast + +import ( + "reflect" + "strings" +) + +// TypeName returns a human readable name for the AST element type. +func TypeName(x interface{}) string { + return strings.ToLower(reflect.Indirect(reflect.ValueOf(x)).Type().Name()) +} diff --git a/vendor/github.com/open-policy-agent/opa/ast/term.go b/vendor/github.com/open-policy-agent/opa/ast/term.go new file mode 100644 index 000000000..b6137d2c0 --- /dev/null +++ b/vendor/github.com/open-policy-agent/opa/ast/term.go @@ -0,0 +1,2572 @@ +// Copyright 2016 The OPA Authors. All rights reserved. +// Use of this source code is governed by an Apache2 +// license that can be found in the LICENSE file. + +package ast + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "math/big" + "net/url" + "regexp" + "sort" + "strconv" + "strings" + + "github.com/OneOfOne/xxhash" + "github.com/pkg/errors" + + "github.com/open-policy-agent/opa/util" +) + +var errFindNotFound = fmt.Errorf("find: not found") + +// Location records a position in source code +type Location struct { + Text []byte `json:"-"` // The original text fragment from the source. + File string `json:"file"` // The name of the source file (which may be empty). + Row int `json:"row"` // The line in the source. + Col int `json:"col"` // The column in the row. +} + +// NewLocation returns a new Location object. +func NewLocation(text []byte, file string, row int, col int) *Location { + return &Location{Text: text, File: file, Row: row, Col: col} +} + +// Equal checks if two locations are equal to each other. +func (loc *Location) Equal(other *Location) bool { + return bytes.Equal(loc.Text, other.Text) && + loc.File == other.File && + loc.Row == other.Row && + loc.Col == other.Col +} + +// Errorf returns a new error value with a message formatted to include the location +// info (e.g., line, column, filename, etc.) +func (loc *Location) Errorf(f string, a ...interface{}) error { + return errors.New(loc.Format(f, a...)) +} + +// Wrapf returns a new error value that wraps an existing error with a message formatted +// to include the location info (e.g., line, column, filename, etc.) +func (loc *Location) Wrapf(err error, f string, a ...interface{}) error { + return errors.Wrap(err, loc.Format(f, a...)) +} + +// Format returns a formatted string prefixed with the location information. +func (loc *Location) Format(f string, a ...interface{}) string { + if len(loc.File) > 0 { + f = fmt.Sprintf("%v:%v: %v", loc.File, loc.Row, f) + } else { + f = fmt.Sprintf("%v:%v: %v", loc.Row, loc.Col, f) + } + return fmt.Sprintf(f, a...) +} + +func (loc *Location) String() string { + if len(loc.File) > 0 { + return fmt.Sprintf("%v:%v", loc.File, loc.Row) + } + if len(loc.Text) > 0 { + return string(loc.Text) + } + return fmt.Sprintf("%v:%v", loc.Row, loc.Col) +} + +// Compare returns -1, 0, or 1 to indicate if this loc is less than, equal to, +// or greater than the other. Comparison is performed on the file, row, and +// column of the Location (but not on the text.) Nil locations are greater than +// non-nil locations. +func (loc *Location) Compare(other *Location) int { + if loc == nil && other == nil { + return 0 + } else if loc == nil { + return 1 + } else if other == nil { + return -1 + } else if loc.File < other.File { + return -1 + } else if loc.File > other.File { + return 1 + } else if loc.Row < other.Row { + return -1 + } else if loc.Row > other.Row { + return 1 + } else if loc.Col < other.Col { + return -1 + } else if loc.Col > other.Col { + return 1 + } + return 0 +} + +// Value declares the common interface for all Term values. Every kind of Term value +// in the language is represented as a type that implements this interface: +// +// - Null, Boolean, Number, String +// - Object, Array, Set +// - Variables, References +// - Array, Set, and Object Comprehensions +// - Calls +type Value interface { + Compare(other Value) int // Compare returns <0, 0, or >0 if this Value is less than, equal to, or greater than other, respectively. + Find(path Ref) (Value, error) // Find returns value referred to by path or an error if path is not found. + Hash() int // Returns hash code of the value. + IsGround() bool // IsGround returns true if this value is not a variable or contains no variables. + String() string // String returns a human readable string representation of the value. +} + +// InterfaceToValue converts a native Go value x to a Value. +func InterfaceToValue(x interface{}) (Value, error) { + switch x := x.(type) { + case nil: + return Null{}, nil + case bool: + return Boolean(x), nil + case json.Number: + return Number(x), nil + case int64: + return int64Number(x), nil + case float64: + return floatNumber(x), nil + case int: + return intNumber(x), nil + case string: + return String(x), nil + case []interface{}: + r := make(Array, 0, len(x)) + for _, e := range x { + e, err := InterfaceToValue(e) + if err != nil { + return nil, err + } + r = append(r, &Term{Value: e}) + } + return r, nil + case map[string]interface{}: + r := newobject(len(x)) + for k, v := range x { + k, err := InterfaceToValue(k) + if err != nil { + return nil, err + } + v, err := InterfaceToValue(v) + if err != nil { + return nil, err + } + r.Insert(NewTerm(k), NewTerm(v)) + } + return r, nil + case map[string]string: + r := newobject(len(x)) + for k, v := range x { + k, err := InterfaceToValue(k) + if err != nil { + return nil, err + } + v, err := InterfaceToValue(v) + if err != nil { + return nil, err + } + r.Insert(NewTerm(k), NewTerm(v)) + } + return r, nil + default: + return nil, fmt.Errorf("ast: illegal value: %T", x) + } +} + +// ValueFromReader returns an AST value from a JSON serialized value in the reader. +func ValueFromReader(r io.Reader) (Value, error) { + var x interface{} + if err := util.NewJSONDecoder(r).Decode(&x); err != nil { + return nil, err + } + return InterfaceToValue(x) +} + +// As converts v into a Go native type referred to by x. +func As(v Value, x interface{}) error { + return util.NewJSONDecoder(bytes.NewBufferString(v.String())).Decode(x) +} + +// Resolver defines the interface for resolving references to native Go values. +type Resolver interface { + Resolve(ref Ref) (value interface{}, err error) +} + +// ValueResolver defines the interface for resolving references to AST values. +type ValueResolver interface { + Resolve(ref Ref) (value Value, err error) +} + +// UnknownValueErr indicates a ValueResolver was unable to resolve a reference +// because the reference refers to an unknown value. +type UnknownValueErr struct{} + +func (UnknownValueErr) Error() string { + return "unknown value" +} + +// IsUnknownValueErr returns true if the err is an UnknownValueErr. +func IsUnknownValueErr(err error) bool { + _, ok := err.(UnknownValueErr) + return ok +} + +type illegalResolver struct{} + +func (illegalResolver) Resolve(ref Ref) (interface{}, error) { + return nil, fmt.Errorf("illegal value: %v", ref) +} + +// ValueToInterface returns the Go representation of an AST value. The AST +// value should not contain any values that require evaluation (e.g., vars, +// comprehensions, etc.) +func ValueToInterface(v Value, resolver Resolver) (interface{}, error) { + switch v := v.(type) { + case Null: + return nil, nil + case Boolean: + return bool(v), nil + case Number: + return json.Number(v), nil + case String: + return string(v), nil + case Array: + buf := []interface{}{} + for _, x := range v { + x1, err := ValueToInterface(x.Value, resolver) + if err != nil { + return nil, err + } + buf = append(buf, x1) + } + return buf, nil + case Object: + buf := map[string]interface{}{} + err := v.Iter(func(k, v *Term) error { + ki, err := ValueToInterface(k.Value, resolver) + if err != nil { + return err + } + asStr, stringKey := ki.(string) + if !stringKey { + return fmt.Errorf("object value has non-string key (%T)", ki) + } + vi, err := ValueToInterface(v.Value, resolver) + if err != nil { + return err + } + buf[asStr] = vi + return nil + }) + if err != nil { + return nil, err + } + return buf, nil + case Set: + buf := []interface{}{} + err := v.Iter(func(x *Term) error { + x1, err := ValueToInterface(x.Value, resolver) + if err != nil { + return err + } + buf = append(buf, x1) + return nil + }) + if err != nil { + return nil, err + } + return buf, nil + case Ref: + return resolver.Resolve(v) + default: + return nil, fmt.Errorf("%v requires evaluation", TypeName(v)) + } +} + +// JSON returns the JSON representation of v. The value must not contain any +// refs or terms that require evaluation (e.g., vars, comprehensions, etc.) +func JSON(v Value) (interface{}, error) { + return ValueToInterface(v, illegalResolver{}) +} + +// MustInterfaceToValue converts a native Go value x to a Value. If the +// conversion fails, this function will panic. This function is mostly for test +// purposes. +func MustInterfaceToValue(x interface{}) Value { + v, err := InterfaceToValue(x) + if err != nil { + panic(err) + } + return v +} + +// Term is an argument to a function. +type Term struct { + Value Value `json:"value"` // the value of the Term as represented in Go + Location *Location `json:"-"` // the location of the Term in the source +} + +// NewTerm returns a new Term object. +func NewTerm(v Value) *Term { + return &Term{ + Value: v, + } +} + +// SetLocation updates the term's Location and returns the term itself. +func (term *Term) SetLocation(loc *Location) *Term { + term.Location = loc + return term +} + +// Loc returns the Location of term. +func (term *Term) Loc() *Location { + if term == nil { + return nil + } + return term.Location +} + +// SetLoc sets the location on term. +func (term *Term) SetLoc(loc *Location) { + term.SetLocation(loc) +} + +// Copy returns a deep copy of term. +func (term *Term) Copy() *Term { + + if term == nil { + return nil + } + + cpy := *term + + switch v := term.Value.(type) { + case Null, Boolean, Number, String, Var: + cpy.Value = v + case Ref: + cpy.Value = v.Copy() + case Array: + cpy.Value = v.Copy() + case Set: + cpy.Value = v.Copy() + case Object: + cpy.Value = v.Copy() + case *ArrayComprehension: + cpy.Value = v.Copy() + case *ObjectComprehension: + cpy.Value = v.Copy() + case *SetComprehension: + cpy.Value = v.Copy() + case Call: + cpy.Value = v.Copy() + } + + return &cpy +} + +// Equal returns true if this term equals the other term. Equality is +// defined for each kind of term. +func (term *Term) Equal(other *Term) bool { + if term == nil && other != nil { + return false + } + if term != nil && other == nil { + return false + } + if term == other { + return true + } + + // TODO(tsandall): This early-exit avoids allocations for types that have + // Equal() functions that just use == underneath. We should revisit the + // other types and implement Equal() functions that do not require + // allocations. + switch v := term.Value.(type) { + case Null: + return v.Equal(other.Value) + case Boolean: + return v.Equal(other.Value) + case Number: + return v.Equal(other.Value) + case String: + return v.Equal(other.Value) + case Var: + return v.Equal(other.Value) + } + + return term.Value.Compare(other.Value) == 0 +} + +// Get returns a value referred to by name from the term. +func (term *Term) Get(name *Term) *Term { + switch v := term.Value.(type) { + case Array: + return v.Get(name) + case Object: + return v.Get(name) + case Set: + if v.Contains(name) { + return name + } + } + return nil +} + +// Hash returns the hash code of the Term's value. +func (term *Term) Hash() int { + return term.Value.Hash() +} + +// IsGround returns true if this terms' Value is ground. +func (term *Term) IsGround() bool { + return term.Value.IsGround() +} + +// MarshalJSON returns the JSON encoding of the term. +// +// Specialized marshalling logic is required to include a type hint for Value. +func (term *Term) MarshalJSON() ([]byte, error) { + d := map[string]interface{}{ + "type": TypeName(term.Value), + "value": term.Value, + } + return json.Marshal(d) +} + +func (term *Term) String() string { + return term.Value.String() +} + +// UnmarshalJSON parses the byte array and stores the result in term. +// Specialized unmarshalling is required to handle Value. +func (term *Term) UnmarshalJSON(bs []byte) error { + v := map[string]interface{}{} + if err := util.UnmarshalJSON(bs, &v); err != nil { + return err + } + val, err := unmarshalValue(v) + if err != nil { + return err + } + term.Value = val + return nil +} + +// Vars returns a VarSet with variables contained in this term. +func (term *Term) Vars() VarSet { + vis := &VarVisitor{vars: VarSet{}} + vis.Walk(term) + return vis.vars +} + +// IsConstant returns true if the AST value is constant. +func IsConstant(v Value) bool { + found := false + vis := GenericVisitor{ + func(x interface{}) bool { + switch x.(type) { + case Var, Ref, *ArrayComprehension, *ObjectComprehension, *SetComprehension, Call: + found = true + return true + } + return false + }, + } + vis.Walk(v) + return !found +} + +// IsComprehension returns true if the supplied value is a comprehension. +func IsComprehension(x Value) bool { + switch x.(type) { + case *ArrayComprehension, *ObjectComprehension, *SetComprehension: + return true + } + return false +} + +// ContainsRefs returns true if the Value v contains refs. +func ContainsRefs(v interface{}) bool { + found := false + WalkRefs(v, func(r Ref) bool { + found = true + return found + }) + return found +} + +// ContainsComprehensions returns true if the Value v contains comprehensions. +func ContainsComprehensions(v interface{}) bool { + found := false + WalkClosures(v, func(x interface{}) bool { + switch x.(type) { + case *ArrayComprehension, *ObjectComprehension, *SetComprehension: + found = true + return found + } + return found + }) + return found +} + +// IsScalar returns true if the AST value is a scalar. +func IsScalar(v Value) bool { + switch v.(type) { + case String: + return true + case Number: + return true + case Boolean: + return true + case Null: + return true + } + return false +} + +// Null represents the null value defined by JSON. +type Null struct{} + +// NullTerm creates a new Term with a Null value. +func NullTerm() *Term { + return &Term{Value: Null{}} +} + +// Equal returns true if the other term Value is also Null. +func (null Null) Equal(other Value) bool { + switch other.(type) { + case Null: + return true + default: + return false + } +} + +// Compare compares null to other, return <0, 0, or >0 if it is less than, equal to, +// or greater than other. +func (null Null) Compare(other Value) int { + return Compare(null, other) +} + +// Find returns the current value or a not found error. +func (null Null) Find(path Ref) (Value, error) { + if len(path) == 0 { + return null, nil + } + return nil, errFindNotFound +} + +// Hash returns the hash code for the Value. +func (null Null) Hash() int { + return 0 +} + +// IsGround always returns true. +func (null Null) IsGround() bool { + return true +} + +func (null Null) String() string { + return "null" +} + +// Boolean represents a boolean value defined by JSON. +type Boolean bool + +// BooleanTerm creates a new Term with a Boolean value. +func BooleanTerm(b bool) *Term { + return &Term{Value: Boolean(b)} +} + +// Equal returns true if the other Value is a Boolean and is equal. +func (bol Boolean) Equal(other Value) bool { + switch other := other.(type) { + case Boolean: + return bol == other + default: + return false + } +} + +// Compare compares bol to other, return <0, 0, or >0 if it is less than, equal to, +// or greater than other. +func (bol Boolean) Compare(other Value) int { + return Compare(bol, other) +} + +// Find returns the current value or a not found error. +func (bol Boolean) Find(path Ref) (Value, error) { + if len(path) == 0 { + return bol, nil + } + return nil, errFindNotFound +} + +// Hash returns the hash code for the Value. +func (bol Boolean) Hash() int { + if bol { + return 1 + } + return 0 +} + +// IsGround always returns true. +func (bol Boolean) IsGround() bool { + return true +} + +func (bol Boolean) String() string { + return strconv.FormatBool(bool(bol)) +} + +// Number represents a numeric value as defined by JSON. +type Number json.Number + +// NumberTerm creates a new Term with a Number value. +func NumberTerm(n json.Number) *Term { + return &Term{Value: Number(n)} +} + +// IntNumberTerm creates a new Term with an integer Number value. +func IntNumberTerm(i int) *Term { + return &Term{Value: Number(strconv.Itoa(i))} +} + +// FloatNumberTerm creates a new Term with a floating point Number value. +func FloatNumberTerm(f float64) *Term { + s := strconv.FormatFloat(f, 'g', -1, 64) + return &Term{Value: Number(s)} +} + +// Equal returns true if the other Value is a Number and is equal. +func (num Number) Equal(other Value) bool { + switch other := other.(type) { + case Number: + return Compare(num, other) == 0 + default: + return false + } +} + +// Compare compares num to other, return <0, 0, or >0 if it is less than, equal to, +// or greater than other. +func (num Number) Compare(other Value) int { + return Compare(num, other) +} + +// Find returns the current value or a not found error. +func (num Number) Find(path Ref) (Value, error) { + if len(path) == 0 { + return num, nil + } + return nil, errFindNotFound +} + +// Hash returns the hash code for the Value. +func (num Number) Hash() int { + f, err := json.Number(num).Float64() + if err != nil { + bs := []byte(num) + h := xxhash.Checksum64(bs) + return int(h) + } + return int(f) +} + +// Int returns the int representation of num if possible. +func (num Number) Int() (int, bool) { + i64, ok := num.Int64() + return int(i64), ok +} + +// Int64 returns the int64 representation of num if possible. +func (num Number) Int64() (int64, bool) { + i, err := json.Number(num).Int64() + if err != nil { + return 0, false + } + return i, true +} + +// Float64 returns the float64 representation of num if possible. +func (num Number) Float64() (float64, bool) { + f, err := json.Number(num).Float64() + if err != nil { + return 0, false + } + return f, true +} + +// IsGround always returns true. +func (num Number) IsGround() bool { + return true +} + +// MarshalJSON returns JSON encoded bytes representing num. +func (num Number) MarshalJSON() ([]byte, error) { + return json.Marshal(json.Number(num)) +} + +func (num Number) String() string { + return string(num) +} + +func intNumber(i int) Number { + return Number(strconv.Itoa(i)) +} + +func int64Number(i int64) Number { + return Number(strconv.FormatInt(i, 10)) +} + +func floatNumber(f float64) Number { + return Number(strconv.FormatFloat(f, 'g', -1, 64)) +} + +// String represents a string value as defined by JSON. +type String string + +// StringTerm creates a new Term with a String value. +func StringTerm(s string) *Term { + return &Term{Value: String(s)} +} + +// Equal returns true if the other Value is a String and is equal. +func (str String) Equal(other Value) bool { + switch other := other.(type) { + case String: + return str == other + default: + return false + } +} + +// Compare compares str to other, return <0, 0, or >0 if it is less than, equal to, +// or greater than other. +func (str String) Compare(other Value) int { + return Compare(str, other) +} + +// Find returns the current value or a not found error. +func (str String) Find(path Ref) (Value, error) { + if len(path) == 0 { + return str, nil + } + return nil, errFindNotFound +} + +// IsGround always returns true. +func (str String) IsGround() bool { + return true +} + +func (str String) String() string { + return strconv.Quote(string(str)) +} + +// Hash returns the hash code for the Value. +func (str String) Hash() int { + h := xxhash.ChecksumString64S(string(str), hashSeed0) + return int(h) +} + +// Var represents a variable as defined by the language. +type Var string + +// VarTerm creates a new Term with a Variable value. +func VarTerm(v string) *Term { + return &Term{Value: Var(v)} +} + +// Equal returns true if the other Value is a Variable and has the same value +// (name). +func (v Var) Equal(other Value) bool { + switch other := other.(type) { + case Var: + return v == other + default: + return false + } +} + +// Compare compares v to other, return <0, 0, or >0 if it is less than, equal to, +// or greater than other. +func (v Var) Compare(other Value) int { + return Compare(v, other) +} + +// Find returns the current value or a not found error. +func (v Var) Find(path Ref) (Value, error) { + if len(path) == 0 { + return v, nil + } + return nil, errFindNotFound +} + +// Hash returns the hash code for the Value. +func (v Var) Hash() int { + h := xxhash.ChecksumString64S(string(v), hashSeed0) + return int(h) +} + +// IsGround always returns false. +func (v Var) IsGround() bool { + return false +} + +// IsWildcard returns true if this is a wildcard variable. +func (v Var) IsWildcard() bool { + return strings.HasPrefix(string(v), WildcardPrefix) +} + +// IsGenerated returns true if this variable was generated during compilation. +func (v Var) IsGenerated() bool { + return strings.HasPrefix(string(v), "__local") +} + +func (v Var) String() string { + // Special case for wildcard so that string representation is parseable. The + // parser mangles wildcard variables to make their names unique and uses an + // illegal variable name character (WildcardPrefix) to avoid conflicts. When + // we serialize the variable here, we need to make sure it's parseable. + if v.IsWildcard() { + return Wildcard.String() + } + return string(v) +} + +// Ref represents a reference as defined by the language. +type Ref []*Term + +// EmptyRef returns a new, empty reference. +func EmptyRef() Ref { + return Ref([]*Term{}) +} + +// PtrRef returns a new reference against the head for the pointer +// s. Path components in the pointer are unescaped. +func PtrRef(head *Term, s string) (Ref, error) { + s = strings.Trim(s, "/") + if s == "" { + return Ref{head}, nil + } + parts := strings.Split(s, "/") + ref := make(Ref, len(parts)+1) + ref[0] = head + for i := 0; i < len(parts); i++ { + var err error + parts[i], err = url.PathUnescape(parts[i]) + if err != nil { + return nil, err + } + ref[i+1] = StringTerm(parts[i]) + } + return ref, nil +} + +// RefTerm creates a new Term with a Ref value. +func RefTerm(r ...*Term) *Term { + return &Term{Value: Ref(r)} +} + +// Append returns a copy of ref with the term appended to the end. +func (ref Ref) Append(term *Term) Ref { + n := len(ref) + dst := make(Ref, n+1) + copy(dst, ref) + dst[n] = term + return dst +} + +// Insert returns a copy of the ref with x inserted at pos. If pos < len(ref), +// existing elements are shifted to the right. If pos > len(ref)+1 this +// function panics. +func (ref Ref) Insert(x *Term, pos int) Ref { + if pos == len(ref) { + return ref.Append(x) + } else if pos > len(ref)+1 { + panic("illegal index") + } + cpy := make(Ref, len(ref)+1) + for i := 0; i < pos; i++ { + cpy[i] = ref[i] + } + cpy[pos] = x + for i := pos; i < len(ref); i++ { + cpy[i+1] = ref[i] + } + return cpy +} + +// Extend returns a copy of ref with the terms from other appended. The head of +// other will be converted to a string. +func (ref Ref) Extend(other Ref) Ref { + dst := make(Ref, len(ref)+len(other)) + for i := range ref { + dst[i] = ref[i] + } + head := other[0].Copy() + head.Value = String(head.Value.(Var)) + offset := len(ref) + dst[offset] = head + for i := range other[1:] { + dst[offset+i+1] = other[i+1] + } + return dst +} + +// Concat returns a ref with the terms appended. +func (ref Ref) Concat(terms []*Term) Ref { + if len(terms) == 0 { + return ref + } + cpy := make(Ref, len(ref)+len(terms)) + for i := range ref { + cpy[i] = ref[i] + } + for i := range terms { + cpy[len(ref)+i] = terms[i] + } + return cpy +} + +// Dynamic returns the offset of the first non-constant operand of ref. +func (ref Ref) Dynamic() int { + switch ref[0].Value.(type) { + case Call: + return 0 + } + for i := 1; i < len(ref); i++ { + if !IsConstant(ref[i].Value) { + return i + } + } + return -1 +} + +// Copy returns a deep copy of ref. +func (ref Ref) Copy() Ref { + return termSliceCopy(ref) +} + +// Equal returns true if ref is equal to other. +func (ref Ref) Equal(other Value) bool { + return Compare(ref, other) == 0 +} + +// Compare compares ref to other, return <0, 0, or >0 if it is less than, equal to, +// or greater than other. +func (ref Ref) Compare(other Value) int { + return Compare(ref, other) +} + +// Find returns the current value or a not found error. +func (ref Ref) Find(path Ref) (Value, error) { + if len(path) == 0 { + return ref, nil + } + return nil, errFindNotFound +} + +// Hash returns the hash code for the Value. +func (ref Ref) Hash() int { + return termSliceHash(ref) +} + +// HasPrefix returns true if the other ref is a prefix of this ref. +func (ref Ref) HasPrefix(other Ref) bool { + if len(other) > len(ref) { + return false + } + for i := range other { + if !ref[i].Equal(other[i]) { + return false + } + } + return true +} + +// ConstantPrefix returns the constant portion of the ref starting from the head. +func (ref Ref) ConstantPrefix() Ref { + ref = ref.Copy() + + i := ref.Dynamic() + if i < 0 { + return ref + } + return ref[:i] +} + +// GroundPrefix returns the ground portion of the ref starting from the head. By +// definition, the head of the reference is always ground. +func (ref Ref) GroundPrefix() Ref { + prefix := make(Ref, 0, len(ref)) + + for i, x := range ref { + if i > 0 && !x.IsGround() { + break + } + prefix = append(prefix, x) + } + + return prefix +} + +// IsGround returns true if all of the parts of the Ref are ground. +func (ref Ref) IsGround() bool { + if len(ref) == 0 { + return true + } + return termSliceIsGround(ref[1:]) +} + +// IsNested returns true if this ref contains other Refs. +func (ref Ref) IsNested() bool { + for _, x := range ref { + if _, ok := x.Value.(Ref); ok { + return true + } + } + return false +} + +// Ptr returns a slash-separated path string for this ref. If the ref +// contains non-string terms this function returns an error. Path +// components are escaped. +func (ref Ref) Ptr() (string, error) { + parts := make([]string, 0, len(ref)-1) + for _, term := range ref[1:] { + if str, ok := term.Value.(String); ok { + parts = append(parts, url.PathEscape(string(str))) + } else { + return "", fmt.Errorf("invalid path value type") + } + } + return strings.Join(parts, "/"), nil +} + +var varRegexp = regexp.MustCompile("^[[:alpha:]_][[:alpha:][:digit:]_]*$") + +func (ref Ref) String() string { + if len(ref) == 0 { + return "" + } + buf := []string{ref[0].Value.String()} + path := ref[1:] + for _, p := range path { + switch p := p.Value.(type) { + case String: + str := string(p) + if varRegexp.MatchString(str) && len(buf) > 0 && !IsKeyword(str) { + buf = append(buf, "."+str) + } else { + buf = append(buf, "["+p.String()+"]") + } + default: + buf = append(buf, "["+p.String()+"]") + } + } + return strings.Join(buf, "") +} + +// OutputVars returns a VarSet containing variables that would be bound by evaluating +// this expression in isolation. +func (ref Ref) OutputVars() VarSet { + vis := NewVarVisitor().WithParams(VarVisitorParams{SkipRefHead: true}) + vis.Walk(ref) + return vis.Vars() +} + +// QueryIterator defines the interface for querying AST documents with references. +type QueryIterator func(map[Var]Value, Value) error + +// Array represents an array as defined by the language. Arrays are similar to the +// same types as defined by JSON with the exception that they can contain Vars +// and References. +type Array []*Term + +// ArrayTerm creates a new Term with an Array value. +func ArrayTerm(a ...*Term) *Term { + return &Term{Value: Array(a)} +} + +// Copy returns a deep copy of arr. +func (arr Array) Copy() Array { + return termSliceCopy(arr) +} + +// Equal returns true if arr is equal to other. +func (arr Array) Equal(other Value) bool { + return Compare(arr, other) == 0 +} + +// Compare compares arr to other, return <0, 0, or >0 if it is less than, equal to, +// or greater than other. +func (arr Array) Compare(other Value) int { + return Compare(arr, other) +} + +// Find returns the value at the index or an out-of-range error. +func (arr Array) Find(path Ref) (Value, error) { + if len(path) == 0 { + return arr, nil + } + num, ok := path[0].Value.(Number) + if !ok { + return nil, errFindNotFound + } + i, ok := num.Int() + if !ok { + return nil, errFindNotFound + } + if i < 0 || i >= len(arr) { + return nil, errFindNotFound + } + return arr[i].Value.Find(path[1:]) +} + +// Get returns the element at pos or nil if not possible. +func (arr Array) Get(pos *Term) *Term { + num, ok := pos.Value.(Number) + if !ok { + return nil + } + + i, ok := num.Int() + if !ok { + return nil + } + + if i >= 0 && i < len(arr) { + return arr[i] + } + + return nil +} + +// Sorted returns a new Array that contains the sorted elements of arr. +func (arr Array) Sorted() Array { + cpy := make(Array, len(arr)) + for i := range cpy { + cpy[i] = arr[i] + } + sort.Sort(termSlice(cpy)) + return cpy +} + +// Hash returns the hash code for the Value. +func (arr Array) Hash() int { + return termSliceHash(arr) +} + +// IsGround returns true if all of the Array elements are ground. +func (arr Array) IsGround() bool { + return termSliceIsGround(arr) +} + +// MarshalJSON returns JSON encoded bytes representing arr. +func (arr Array) MarshalJSON() ([]byte, error) { + if len(arr) == 0 { + return json.Marshal([]interface{}{}) + } + return json.Marshal([]*Term(arr)) +} + +func (arr Array) String() string { + var buf []string + for _, e := range arr { + buf = append(buf, e.String()) + } + return "[" + strings.Join(buf, ", ") + "]" +} + +// Set represents a set as defined by the language. +type Set interface { + Value + Len() int + Copy() Set + Diff(Set) Set + Intersect(Set) Set + Union(Set) Set + Add(*Term) + Iter(func(*Term) error) error + Until(func(*Term) bool) bool + Foreach(func(*Term)) + Contains(*Term) bool + Map(func(*Term) (*Term, error)) (Set, error) + Reduce(*Term, func(*Term, *Term) (*Term, error)) (*Term, error) + Sorted() Array + Slice() []*Term +} + +// NewSet returns a new Set containing t. +func NewSet(t ...*Term) Set { + s := newset(len(t)) + for i := range t { + s.Add(t[i]) + } + return s +} + +func newset(n int) *set { + var keys []*Term + if n > 0 { + keys = make([]*Term, 0, n) + } + return &set{ + elems: make(map[int]*Term, n), + keys: keys, + } +} + +// SetTerm returns a new Term representing a set containing terms t. +func SetTerm(t ...*Term) *Term { + set := NewSet(t...) + return &Term{ + Value: set, + } +} + +type set struct { + elems map[int]*Term + keys []*Term +} + +// Copy returns a deep copy of s. +func (s *set) Copy() Set { + cpy := newset(s.Len()) + s.Foreach(func(x *Term) { + cpy.Add(x.Copy()) + }) + return cpy +} + +// IsGround returns true if all terms in s are ground. +func (s *set) IsGround() bool { + return !s.Until(func(x *Term) bool { + return !x.IsGround() + }) +} + +// Hash returns a hash code for s. +func (s *set) Hash() int { + var hash int + s.Foreach(func(x *Term) { + hash += x.Hash() + }) + return hash +} + +func (s *set) String() string { + if s.Len() == 0 { + return "set()" + } + buf := []string{} + s.Foreach(func(x *Term) { + buf = append(buf, fmt.Sprint(x)) + }) + return "{" + strings.Join(buf, ", ") + "}" +} + +// Compare compares s to other, return <0, 0, or >0 if it is less than, equal to, +// or greater than other. +func (s *set) Compare(other Value) int { + o1 := sortOrder(s) + o2 := sortOrder(other) + if o1 < o2 { + return -1 + } else if o1 > o2 { + return 1 + } + t := other.(*set) + sort.Sort(termSlice(s.keys)) + sort.Sort(termSlice(t.keys)) + return termSliceCompare(s.keys, t.keys) +} + +// Find returns the set or dereferences the element itself. +func (s *set) Find(path Ref) (Value, error) { + if len(path) == 0 { + return s, nil + } + if !s.Contains(path[0]) { + return nil, errFindNotFound + } + return path[0].Value.Find(path[1:]) +} + +// Diff returns elements in s that are not in other. +func (s *set) Diff(other Set) Set { + r := NewSet() + s.Foreach(func(x *Term) { + if !other.Contains(x) { + r.Add(x) + } + }) + return r +} + +// Intersect returns the set containing elements in both s and other. +func (s *set) Intersect(other Set) Set { + o := other.(*set) + n, m := s.Len(), o.Len() + ss := s + so := o + if m < n { + ss = o + so = s + n = m + } + + r := newset(n) + ss.Foreach(func(x *Term) { + if so.Contains(x) { + r.Add(x) + } + }) + return r +} + +// Union returns the set containing all elements of s and other. +func (s *set) Union(other Set) Set { + r := NewSet() + s.Foreach(func(x *Term) { + r.Add(x) + }) + other.Foreach(func(x *Term) { + r.Add(x) + }) + return r +} + +// Add updates s to include t. +func (s *set) Add(t *Term) { + s.insert(t) +} + +// Iter calls f on each element in s. If f returns an error, iteration stops +// and the return value is the error. +func (s *set) Iter(f func(*Term) error) error { + for i := range s.keys { + if err := f(s.keys[i]); err != nil { + return err + } + } + return nil +} + +var errStop = errors.New("stop") + +// Until calls f on each element in s. If f returns true, iteration stops. +func (s *set) Until(f func(*Term) bool) bool { + err := s.Iter(func(t *Term) error { + if f(t) { + return errStop + } + return nil + }) + return err != nil +} + +// Foreach calls f on each element in s. +func (s *set) Foreach(f func(*Term)) { + s.Iter(func(t *Term) error { + f(t) + return nil + }) +} + +// Map returns a new Set obtained by applying f to each value in s. +func (s *set) Map(f func(*Term) (*Term, error)) (Set, error) { + set := NewSet() + err := s.Iter(func(x *Term) error { + term, err := f(x) + if err != nil { + return err + } + set.Add(term) + return nil + }) + if err != nil { + return nil, err + } + return set, nil +} + +// Reduce returns a Term produced by applying f to each value in s. The first +// argument to f is the reduced value (starting with i) and the second argument +// to f is the element in s. +func (s *set) Reduce(i *Term, f func(*Term, *Term) (*Term, error)) (*Term, error) { + err := s.Iter(func(x *Term) error { + var err error + i, err = f(i, x) + if err != nil { + return err + } + return nil + }) + return i, err +} + +// Contains returns true if t is in s. +func (s *set) Contains(t *Term) bool { + return s.get(t) != nil +} + +// Len returns the number of elements in the set. +func (s *set) Len() int { + return len(s.keys) +} + +// MarshalJSON returns JSON encoded bytes representing s. +func (s *set) MarshalJSON() ([]byte, error) { + if s.keys == nil { + return json.Marshal([]interface{}{}) + } + return json.Marshal(s.keys) +} + +// Sorted returns an Array that contains the sorted elements of s. +func (s *set) Sorted() Array { + cpy := make(Array, len(s.keys)) + for i := range cpy { + cpy[i] = s.keys[i] + } + sort.Sort(termSlice(cpy)) + return cpy +} + +// Slice returns a slice of terms contained in the set. +func (s *set) Slice() []*Term { + return s.keys +} + +func (s *set) insert(x *Term) { + hash := x.Hash() + var equal func(v Value) bool + + switch x := x.Value.(type) { + case Null, Boolean, String, Var: + equal = func(y Value) bool { return x == y } + case Number: + if xi, err := json.Number(x).Int64(); err == nil { + equal = func(y Value) bool { + if y, ok := y.(Number); ok { + if yi, err := json.Number(y).Int64(); err == nil { + return xi == yi + } + } + + return false + } + break + } + + a, ok := new(big.Float).SetString(string(x)) + if !ok { + panic("illegal value") + } + + equal = func(b Value) bool { + if b, ok := b.(Number); ok { + b, ok := new(big.Float).SetString(string(b)) + if !ok { + panic("illegal value") + } + + return a.Cmp(b) == 0 + } + + return false + } + default: + equal = func(y Value) bool { return Compare(x, y) == 0 } + } + + for curr, ok := s.elems[hash]; ok; { + if equal(curr.Value) { + return + } + + hash++ + curr, ok = s.elems[hash] + } + + s.elems[hash] = x + s.keys = append(s.keys, x) +} + +func (s *set) get(x *Term) *Term { + hash := x.Hash() + var equal func(v Value) bool + + switch x := x.Value.(type) { + case Null, Boolean, String, Var: + equal = func(y Value) bool { return x == y } + case Number: + if xi, err := json.Number(x).Int64(); err == nil { + equal = func(y Value) bool { + if y, ok := y.(Number); ok { + if yi, err := json.Number(y).Int64(); err == nil { + return xi == yi + } + } + + return false + } + break + } + + a, ok := new(big.Float).SetString(string(x)) + if !ok { + panic("illegal value") + } + + equal = func(b Value) bool { + if b, ok := b.(Number); ok { + b, ok := new(big.Float).SetString(string(b)) + if !ok { + panic("illegal value") + } + + return a.Cmp(b) == 0 + } + + return false + } + default: + equal = func(y Value) bool { return Compare(x, y) == 0 } + } + + for curr, ok := s.elems[hash]; ok; { + if equal(curr.Value) { + return curr + } + + hash++ + curr, ok = s.elems[hash] + } + return nil +} + +// Object represents an object as defined by the language. +type Object interface { + Value + Len() int + Get(*Term) *Term + Copy() Object + Insert(*Term, *Term) + Iter(func(*Term, *Term) error) error + Until(func(*Term, *Term) bool) bool + Foreach(func(*Term, *Term)) + Map(func(*Term, *Term) (*Term, *Term, error)) (Object, error) + Diff(other Object) Object + Intersect(other Object) [][3]*Term + Merge(other Object) (Object, bool) + MergeWith(other Object, conflictResolver func(v1, v2 *Term) (*Term, bool)) (Object, bool) + Filter(filter Object) (Object, error) + Keys() []*Term +} + +// NewObject creates a new Object with t. +func NewObject(t ...[2]*Term) Object { + obj := newobject(len(t)) + for i := range t { + obj.Insert(t[i][0], t[i][1]) + } + return obj +} + +// ObjectTerm creates a new Term with an Object value. +func ObjectTerm(o ...[2]*Term) *Term { + return &Term{Value: NewObject(o...)} +} + +type object struct { + elems map[int]*objectElem + keys []*Term + ground bool +} + +func newobject(n int) *object { + var keys []*Term + if n > 0 { + keys = make([]*Term, 0, n) + } + return &object{ + elems: make(map[int]*objectElem, n), + keys: keys, + ground: true, + } +} + +type objectElem struct { + key *Term + value *Term + next *objectElem +} + +// Item is a helper for constructing an tuple containing two Terms +// representing a key/value pair in an Object. +func Item(key, value *Term) [2]*Term { + return [2]*Term{key, value} +} + +// Compare compares obj to other, return <0, 0, or >0 if it is less than, equal to, +// or greater than other. +func (obj *object) Compare(other Value) int { + o1 := sortOrder(obj) + o2 := sortOrder(other) + if o1 < o2 { + return -1 + } else if o2 < o1 { + return 1 + } + a := obj + b := other.(*object) + keysA := a.Keys() + keysB := b.Keys() + sort.Sort(termSlice(keysA)) + sort.Sort(termSlice(keysB)) + minLen := a.Len() + if b.Len() < a.Len() { + minLen = b.Len() + } + for i := 0; i < minLen; i++ { + keysCmp := Compare(keysA[i], keysB[i]) + if keysCmp < 0 { + return -1 + } + if keysCmp > 0 { + return 1 + } + valA := a.Get(keysA[i]) + valB := b.Get(keysB[i]) + valCmp := Compare(valA, valB) + if valCmp != 0 { + return valCmp + } + } + if a.Len() < b.Len() { + return -1 + } + if b.Len() < a.Len() { + return 1 + } + return 0 +} + +// Find returns the value at the key or undefined. +func (obj *object) Find(path Ref) (Value, error) { + if len(path) == 0 { + return obj, nil + } + value := obj.Get(path[0]) + if value == nil { + return nil, errFindNotFound + } + return value.Value.Find(path[1:]) +} + +func (obj *object) Insert(k, v *Term) { + obj.insert(k, v) +} + +// Get returns the value of k in obj if k exists, otherwise nil. +func (obj *object) Get(k *Term) *Term { + if elem := obj.get(k); elem != nil { + return elem.value + } + return nil +} + +// Hash returns the hash code for the Value. +func (obj *object) Hash() int { + var hash int + obj.Foreach(func(k, v *Term) { + hash += k.Value.Hash() + hash += v.Value.Hash() + }) + return hash +} + +// IsGround returns true if all of the Object key/value pairs are ground. +func (obj *object) IsGround() bool { + return obj.ground +} + +// Copy returns a deep copy of obj. +func (obj *object) Copy() Object { + cpy, _ := obj.Map(func(k, v *Term) (*Term, *Term, error) { + return k.Copy(), v.Copy(), nil + }) + return cpy +} + +// Diff returns a new Object that contains only the key/value pairs that exist in obj. +func (obj *object) Diff(other Object) Object { + r := NewObject() + obj.Foreach(func(k, v *Term) { + if other.Get(k) == nil { + r.Insert(k, v) + } + }) + return r +} + +// Intersect returns a slice of term triplets that represent the intersection of keys +// between obj and other. For each intersecting key, the values from obj and other are included +// as the last two terms in the triplet (respectively). +func (obj *object) Intersect(other Object) [][3]*Term { + r := [][3]*Term{} + obj.Foreach(func(k, v *Term) { + if v2 := other.Get(k); v2 != nil { + r = append(r, [3]*Term{k, v, v2}) + } + }) + return r +} + +// Iter calls the function f for each key-value pair in the object. If f +// returns an error, iteration stops and the error is returned. +func (obj *object) Iter(f func(*Term, *Term) error) error { + for i := range obj.keys { + k := obj.keys[i] + node := obj.get(k) + if node == nil { + panic("corrupt object") + } + if err := f(k, node.value); err != nil { + return err + } + } + return nil +} + +// Until calls f for each key-value pair in the object. If f returns true, +// iteration stops. +func (obj *object) Until(f func(*Term, *Term) bool) bool { + err := obj.Iter(func(k, v *Term) error { + if f(k, v) { + return errStop + } + return nil + }) + return err != nil +} + +// Foreach calls f for each key-value pair in the object. +func (obj *object) Foreach(f func(*Term, *Term)) { + obj.Iter(func(k, v *Term) error { + f(k, v) + return nil + }) +} + +// Map returns a new Object constructed by mapping each element in the object +// using the function f. +func (obj *object) Map(f func(*Term, *Term) (*Term, *Term, error)) (Object, error) { + cpy := newobject(obj.Len()) + err := obj.Iter(func(k, v *Term) error { + var err error + k, v, err = f(k, v) + if err != nil { + return err + } + cpy.insert(k, v) + return nil + }) + if err != nil { + return nil, err + } + return cpy, nil +} + +// Keys returns the keys of obj. +func (obj *object) Keys() []*Term { + return obj.keys +} + +// MarshalJSON returns JSON encoded bytes representing obj. +func (obj *object) MarshalJSON() ([]byte, error) { + sl := make([][2]*Term, obj.Len()) + for i := range obj.keys { + k := obj.keys[i] + sl[i] = Item(k, obj.get(k).value) + } + return json.Marshal(sl) +} + +// Merge returns a new Object containing the non-overlapping keys of obj and other. If there are +// overlapping keys between obj and other, the values of associated with the keys are merged. Only +// objects can be merged with other objects. If the values cannot be merged, the second turn value +// will be false. +func (obj object) Merge(other Object) (Object, bool) { + return obj.MergeWith(other, func(v1, v2 *Term) (*Term, bool) { + obj1, ok1 := v1.Value.(Object) + obj2, ok2 := v2.Value.(Object) + if !ok1 || !ok2 { + return nil, true + } + obj3, ok := obj1.Merge(obj2) + if !ok { + return nil, true + } + return NewTerm(obj3), false + }) +} + +// MergeWith returns a new Object containing the merged keys of obj and other. +// If there are overlapping keys between obj and other, the conflictResolver +// is called. The conflictResolver can return a merged value and a boolean +// indicating if the merge has failed and should stop. +func (obj object) MergeWith(other Object, conflictResolver func(v1, v2 *Term) (*Term, bool)) (Object, bool) { + result := NewObject() + stop := obj.Until(func(k, v *Term) bool { + v2 := other.Get(k) + // The key didn't exist in other, keep the original value + if v2 == nil { + result.Insert(k, v) + return false + } + + // The key exists in both, resolve the conflict if possible + merged, stop := conflictResolver(v, v2) + if !stop { + result.Insert(k, merged) + } + return stop + }) + + if stop { + return nil, false + } + + // Copy in any values from other for keys that don't exist in obj + other.Foreach(func(k, v *Term) { + if v2 := obj.Get(k); v2 == nil { + result.Insert(k, v) + } + }) + return result, true +} + +// Filter returns a new object from values in obj where the keys are +// found in filter. Array indices for values can be specified as +// number strings. +func (obj *object) Filter(filter Object) (Object, error) { + filtered, err := filterObject(obj, filter) + if err != nil { + return nil, err + } + return filtered.(Object), nil +} + +// Len returns the number of elements in the object. +func (obj object) Len() int { + return len(obj.keys) +} + +func (obj object) String() string { + var buf []string + obj.Foreach(func(k, v *Term) { + buf = append(buf, fmt.Sprintf("%s: %s", k, v)) + }) + return "{" + strings.Join(buf, ", ") + "}" +} + +func (obj *object) get(k *Term) *objectElem { + hash := k.Hash() + + var equal func(v Value) bool + + switch x := k.Value.(type) { + case Null, Boolean, String, Var: + equal = func(y Value) bool { return x == y } + case Number: + if xi, err := json.Number(x).Int64(); err == nil { + equal = func(y Value) bool { + if y, ok := y.(Number); ok { + if yi, err := json.Number(y).Int64(); err == nil { + return xi == yi + } + } + + return false + } + break + } + + a, ok := new(big.Float).SetString(string(x)) + if !ok { + panic("illegal value") + } + + equal = func(b Value) bool { + if b, ok := b.(Number); ok { + b, ok := new(big.Float).SetString(string(b)) + if !ok { + panic("illegal value") + } + + return a.Cmp(b) == 0 + } + + return false + } + default: + equal = func(y Value) bool { return Compare(x, y) == 0 } + } + + for curr := obj.elems[hash]; curr != nil; curr = curr.next { + if equal(curr.key.Value) { + return curr + } + } + return nil +} + +func (obj *object) insert(k, v *Term) { + hash := k.Hash() + head := obj.elems[hash] + var equal func(v Value) bool + + switch x := k.Value.(type) { + case Null, Boolean, String, Var: + equal = func(y Value) bool { return x == y } + case Number: + if xi, err := json.Number(x).Int64(); err == nil { + equal = func(y Value) bool { + if y, ok := y.(Number); ok { + if yi, err := json.Number(y).Int64(); err == nil { + return xi == yi + } + } + + return false + } + break + } + + a, ok := new(big.Float).SetString(string(x)) + if !ok { + panic("illegal value") + } + + equal = func(b Value) bool { + if b, ok := b.(Number); ok { + b, ok := new(big.Float).SetString(string(b)) + if !ok { + panic("illegal value") + } + + return a.Cmp(b) == 0 + } + + return false + } + default: + equal = func(y Value) bool { return Compare(x, y) == 0 } + } + + for curr := head; curr != nil; curr = curr.next { + if equal(curr.key.Value) { + curr.value = v + return + } + } + obj.elems[hash] = &objectElem{ + key: k, + value: v, + next: head, + } + obj.keys = append(obj.keys, k) + obj.ground = obj.ground && k.IsGround() && v.IsGround() +} + +func filterObject(o Value, filter Value) (Value, error) { + if filter.Compare(Null{}) == 0 { + return o, nil + } + + filteredObj, ok := filter.(Object) + if !ok { + return nil, fmt.Errorf("invalid filter value %q, expected an object", filter) + } + + switch v := o.(type) { + case String, Number, Boolean, Null: + return o, nil + case Array: + var values Array + for i, t := range v { + subFilter := filteredObj.Get(StringTerm(strconv.Itoa(i))) + if subFilter != nil { + filteredValue, err := filterObject(t.Value, subFilter.Value) + if err != nil { + return nil, err + } + values = append(values, NewTerm(filteredValue)) + } + } + return values, nil + case Set: + values := NewSet() + err := v.Iter(func(t *Term) error { + if filteredObj.Get(t) != nil { + filteredValue, err := filterObject(t.Value, filteredObj.Get(t).Value) + if err != nil { + return err + } + values.Add(NewTerm(filteredValue)) + } + return nil + }) + return values, err + case Object: + values := NewObject() + + iterObj := v + other := filteredObj + if v.Len() < filteredObj.Len() { + iterObj = filteredObj + other = v + } + + err := iterObj.Iter(func(key *Term, value *Term) error { + if other.Get(key) != nil { + filteredValue, err := filterObject(v.Get(key).Value, filteredObj.Get(key).Value) + if err != nil { + return err + } + values.Insert(key, NewTerm(filteredValue)) + } + return nil + }) + return values, err + default: + return nil, fmt.Errorf("invalid object value type %q", v) + } +} + +// ArrayComprehension represents an array comprehension as defined in the language. +type ArrayComprehension struct { + Term *Term `json:"term"` + Body Body `json:"body"` +} + +// ArrayComprehensionTerm creates a new Term with an ArrayComprehension value. +func ArrayComprehensionTerm(term *Term, body Body) *Term { + return &Term{ + Value: &ArrayComprehension{ + Term: term, + Body: body, + }, + } +} + +// Copy returns a deep copy of ac. +func (ac *ArrayComprehension) Copy() *ArrayComprehension { + cpy := *ac + cpy.Body = ac.Body.Copy() + cpy.Term = ac.Term.Copy() + return &cpy +} + +// Equal returns true if ac is equal to other. +func (ac *ArrayComprehension) Equal(other Value) bool { + return Compare(ac, other) == 0 +} + +// Compare compares ac to other, return <0, 0, or >0 if it is less than, equal to, +// or greater than other. +func (ac *ArrayComprehension) Compare(other Value) int { + return Compare(ac, other) +} + +// Find returns the current value or a not found error. +func (ac *ArrayComprehension) Find(path Ref) (Value, error) { + if len(path) == 0 { + return ac, nil + } + return nil, errFindNotFound +} + +// Hash returns the hash code of the Value. +func (ac *ArrayComprehension) Hash() int { + return ac.Term.Hash() + ac.Body.Hash() +} + +// IsGround returns true if the Term and Body are ground. +func (ac *ArrayComprehension) IsGround() bool { + return ac.Term.IsGround() && ac.Body.IsGround() +} + +func (ac *ArrayComprehension) String() string { + return "[" + ac.Term.String() + " | " + ac.Body.String() + "]" +} + +// ObjectComprehension represents an object comprehension as defined in the language. +type ObjectComprehension struct { + Key *Term `json:"key"` + Value *Term `json:"value"` + Body Body `json:"body"` +} + +// ObjectComprehensionTerm creates a new Term with an ObjectComprehension value. +func ObjectComprehensionTerm(key, value *Term, body Body) *Term { + return &Term{ + Value: &ObjectComprehension{ + Key: key, + Value: value, + Body: body, + }, + } +} + +// Copy returns a deep copy of oc. +func (oc *ObjectComprehension) Copy() *ObjectComprehension { + cpy := *oc + cpy.Body = oc.Body.Copy() + cpy.Key = oc.Key.Copy() + cpy.Value = oc.Value.Copy() + return &cpy +} + +// Equal returns true if oc is equal to other. +func (oc *ObjectComprehension) Equal(other Value) bool { + return Compare(oc, other) == 0 +} + +// Compare compares oc to other, return <0, 0, or >0 if it is less than, equal to, +// or greater than other. +func (oc *ObjectComprehension) Compare(other Value) int { + return Compare(oc, other) +} + +// Find returns the current value or a not found error. +func (oc *ObjectComprehension) Find(path Ref) (Value, error) { + if len(path) == 0 { + return oc, nil + } + return nil, errFindNotFound +} + +// Hash returns the hash code of the Value. +func (oc *ObjectComprehension) Hash() int { + return oc.Key.Hash() + oc.Value.Hash() + oc.Body.Hash() +} + +// IsGround returns true if the Key, Value and Body are ground. +func (oc *ObjectComprehension) IsGround() bool { + return oc.Key.IsGround() && oc.Value.IsGround() && oc.Body.IsGround() +} + +func (oc *ObjectComprehension) String() string { + return "{" + oc.Key.String() + ": " + oc.Value.String() + " | " + oc.Body.String() + "}" +} + +// SetComprehension represents a set comprehension as defined in the language. +type SetComprehension struct { + Term *Term `json:"term"` + Body Body `json:"body"` +} + +// SetComprehensionTerm creates a new Term with an SetComprehension value. +func SetComprehensionTerm(term *Term, body Body) *Term { + return &Term{ + Value: &SetComprehension{ + Term: term, + Body: body, + }, + } +} + +// Copy returns a deep copy of sc. +func (sc *SetComprehension) Copy() *SetComprehension { + cpy := *sc + cpy.Body = sc.Body.Copy() + cpy.Term = sc.Term.Copy() + return &cpy +} + +// Equal returns true if sc is equal to other. +func (sc *SetComprehension) Equal(other Value) bool { + return Compare(sc, other) == 0 +} + +// Compare compares sc to other, return <0, 0, or >0 if it is less than, equal to, +// or greater than other. +func (sc *SetComprehension) Compare(other Value) int { + return Compare(sc, other) +} + +// Find returns the current value or a not found error. +func (sc *SetComprehension) Find(path Ref) (Value, error) { + if len(path) == 0 { + return sc, nil + } + return nil, errFindNotFound +} + +// Hash returns the hash code of the Value. +func (sc *SetComprehension) Hash() int { + return sc.Term.Hash() + sc.Body.Hash() +} + +// IsGround returns true if the Term and Body are ground. +func (sc *SetComprehension) IsGround() bool { + return sc.Term.IsGround() && sc.Body.IsGround() +} + +func (sc *SetComprehension) String() string { + return "{" + sc.Term.String() + " | " + sc.Body.String() + "}" +} + +// Call represents as function call in the language. +type Call []*Term + +// CallTerm returns a new Term with a Call value defined by terms. The first +// term is the operator and the rest are operands. +func CallTerm(terms ...*Term) *Term { + return NewTerm(Call(terms)) +} + +// Copy returns a deep copy of c. +func (c Call) Copy() Call { + return termSliceCopy(c) +} + +// Compare compares c to other, return <0, 0, or >0 if it is less than, equal to, +// or greater than other. +func (c Call) Compare(other Value) int { + return Compare(c, other) +} + +// Find returns the current value or a not found error. +func (c Call) Find(Ref) (Value, error) { + return nil, errFindNotFound +} + +// Hash returns the hash code for the Value. +func (c Call) Hash() int { + return termSliceHash(c) +} + +// IsGround returns true if the Value is ground. +func (c Call) IsGround() bool { + return termSliceIsGround(c) +} + +// MakeExpr returns an ew Expr from this call. +func (c Call) MakeExpr(output *Term) *Expr { + terms := []*Term(c) + return NewExpr(append(terms, output)) +} + +func (c Call) String() string { + args := make([]string, len(c)-1) + for i := 1; i < len(c); i++ { + args[i-1] = c[i].String() + } + return fmt.Sprintf("%v(%v)", c[0], strings.Join(args, ", ")) +} + +func termSliceCopy(a []*Term) []*Term { + cpy := make([]*Term, len(a)) + for i := range a { + cpy[i] = a[i].Copy() + } + return cpy +} + +func termSliceEqual(a, b []*Term) bool { + if len(a) == len(b) { + for i := range a { + if !a[i].Equal(b[i]) { + return false + } + } + return true + } + return false +} + +func termSliceHash(a []*Term) int { + var hash int + for _, v := range a { + hash += v.Value.Hash() + } + return hash +} + +func termSliceIsGround(a []*Term) bool { + for _, v := range a { + if !v.IsGround() { + return false + } + } + return true +} + +// NOTE(tsandall): The unmarshalling errors in these functions are not +// helpful for callers because they do not identify the source of the +// unmarshalling error. Because OPA doesn't accept JSON describing ASTs +// from callers, this is acceptable (for now). If that changes in the future, +// the error messages should be revisited. The current approach focuses +// on the happy path and treats all errors the same. If better error +// reporting is needed, the error paths will need to be fleshed out. + +func unmarshalBody(b []interface{}) (Body, error) { + buf := Body{} + for _, e := range b { + if m, ok := e.(map[string]interface{}); ok { + expr := &Expr{} + if err := unmarshalExpr(expr, m); err == nil { + buf = append(buf, expr) + continue + } + } + goto unmarshal_error + } + return buf, nil +unmarshal_error: + return nil, fmt.Errorf("ast: unable to unmarshal body") +} + +func unmarshalExpr(expr *Expr, v map[string]interface{}) error { + if x, ok := v["negated"]; ok { + if b, ok := x.(bool); ok { + expr.Negated = b + } else { + return fmt.Errorf("ast: unable to unmarshal negated field with type: %T (expected true or false)", v["negated"]) + } + } + if err := unmarshalExprIndex(expr, v); err != nil { + return err + } + switch ts := v["terms"].(type) { + case map[string]interface{}: + t, err := unmarshalTerm(ts) + if err != nil { + return err + } + expr.Terms = t + case []interface{}: + terms, err := unmarshalTermSlice(ts) + if err != nil { + return err + } + expr.Terms = terms + default: + return fmt.Errorf(`ast: unable to unmarshal terms field with type: %T (expected {"value": ..., "type": ...} or [{"value": ..., "type": ...}, ...])`, v["terms"]) + } + if x, ok := v["with"]; ok { + if sl, ok := x.([]interface{}); ok { + ws := make([]*With, len(sl)) + for i := range sl { + var err error + ws[i], err = unmarshalWith(sl[i]) + if err != nil { + return err + } + } + expr.With = ws + } + } + return nil +} + +func unmarshalExprIndex(expr *Expr, v map[string]interface{}) error { + if x, ok := v["index"]; ok { + if n, ok := x.(json.Number); ok { + i, err := n.Int64() + if err == nil { + expr.Index = int(i) + return nil + } + } + } + return fmt.Errorf("ast: unable to unmarshal index field with type: %T (expected integer)", v["index"]) +} + +func unmarshalTerm(m map[string]interface{}) (*Term, error) { + v, err := unmarshalValue(m) + if err != nil { + return nil, err + } + return &Term{Value: v}, nil +} + +func unmarshalTermSlice(s []interface{}) ([]*Term, error) { + buf := []*Term{} + for _, x := range s { + if m, ok := x.(map[string]interface{}); ok { + if t, err := unmarshalTerm(m); err == nil { + buf = append(buf, t) + continue + } else { + return nil, err + } + } + return nil, fmt.Errorf("ast: unable to unmarshal term") + } + return buf, nil +} + +func unmarshalTermSliceValue(d map[string]interface{}) ([]*Term, error) { + if s, ok := d["value"].([]interface{}); ok { + return unmarshalTermSlice(s) + } + return nil, fmt.Errorf(`ast: unable to unmarshal term (expected {"value": [...], "type": ...} where type is one of: ref, array, or set)`) +} + +func unmarshalWith(i interface{}) (*With, error) { + if m, ok := i.(map[string]interface{}); ok { + tgt, _ := m["target"].(map[string]interface{}) + target, err := unmarshalTerm(tgt) + if err == nil { + val, _ := m["value"].(map[string]interface{}) + value, err := unmarshalTerm(val) + if err == nil { + return &With{ + Target: target, + Value: value, + }, nil + } + return nil, err + } + return nil, err + } + return nil, fmt.Errorf(`ast: unable to unmarshal with modifier (expected {"target": {...}, "value": {...}})`) +} + +func unmarshalValue(d map[string]interface{}) (Value, error) { + v := d["value"] + switch d["type"] { + case "null": + return Null{}, nil + case "boolean": + if b, ok := v.(bool); ok { + return Boolean(b), nil + } + case "number": + if n, ok := v.(json.Number); ok { + return Number(n), nil + } + case "string": + if s, ok := v.(string); ok { + return String(s), nil + } + case "var": + if s, ok := v.(string); ok { + return Var(s), nil + } + case "ref": + if s, err := unmarshalTermSliceValue(d); err == nil { + return Ref(s), nil + } + case "array": + if s, err := unmarshalTermSliceValue(d); err == nil { + return Array(s), nil + } + case "set": + if s, err := unmarshalTermSliceValue(d); err == nil { + set := NewSet() + for _, x := range s { + set.Add(x) + } + return set, nil + } + case "object": + if s, ok := v.([]interface{}); ok { + buf := NewObject() + for _, x := range s { + if i, ok := x.([]interface{}); ok && len(i) == 2 { + p, err := unmarshalTermSlice(i) + if err == nil { + buf.Insert(p[0], p[1]) + continue + } + } + goto unmarshal_error + } + return buf, nil + } + case "arraycomprehension", "setcomprehension": + if m, ok := v.(map[string]interface{}); ok { + t, ok := m["term"].(map[string]interface{}) + if !ok { + goto unmarshal_error + } + + term, err := unmarshalTerm(t) + if err != nil { + goto unmarshal_error + } + + b, ok := m["body"].([]interface{}) + if !ok { + goto unmarshal_error + } + + body, err := unmarshalBody(b) + if err != nil { + goto unmarshal_error + } + + if d["type"] == "arraycomprehension" { + return &ArrayComprehension{Term: term, Body: body}, nil + } + return &SetComprehension{Term: term, Body: body}, nil + } + case "objectcomprehension": + if m, ok := v.(map[string]interface{}); ok { + k, ok := m["key"].(map[string]interface{}) + if !ok { + goto unmarshal_error + } + + key, err := unmarshalTerm(k) + if err != nil { + goto unmarshal_error + } + + v, ok := m["value"].(map[string]interface{}) + if !ok { + goto unmarshal_error + } + + value, err := unmarshalTerm(v) + if err != nil { + goto unmarshal_error + } + + b, ok := m["body"].([]interface{}) + if !ok { + goto unmarshal_error + } + + body, err := unmarshalBody(b) + if err != nil { + goto unmarshal_error + } + + return &ObjectComprehension{Key: key, Value: value, Body: body}, nil + } + case "call": + if s, err := unmarshalTermSliceValue(d); err == nil { + return Call(s), nil + } + } +unmarshal_error: + return nil, fmt.Errorf("ast: unable to unmarshal term") +} diff --git a/vendor/github.com/open-policy-agent/opa/ast/transform.go b/vendor/github.com/open-policy-agent/opa/ast/transform.go new file mode 100644 index 000000000..8472d2b47 --- /dev/null +++ b/vendor/github.com/open-policy-agent/opa/ast/transform.go @@ -0,0 +1,382 @@ +// Copyright 2016 The OPA Authors. All rights reserved. +// Use of this source code is governed by an Apache2 +// license that can be found in the LICENSE file. + +package ast + +import "fmt" + +// Transformer defines the interface for transforming AST elements. If the +// transformer returns nil and does not indicate an error, the AST element will +// be set to nil and no transformations will be applied to children of the +// element. +type Transformer interface { + Transform(v interface{}) (interface{}, error) +} + +// Transform iterates the AST and calls the Transform function on the +// Transformer t for x before recursing. +func Transform(t Transformer, x interface{}) (interface{}, error) { + + if term, ok := x.(*Term); ok { + return Transform(t, term.Value) + } + + y, err := t.Transform(x) + if err != nil { + return x, err + } + + if y == nil { + return nil, nil + } + + var ok bool + switch y := y.(type) { + case *Module: + p, err := Transform(t, y.Package) + if err != nil { + return nil, err + } + if y.Package, ok = p.(*Package); !ok { + return nil, fmt.Errorf("illegal transform: %T != %T", y.Package, p) + } + for i := range y.Imports { + imp, err := Transform(t, y.Imports[i]) + if err != nil { + return nil, err + } + if y.Imports[i], ok = imp.(*Import); !ok { + return nil, fmt.Errorf("illegal transform: %T != %T", y.Imports[i], imp) + } + } + for i := range y.Rules { + rule, err := Transform(t, y.Rules[i]) + if err != nil { + return nil, err + } + if y.Rules[i], ok = rule.(*Rule); !ok { + return nil, fmt.Errorf("illegal transform: %T != %T", y.Rules[i], rule) + } + } + for i := range y.Comments { + comment, err := Transform(t, y.Comments[i]) + if err != nil { + return nil, err + } + if y.Comments[i], ok = comment.(*Comment); !ok { + return nil, fmt.Errorf("illegal transform: %T != %T", y.Comments[i], comment) + } + } + return y, nil + case *Package: + ref, err := Transform(t, y.Path) + if err != nil { + return nil, err + } + if y.Path, ok = ref.(Ref); !ok { + return nil, fmt.Errorf("illegal transform: %T != %T", y.Path, ref) + } + return y, nil + case *Import: + y.Path, err = transformTerm(t, y.Path) + if err != nil { + return nil, err + } + if y.Alias, err = transformVar(t, y.Alias); err != nil { + return nil, err + } + return y, nil + case *Rule: + if y.Head, err = transformHead(t, y.Head); err != nil { + return nil, err + } + if y.Body, err = transformBody(t, y.Body); err != nil { + return nil, err + } + if y.Else != nil { + rule, err := Transform(t, y.Else) + if err != nil { + return nil, err + } + if y.Else, ok = rule.(*Rule); !ok { + return nil, fmt.Errorf("illegal transform: %T != %T", y.Else, rule) + } + } + return y, nil + case *Head: + if y.Name, err = transformVar(t, y.Name); err != nil { + return nil, err + } + if y.Args, err = transformArgs(t, y.Args); err != nil { + return nil, err + } + if y.Key != nil { + if y.Key, err = transformTerm(t, y.Key); err != nil { + return nil, err + } + } + if y.Value != nil { + if y.Value, err = transformTerm(t, y.Value); err != nil { + return nil, err + } + } + return y, nil + case Args: + for i := range y { + if y[i], err = transformTerm(t, y[i]); err != nil { + return nil, err + } + } + return y, nil + case Body: + for i, e := range y { + e, err := Transform(t, e) + if err != nil { + return nil, err + } + if y[i], ok = e.(*Expr); !ok { + return nil, fmt.Errorf("illegal transform: %T != %T", y[i], e) + } + } + return y, nil + case *Expr: + switch ts := y.Terms.(type) { + case *SomeDecl: + decl, err := Transform(t, ts) + if err != nil { + return nil, err + } + if y.Terms, ok = decl.(*SomeDecl); !ok { + return nil, fmt.Errorf("illegal transform: %T != %T", y, decl) + } + return y, nil + case []*Term: + for i := range ts { + if ts[i], err = transformTerm(t, ts[i]); err != nil { + return nil, err + } + } + case *Term: + if y.Terms, err = transformTerm(t, ts); err != nil { + return nil, err + } + } + for i, w := range y.With { + w, err := Transform(t, w) + if err != nil { + return nil, err + } + if y.With[i], ok = w.(*With); !ok { + return nil, fmt.Errorf("illegal transform: %T != %T", y.With[i], w) + } + } + return y, nil + case *With: + if y.Target, err = transformTerm(t, y.Target); err != nil { + return nil, err + } + if y.Value, err = transformTerm(t, y.Value); err != nil { + return nil, err + } + return y, nil + case Ref: + for i, term := range y { + if y[i], err = transformTerm(t, term); err != nil { + return nil, err + } + } + return y, nil + case Object: + return y.Map(func(k, v *Term) (*Term, *Term, error) { + k, err := transformTerm(t, k) + if err != nil { + return nil, nil, err + } + v, err = transformTerm(t, v) + if err != nil { + return nil, nil, err + } + return k, v, nil + }) + case Array: + for i := range y { + if y[i], err = transformTerm(t, y[i]); err != nil { + return nil, err + } + } + return y, nil + case Set: + y, err = y.Map(func(term *Term) (*Term, error) { + return transformTerm(t, term) + }) + if err != nil { + return nil, err + } + return y, nil + case *ArrayComprehension: + if y.Term, err = transformTerm(t, y.Term); err != nil { + return nil, err + } + if y.Body, err = transformBody(t, y.Body); err != nil { + return nil, err + } + return y, nil + case *ObjectComprehension: + if y.Key, err = transformTerm(t, y.Key); err != nil { + return nil, err + } + if y.Value, err = transformTerm(t, y.Value); err != nil { + return nil, err + } + if y.Body, err = transformBody(t, y.Body); err != nil { + return nil, err + } + return y, nil + case *SetComprehension: + if y.Term, err = transformTerm(t, y.Term); err != nil { + return nil, err + } + if y.Body, err = transformBody(t, y.Body); err != nil { + return nil, err + } + return y, nil + case Call: + for i := range y { + if y[i], err = transformTerm(t, y[i]); err != nil { + return nil, err + } + } + return y, nil + default: + return y, nil + } +} + +// TransformRefs calls the function f on all references under x. +func TransformRefs(x interface{}, f func(Ref) (Value, error)) (interface{}, error) { + t := &GenericTransformer{func(x interface{}) (interface{}, error) { + if r, ok := x.(Ref); ok { + return f(r) + } + return x, nil + }} + return Transform(t, x) +} + +// TransformVars calls the function f on all vars under x. +func TransformVars(x interface{}, f func(Var) (Value, error)) (interface{}, error) { + t := &GenericTransformer{func(x interface{}) (interface{}, error) { + if v, ok := x.(Var); ok { + return f(v) + } + return x, nil + }} + return Transform(t, x) +} + +// TransformComprehensions calls the functio nf on all comprehensions under x. +func TransformComprehensions(x interface{}, f func(interface{}) (Value, error)) (interface{}, error) { + t := &GenericTransformer{func(x interface{}) (interface{}, error) { + switch x := x.(type) { + case *ArrayComprehension: + return f(x) + case *SetComprehension: + return f(x) + case *ObjectComprehension: + return f(x) + } + return x, nil + }} + return Transform(t, x) +} + +// GenericTransformer implements the Transformer interface to provide a utility +// to transform AST nodes using a closure. +type GenericTransformer struct { + f func(x interface{}) (interface{}, error) +} + +// NewGenericTransformer returns a new GenericTransformer that will transform +// AST nodes using the function f. +func NewGenericTransformer(f func(x interface{}) (interface{}, error)) *GenericTransformer { + return &GenericTransformer{ + f: f, + } +} + +// Transform calls the function f on the GenericTransformer. +func (t *GenericTransformer) Transform(x interface{}) (interface{}, error) { + return t.f(x) +} + +func transformHead(t Transformer, head *Head) (*Head, error) { + y, err := Transform(t, head) + if err != nil { + return nil, err + } + h, ok := y.(*Head) + if !ok { + return nil, fmt.Errorf("illegal transform: %T != %T", head, y) + } + return h, nil +} +func transformArgs(t Transformer, args Args) (Args, error) { + y, err := Transform(t, args) + if err != nil { + return nil, err + } + a, ok := y.(Args) + if !ok { + return nil, fmt.Errorf("illegal transform: %T != %T", args, y) + } + return a, nil +} + +func transformBody(t Transformer, body Body) (Body, error) { + y, err := Transform(t, body) + if err != nil { + return nil, err + } + r, ok := y.(Body) + if !ok { + return nil, fmt.Errorf("illegal transform: %T != %T", body, y) + } + return r, nil +} + +func transformTerm(t Transformer, term *Term) (*Term, error) { + v, err := transformValue(t, term.Value) + if err != nil { + return nil, err + } + r := &Term{ + Value: v, + Location: term.Location, + } + return r, nil +} + +func transformValue(t Transformer, v Value) (Value, error) { + v1, err := Transform(t, v) + if err != nil { + return nil, err + } + r, ok := v1.(Value) + if !ok { + return nil, fmt.Errorf("illegal transform: %T != %T", v, v1) + } + return r, nil +} + +func transformVar(t Transformer, v Var) (Var, error) { + v1, err := Transform(t, v) + if err != nil { + return "", err + } + r, ok := v1.(Var) + if !ok { + return "", fmt.Errorf("illegal transform: %T != %T", v, v1) + } + return r, nil +} diff --git a/vendor/github.com/open-policy-agent/opa/ast/unify.go b/vendor/github.com/open-policy-agent/opa/ast/unify.go new file mode 100644 index 000000000..511dbe4c5 --- /dev/null +++ b/vendor/github.com/open-policy-agent/opa/ast/unify.go @@ -0,0 +1,184 @@ +// Copyright 2016 The OPA Authors. All rights reserved. +// Use of this source code is governed by an Apache2 +// license that can be found in the LICENSE file. + +package ast + +// Unify returns a set of variables that will be unified when the equality expression defined by +// terms a and b is evaluated. The unifier assumes that variables in the VarSet safe are already +// unified. +func Unify(safe VarSet, a *Term, b *Term) VarSet { + u := &unifier{ + safe: safe, + unified: VarSet{}, + unknown: map[Var]VarSet{}, + } + u.unify(a, b) + return u.unified +} + +type unifier struct { + safe VarSet + unified VarSet + unknown map[Var]VarSet +} + +func (u *unifier) isSafe(x Var) bool { + return u.safe.Contains(x) || u.unified.Contains(x) +} + +func (u *unifier) unify(a *Term, b *Term) { + + switch a := a.Value.(type) { + + case Var: + switch b := b.Value.(type) { + case Var: + if u.isSafe(b) { + u.markSafe(a) + } else if u.isSafe(a) { + u.markSafe(b) + } else { + u.markUnknown(a, b) + u.markUnknown(b, a) + } + case Array, Object: + u.unifyAll(a, b) + case Ref: + if u.isSafe(b[0].Value.(Var)) { + u.markSafe(a) + } + default: + u.markSafe(a) + } + + case Ref: + if u.isSafe(a[0].Value.(Var)) { + switch b := b.Value.(type) { + case Var: + u.markSafe(b) + case Array, Object: + u.markAllSafe(b) + } + } + + case *ArrayComprehension: + switch b := b.Value.(type) { + case Var: + u.markSafe(b) + case Array: + u.markAllSafe(b) + } + case *ObjectComprehension: + switch b := b.Value.(type) { + case Var: + u.markSafe(b) + case Object: + u.markAllSafe(b) + } + case *SetComprehension: + switch b := b.Value.(type) { + case Var: + u.markSafe(b) + } + + case Array: + switch b := b.Value.(type) { + case Var: + u.unifyAll(b, a) + case Ref, *ArrayComprehension, *ObjectComprehension, *SetComprehension: + u.markAllSafe(a) + case Array: + if len(a) == len(b) { + for i := range a { + u.unify(a[i], b[i]) + } + } + } + + case Object: + switch b := b.Value.(type) { + case Var: + u.unifyAll(b, a) + case Ref: + u.markAllSafe(a) + case Object: + if a.Len() == b.Len() { + a.Iter(func(k, v *Term) error { + if v2 := b.Get(k); v2 != nil { + u.unify(v, v2) + } + return nil + }) + } + } + + default: + switch b := b.Value.(type) { + case Var: + u.markSafe(b) + } + } +} + +func (u *unifier) markAllSafe(x Value) { + vis := u.varVisitor() + vis.Walk(x) + for v := range vis.Vars() { + u.markSafe(v) + } +} + +func (u *unifier) markSafe(x Var) { + u.unified.Add(x) + + // Add dependencies of 'x' to safe set + vs := u.unknown[x] + delete(u.unknown, x) + for v := range vs { + u.markSafe(v) + } + + // Add dependants of 'x' to safe set if they have no more + // dependencies. + for v, deps := range u.unknown { + if deps.Contains(x) { + delete(deps, x) + if len(deps) == 0 { + u.markSafe(v) + } + } + } +} + +func (u *unifier) markUnknown(a, b Var) { + if _, ok := u.unknown[a]; !ok { + u.unknown[a] = NewVarSet() + } + u.unknown[a].Add(b) +} + +func (u *unifier) unifyAll(a Var, b Value) { + if u.isSafe(a) { + u.markAllSafe(b) + } else { + vis := u.varVisitor() + vis.Walk(b) + unsafe := vis.Vars().Diff(u.safe).Diff(u.unified) + if len(unsafe) == 0 { + u.markSafe(a) + } else { + for v := range unsafe { + u.markUnknown(a, v) + } + } + } +} + +func (u *unifier) varVisitor() *VarVisitor { + return NewVarVisitor().WithParams(VarVisitorParams{ + SkipRefHead: true, + SkipObjectKeys: true, + SkipClosures: true, + }) +} diff --git a/vendor/github.com/open-policy-agent/opa/ast/varset.go b/vendor/github.com/open-policy-agent/opa/ast/varset.go new file mode 100644 index 000000000..16dc3f584 --- /dev/null +++ b/vendor/github.com/open-policy-agent/opa/ast/varset.go @@ -0,0 +1,100 @@ +// Copyright 2016 The OPA Authors. All rights reserved. +// Use of this source code is governed by an Apache2 +// license that can be found in the LICENSE file. + +package ast + +import ( + "fmt" + "sort" +) + +// VarSet represents a set of variables. +type VarSet map[Var]struct{} + +// NewVarSet returns a new VarSet containing the specified variables. +func NewVarSet(vs ...Var) VarSet { + s := VarSet{} + for _, v := range vs { + s.Add(v) + } + return s +} + +// Add updates the set to include the variable "v". +func (s VarSet) Add(v Var) { + s[v] = struct{}{} +} + +// Contains returns true if the set contains the variable "v". +func (s VarSet) Contains(v Var) bool { + _, ok := s[v] + return ok +} + +// Copy returns a shallow copy of the VarSet. +func (s VarSet) Copy() VarSet { + cpy := VarSet{} + for v := range s { + cpy.Add(v) + } + return cpy +} + +// Diff returns a VarSet containing variables in s that are not in vs. +func (s VarSet) Diff(vs VarSet) VarSet { + r := VarSet{} + for v := range s { + if !vs.Contains(v) { + r.Add(v) + } + } + return r +} + +// Equal returns true if s contains exactly the same elements as vs. +func (s VarSet) Equal(vs VarSet) bool { + if len(s.Diff(vs)) > 0 { + return false + } + return len(vs.Diff(s)) == 0 +} + +// Intersect returns a VarSet containing variables in s that are in vs. +func (s VarSet) Intersect(vs VarSet) VarSet { + r := VarSet{} + for v := range s { + if vs.Contains(v) { + r.Add(v) + } + } + return r +} + +// Sorted returns a sorted slice of vars from s. +func (s VarSet) Sorted() []Var { + sorted := make([]Var, 0, len(s)) + for v := range s { + sorted = append(sorted, v) + } + sort.Slice(sorted, func(i, j int) bool { + return sorted[i].Compare(sorted[j]) < 0 + }) + return sorted +} + +// Update merges the other VarSet into this VarSet. +func (s VarSet) Update(vs VarSet) { + for v := range vs { + s.Add(v) + } +} + +func (s VarSet) String() string { + tmp := []string{} + for v := range s { + tmp = append(tmp, string(v)) + } + sort.Strings(tmp) + return fmt.Sprintf("%v", tmp) +} diff --git a/vendor/github.com/open-policy-agent/opa/ast/visit.go b/vendor/github.com/open-policy-agent/opa/ast/visit.go new file mode 100644 index 000000000..19bd14f36 --- /dev/null +++ b/vendor/github.com/open-policy-agent/opa/ast/visit.go @@ -0,0 +1,686 @@ +// Copyright 2016 The OPA Authors. All rights reserved. +// Use of this source code is governed by an Apache2 +// license that can be found in the LICENSE file. + +package ast + +// Visitor defines the interface for iterating AST elements. The Visit function +// can return a Visitor w which will be used to visit the children of the AST +// element v. If the Visit function returns nil, the children will not be +// visited. This is deprecated. +type Visitor interface { + Visit(v interface{}) (w Visitor) +} + +// BeforeAndAfterVisitor wraps Visitor to provide hooks for being called before +// and after the AST has been visited. This is deprecated. +type BeforeAndAfterVisitor interface { + Visitor + Before(x interface{}) + After(x interface{}) +} + +// Walk iterates the AST by calling the Visit function on the Visitor +// v for x before recursing. This is deprecated. +func Walk(v Visitor, x interface{}) { + if bav, ok := v.(BeforeAndAfterVisitor); !ok { + walk(v, x) + } else { + bav.Before(x) + defer bav.After(x) + walk(bav, x) + } +} + +// WalkBeforeAndAfter iterates the AST by calling the Visit function on the +// Visitor v for x before recursing. This is deprecated. +func WalkBeforeAndAfter(v BeforeAndAfterVisitor, x interface{}) { + Walk(v, x) +} + +func walk(v Visitor, x interface{}) { + w := v.Visit(x) + if w == nil { + return + } + switch x := x.(type) { + case *Module: + Walk(w, x.Package) + for _, i := range x.Imports { + Walk(w, i) + } + for _, r := range x.Rules { + Walk(w, r) + } + for _, c := range x.Comments { + Walk(w, c) + } + case *Package: + Walk(w, x.Path) + case *Import: + Walk(w, x.Path) + Walk(w, x.Alias) + case *Rule: + Walk(w, x.Head) + Walk(w, x.Body) + if x.Else != nil { + Walk(w, x.Else) + } + case *Head: + Walk(w, x.Name) + Walk(w, x.Args) + if x.Key != nil { + Walk(w, x.Key) + } + if x.Value != nil { + Walk(w, x.Value) + } + case Body: + for _, e := range x { + Walk(w, e) + } + case Args: + for _, t := range x { + Walk(w, t) + } + case *Expr: + switch ts := x.Terms.(type) { + case *SomeDecl: + Walk(w, ts) + case []*Term: + for _, t := range ts { + Walk(w, t) + } + case *Term: + Walk(w, ts) + } + for i := range x.With { + Walk(w, x.With[i]) + } + case *With: + Walk(w, x.Target) + Walk(w, x.Value) + case *Term: + Walk(w, x.Value) + case Ref: + for _, t := range x { + Walk(w, t) + } + case Object: + x.Foreach(func(k, vv *Term) { + Walk(w, k) + Walk(w, vv) + }) + case Array: + for _, t := range x { + Walk(w, t) + } + case Set: + x.Foreach(func(t *Term) { + Walk(w, t) + }) + case *ArrayComprehension: + Walk(w, x.Term) + Walk(w, x.Body) + case *ObjectComprehension: + Walk(w, x.Key) + Walk(w, x.Value) + Walk(w, x.Body) + case *SetComprehension: + Walk(w, x.Term) + Walk(w, x.Body) + case Call: + for _, t := range x { + Walk(w, t) + } + } +} + +// WalkVars calls the function f on all vars under x. If the function f +// returns true, AST nodes under the last node will not be visited. +func WalkVars(x interface{}, f func(Var) bool) { + vis := &GenericVisitor{func(x interface{}) bool { + if v, ok := x.(Var); ok { + return f(v) + } + return false + }} + vis.Walk(x) +} + +// WalkClosures calls the function f on all closures under x. If the function f +// returns true, AST nodes under the last node will not be visited. +func WalkClosures(x interface{}, f func(interface{}) bool) { + vis := &GenericVisitor{func(x interface{}) bool { + switch x.(type) { + case *ArrayComprehension, *ObjectComprehension, *SetComprehension: + return f(x) + } + return false + }} + vis.Walk(x) +} + +// WalkRefs calls the function f on all references under x. If the function f +// returns true, AST nodes under the last node will not be visited. +func WalkRefs(x interface{}, f func(Ref) bool) { + vis := &GenericVisitor{func(x interface{}) bool { + if r, ok := x.(Ref); ok { + return f(r) + } + return false + }} + vis.Walk(x) +} + +// WalkTerms calls the function f on all terms under x. If the function f +// returns true, AST nodes under the last node will not be visited. +func WalkTerms(x interface{}, f func(*Term) bool) { + vis := &GenericVisitor{func(x interface{}) bool { + if term, ok := x.(*Term); ok { + return f(term) + } + return false + }} + vis.Walk(x) +} + +// WalkWiths calls the function f on all with modifiers under x. If the function f +// returns true, AST nodes under the last node will not be visited. +func WalkWiths(x interface{}, f func(*With) bool) { + vis := &GenericVisitor{func(x interface{}) bool { + if w, ok := x.(*With); ok { + return f(w) + } + return false + }} + vis.Walk(x) +} + +// WalkExprs calls the function f on all expressions under x. If the function f +// returns true, AST nodes under the last node will not be visited. +func WalkExprs(x interface{}, f func(*Expr) bool) { + vis := &GenericVisitor{func(x interface{}) bool { + if r, ok := x.(*Expr); ok { + return f(r) + } + return false + }} + vis.Walk(x) +} + +// WalkBodies calls the function f on all bodies under x. If the function f +// returns true, AST nodes under the last node will not be visited. +func WalkBodies(x interface{}, f func(Body) bool) { + vis := &GenericVisitor{func(x interface{}) bool { + if b, ok := x.(Body); ok { + return f(b) + } + return false + }} + vis.Walk(x) +} + +// WalkRules calls the function f on all rules under x. If the function f +// returns true, AST nodes under the last node will not be visited. +func WalkRules(x interface{}, f func(*Rule) bool) { + vis := &GenericVisitor{func(x interface{}) bool { + if r, ok := x.(*Rule); ok { + stop := f(r) + // NOTE(tsandall): since rules cannot be embedded inside of queries + // we can stop early if there is no else block. + if stop || r.Else == nil { + return true + } + } + return false + }} + vis.Walk(x) +} + +// WalkNodes calls the function f on all nodes under x. If the function f +// returns true, AST nodes under the last node will not be visited. +func WalkNodes(x interface{}, f func(Node) bool) { + vis := &GenericVisitor{func(x interface{}) bool { + if n, ok := x.(Node); ok { + return f(n) + } + return false + }} + vis.Walk(x) +} + +// GenericVisitor provides a utility to walk over AST nodes using a +// closure. If the closure returns true, the visitor will not walk +// over AST nodes under x. +type GenericVisitor struct { + f func(x interface{}) bool +} + +// NewGenericVisitor returns a new GenericVisitor that will invoke the function +// f on AST nodes. +func NewGenericVisitor(f func(x interface{}) bool) *GenericVisitor { + return &GenericVisitor{f} +} + +// Walk iterates the AST by calling the function f on the +// GenericVisitor before recursing. Contrary to the generic Walk, this +// does not require allocating the visitor from heap. +func (vis *GenericVisitor) Walk(x interface{}) { + if vis.f(x) { + return + } + + switch x := x.(type) { + case *Module: + vis.Walk(x.Package) + for _, i := range x.Imports { + vis.Walk(i) + } + for _, r := range x.Rules { + vis.Walk(r) + } + for _, c := range x.Comments { + vis.Walk(c) + } + case *Package: + vis.Walk(x.Path) + case *Import: + vis.Walk(x.Path) + vis.Walk(x.Alias) + case *Rule: + vis.Walk(x.Head) + vis.Walk(x.Body) + if x.Else != nil { + vis.Walk(x.Else) + } + case *Head: + vis.Walk(x.Name) + vis.Walk(x.Args) + if x.Key != nil { + vis.Walk(x.Key) + } + if x.Value != nil { + vis.Walk(x.Value) + } + case Body: + for _, e := range x { + vis.Walk(e) + } + case Args: + for _, t := range x { + vis.Walk(t) + } + case *Expr: + switch ts := x.Terms.(type) { + case *SomeDecl: + vis.Walk(ts) + case []*Term: + for _, t := range ts { + vis.Walk(t) + } + case *Term: + vis.Walk(ts) + } + for i := range x.With { + vis.Walk(x.With[i]) + } + case *With: + vis.Walk(x.Target) + vis.Walk(x.Value) + case *Term: + vis.Walk(x.Value) + case Ref: + for _, t := range x { + vis.Walk(t) + } + case Object: + for _, k := range x.Keys() { + vis.Walk(k) + vis.Walk(x.Get(k)) + } + case Array: + for _, t := range x { + vis.Walk(t) + } + case Set: + for _, t := range x.Slice() { + vis.Walk(t) + } + case *ArrayComprehension: + vis.Walk(x.Term) + vis.Walk(x.Body) + case *ObjectComprehension: + vis.Walk(x.Key) + vis.Walk(x.Value) + vis.Walk(x.Body) + case *SetComprehension: + vis.Walk(x.Term) + vis.Walk(x.Body) + case Call: + for _, t := range x { + vis.Walk(t) + } + } +} + +// BeforeAfterVisitor provides a utility to walk over AST nodes using +// closures. If the before closure returns true, the visitor will not +// walk over AST nodes under x. The after closure is invoked always +// after visiting a node. +type BeforeAfterVisitor struct { + before func(x interface{}) bool + after func(x interface{}) +} + +// NewBeforeAfterVisitor returns a new BeforeAndAfterVisitor that +// will invoke the functions before and after AST nodes. +func NewBeforeAfterVisitor(before func(x interface{}) bool, after func(x interface{})) *BeforeAfterVisitor { + return &BeforeAfterVisitor{before, after} +} + +// Walk iterates the AST by calling the functions on the +// BeforeAndAfterVisitor before and after recursing. Contrary to the +// generic Walk, this does not require allocating the visitor from +// heap. +func (vis *BeforeAfterVisitor) Walk(x interface{}) { + defer vis.after(x) + if vis.before(x) { + return + } + + switch x := x.(type) { + case *Module: + vis.Walk(x.Package) + for _, i := range x.Imports { + vis.Walk(i) + } + for _, r := range x.Rules { + vis.Walk(r) + } + for _, c := range x.Comments { + vis.Walk(c) + } + case *Package: + vis.Walk(x.Path) + case *Import: + vis.Walk(x.Path) + vis.Walk(x.Alias) + case *Rule: + vis.Walk(x.Head) + vis.Walk(x.Body) + if x.Else != nil { + vis.Walk(x.Else) + } + case *Head: + vis.Walk(x.Name) + vis.Walk(x.Args) + if x.Key != nil { + vis.Walk(x.Key) + } + if x.Value != nil { + vis.Walk(x.Value) + } + case Body: + for _, e := range x { + vis.Walk(e) + } + case Args: + for _, t := range x { + vis.Walk(t) + } + case *Expr: + switch ts := x.Terms.(type) { + case *SomeDecl: + vis.Walk(ts) + case []*Term: + for _, t := range ts { + vis.Walk(t) + } + case *Term: + vis.Walk(ts) + } + for i := range x.With { + vis.Walk(x.With[i]) + } + case *With: + vis.Walk(x.Target) + vis.Walk(x.Value) + case *Term: + vis.Walk(x.Value) + case Ref: + for _, t := range x { + vis.Walk(t) + } + case Object: + for _, k := range x.Keys() { + vis.Walk(k) + vis.Walk(x.Get(k)) + } + case Array: + for _, t := range x { + vis.Walk(t) + } + case Set: + for _, t := range x.Slice() { + vis.Walk(t) + } + case *ArrayComprehension: + vis.Walk(x.Term) + vis.Walk(x.Body) + case *ObjectComprehension: + vis.Walk(x.Key) + vis.Walk(x.Value) + vis.Walk(x.Body) + case *SetComprehension: + vis.Walk(x.Term) + vis.Walk(x.Body) + case Call: + for _, t := range x { + vis.Walk(t) + } + } +} + +// VarVisitor walks AST nodes under a given node and collects all encountered +// variables. The collected variables can be controlled by specifying +// VarVisitorParams when creating the visitor. +type VarVisitor struct { + params VarVisitorParams + vars VarSet +} + +// VarVisitorParams contains settings for a VarVisitor. +type VarVisitorParams struct { + SkipRefHead bool + SkipRefCallHead bool + SkipObjectKeys bool + SkipClosures bool + SkipWithTarget bool + SkipSets bool +} + +// NewVarVisitor returns a new VarVisitor object. +func NewVarVisitor() *VarVisitor { + return &VarVisitor{ + vars: NewVarSet(), + } +} + +// WithParams sets the parameters in params on vis. +func (vis *VarVisitor) WithParams(params VarVisitorParams) *VarVisitor { + vis.params = params + return vis +} + +// Vars returns a VarSet that contains collected vars. +func (vis *VarVisitor) Vars() VarSet { + return vis.vars +} + +func (vis *VarVisitor) visit(v interface{}) bool { + if vis.params.SkipObjectKeys { + if o, ok := v.(Object); ok { + for _, k := range o.Keys() { + vis.Walk(o.Get(k)) + } + return true + } + } + if vis.params.SkipRefHead { + if r, ok := v.(Ref); ok { + for _, t := range r[1:] { + vis.Walk(t) + } + return true + } + } + if vis.params.SkipClosures { + switch v.(type) { + case *ArrayComprehension, *ObjectComprehension, *SetComprehension: + return true + } + } + if vis.params.SkipWithTarget { + if v, ok := v.(*With); ok { + vis.Walk(v.Value) + return true + } + } + if vis.params.SkipSets { + if _, ok := v.(Set); ok { + return true + } + } + if vis.params.SkipRefCallHead { + switch v := v.(type) { + case *Expr: + if terms, ok := v.Terms.([]*Term); ok { + for _, t := range terms[0].Value.(Ref)[1:] { + vis.Walk(t) + } + for i := 1; i < len(terms); i++ { + vis.Walk(terms[i]) + } + for _, w := range v.With { + vis.Walk(w) + } + return true + } + case Call: + operator := v[0].Value.(Ref) + for i := 1; i < len(operator); i++ { + vis.Walk(operator[i]) + } + for i := 1; i < len(v); i++ { + vis.Walk(v[i]) + } + return true + } + } + if v, ok := v.(Var); ok { + vis.vars.Add(v) + } + return false +} + +// Walk iterates the AST by calling the function f on the +// GenericVisitor before recursing. Contrary to the generic Walk, this +// does not require allocating the visitor from heap. +func (vis *VarVisitor) Walk(x interface{}) { + if vis.visit(x) { + return + } + + switch x := x.(type) { + case *Module: + vis.Walk(x.Package) + for _, i := range x.Imports { + vis.Walk(i) + } + for _, r := range x.Rules { + vis.Walk(r) + } + for _, c := range x.Comments { + vis.Walk(c) + } + case *Package: + vis.Walk(x.Path) + case *Import: + vis.Walk(x.Path) + vis.Walk(x.Alias) + case *Rule: + vis.Walk(x.Head) + vis.Walk(x.Body) + if x.Else != nil { + vis.Walk(x.Else) + } + case *Head: + vis.Walk(x.Name) + vis.Walk(x.Args) + if x.Key != nil { + vis.Walk(x.Key) + } + if x.Value != nil { + vis.Walk(x.Value) + } + case Body: + for _, e := range x { + vis.Walk(e) + } + case Args: + for _, t := range x { + vis.Walk(t) + } + case *Expr: + switch ts := x.Terms.(type) { + case *SomeDecl: + vis.Walk(ts) + case []*Term: + for _, t := range ts { + vis.Walk(t) + } + case *Term: + vis.Walk(ts) + } + for i := range x.With { + vis.Walk(x.With[i]) + } + case *With: + vis.Walk(x.Target) + vis.Walk(x.Value) + case *Term: + vis.Walk(x.Value) + case Ref: + for _, t := range x { + vis.Walk(t) + } + case Object: + for _, k := range x.Keys() { + vis.Walk(k) + vis.Walk(x.Get(k)) + } + case Array: + for _, t := range x { + vis.Walk(t) + } + case Set: + for _, t := range x.Slice() { + vis.Walk(t) + } + case *ArrayComprehension: + vis.Walk(x.Term) + vis.Walk(x.Body) + case *ObjectComprehension: + vis.Walk(x.Key) + vis.Walk(x.Value) + vis.Walk(x.Body) + case *SetComprehension: + vis.Walk(x.Term) + vis.Walk(x.Body) + case Call: + for _, t := range x { + vis.Walk(t) + } + } +} diff --git a/vendor/github.com/open-policy-agent/opa/bundle/bundle.go b/vendor/github.com/open-policy-agent/opa/bundle/bundle.go new file mode 100644 index 000000000..7e82a8127 --- /dev/null +++ b/vendor/github.com/open-policy-agent/opa/bundle/bundle.go @@ -0,0 +1,493 @@ +// Copyright 2018 The OPA Authors. All rights reserved. +// Use of this source code is governed by an Apache2 +// license that can be found in the LICENSE file. + +// Package bundle implements bundle loading. +package bundle + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "encoding/json" + "fmt" + "io" + "net/url" + "path/filepath" + "reflect" + "strings" + + "github.com/open-policy-agent/opa/internal/file/archive" + "github.com/open-policy-agent/opa/internal/merge" + "github.com/open-policy-agent/opa/metrics" + + "github.com/pkg/errors" + + "github.com/open-policy-agent/opa/ast" + "github.com/open-policy-agent/opa/util" +) + +// Common file extensions and file names. +const ( + RegoExt = ".rego" + WasmFile = "/policy.wasm" + manifestExt = ".manifest" + dataFile = "data.json" + yamlDataFile = "data.yaml" +) + +const bundleLimitBytes = (1024 * 1024 * 1024) + 1 // limit bundle reads to 1GB to protect against gzip bombs + +// Bundle represents a loaded bundle. The bundle can contain data and policies. +type Bundle struct { + Manifest Manifest + Data map[string]interface{} + Modules []ModuleFile + Wasm []byte +} + +// Manifest represents the manifest from a bundle. The manifest may contain +// metadata such as the bundle revision. +type Manifest struct { + Revision string `json:"revision"` + Roots *[]string `json:"roots,omitempty"` +} + +// Init initializes the manifest. If you instantiate a manifest +// manually, call Init to ensure that the roots are set properly. +func (m *Manifest) Init() { + if m.Roots == nil { + defaultRoots := []string{""} + m.Roots = &defaultRoots + } +} + +func (m *Manifest) validateAndInjectDefaults(b Bundle) error { + + m.Init() + + // Validate roots in bundle. + roots := *m.Roots + + // Standardize the roots (no starting or trailing slash) + for i := range roots { + roots[i] = strings.Trim(roots[i], "/") + } + + for i := 0; i < len(roots)-1; i++ { + for j := i + 1; j < len(roots); j++ { + if RootPathsOverlap(roots[i], roots[j]) { + return fmt.Errorf("manifest has overlapped roots: %v and %v", roots[i], roots[j]) + } + } + } + + // Validate modules in bundle. + for _, module := range b.Modules { + found := false + if path, err := module.Parsed.Package.Path.Ptr(); err == nil { + for i := range roots { + if strings.HasPrefix(path, roots[i]) { + found = true + break + } + } + } + if !found { + return fmt.Errorf("manifest roots %v do not permit '%v' in module '%v'", roots, module.Parsed.Package, module.Path) + } + } + + // Validate data in bundle. + return dfs(b.Data, "", func(path string, node interface{}) (bool, error) { + path = strings.Trim(path, "/") + for i := range roots { + if strings.HasPrefix(path, roots[i]) { + return true, nil + } + } + if _, ok := node.(map[string]interface{}); ok { + for i := range roots { + if strings.HasPrefix(roots[i], path) { + return false, nil + } + } + } + return false, fmt.Errorf("manifest roots %v do not permit data at path '/%s' (hint: check bundle directory structure)", roots, path) + }) +} + +// ModuleFile represents a single module contained a bundle. +type ModuleFile struct { + Path string + Raw []byte + Parsed *ast.Module +} + +// Reader contains the reader to load the bundle from. +type Reader struct { + loader DirectoryLoader + includeManifestInData bool + metrics metrics.Metrics + baseDir string +} + +// NewReader returns a new Reader which is configured for reading tarballs. +func NewReader(r io.Reader) *Reader { + return NewCustomReader(NewTarballLoader(r)) +} + +// NewCustomReader returns a new Reader configured to use the +// specified DirectoryLoader. +func NewCustomReader(loader DirectoryLoader) *Reader { + nr := Reader{ + loader: loader, + metrics: metrics.New(), + } + return &nr +} + +// IncludeManifestInData sets whether the manifest metadata should be +// included in the bundle's data. +func (r *Reader) IncludeManifestInData(includeManifestInData bool) *Reader { + r.includeManifestInData = includeManifestInData + return r +} + +// WithMetrics sets the metrics object to be used while loading bundles +func (r *Reader) WithMetrics(m metrics.Metrics) *Reader { + r.metrics = m + return r +} + +// WithBaseDir sets a base directory for file paths of loaded Rego +// modules. This will *NOT* affect the loaded path of data files. +func (r *Reader) WithBaseDir(dir string) *Reader { + r.baseDir = dir + return r +} + +// Read returns a new Bundle loaded from the reader. +func (r *Reader) Read() (Bundle, error) { + + var bundle Bundle + + bundle.Data = map[string]interface{}{} + + for { + f, err := r.loader.NextFile() + if err == io.EOF { + break + } + if err != nil { + return bundle, errors.Wrap(err, "bundle read failed") + } + + var buf bytes.Buffer + n, err := f.Read(&buf, bundleLimitBytes) + f.Close() // always close, even on error + if err != nil && err != io.EOF { + return bundle, err + } else if err == nil && n >= bundleLimitBytes { + return bundle, fmt.Errorf("bundle exceeded max size (%v bytes)", bundleLimitBytes-1) + } + + // Normalize the paths to use `/` separators + path := filepath.ToSlash(f.Path()) + + if strings.HasSuffix(path, RegoExt) { + fullPath := r.fullPath(path) + r.metrics.Timer(metrics.RegoModuleParse).Start() + module, err := ast.ParseModule(fullPath, buf.String()) + r.metrics.Timer(metrics.RegoModuleParse).Stop() + if err != nil { + return bundle, err + } + + mf := ModuleFile{ + Path: fullPath, + Raw: buf.Bytes(), + Parsed: module, + } + bundle.Modules = append(bundle.Modules, mf) + + } else if path == WasmFile { + bundle.Wasm = buf.Bytes() + + } else if filepath.Base(path) == dataFile { + var value interface{} + + r.metrics.Timer(metrics.RegoDataParse).Start() + err := util.NewJSONDecoder(&buf).Decode(&value) + r.metrics.Timer(metrics.RegoDataParse).Stop() + + if err != nil { + return bundle, errors.Wrapf(err, "bundle load failed on %v", r.fullPath(path)) + } + + if err := insertValue(&bundle, path, value); err != nil { + return bundle, err + } + + } else if filepath.Base(path) == yamlDataFile { + + var value interface{} + + r.metrics.Timer(metrics.RegoDataParse).Start() + err := util.Unmarshal(buf.Bytes(), &value) + r.metrics.Timer(metrics.RegoDataParse).Stop() + + if err != nil { + return bundle, errors.Wrapf(err, "bundle load failed on %v", r.fullPath(path)) + } + + if err := insertValue(&bundle, path, value); err != nil { + return bundle, err + } + + } else if strings.HasSuffix(path, manifestExt) { + if err := util.NewJSONDecoder(&buf).Decode(&bundle.Manifest); err != nil { + return bundle, errors.Wrap(err, "bundle load failed on manifest decode") + } + } + } + + if err := bundle.Manifest.validateAndInjectDefaults(bundle); err != nil { + return bundle, err + } + + if r.includeManifestInData { + var metadata map[string]interface{} + + b, err := json.Marshal(&bundle.Manifest) + if err != nil { + return bundle, errors.Wrap(err, "bundle load failed on manifest marshal") + } + + err = util.UnmarshalJSON(b, &metadata) + if err != nil { + return bundle, errors.Wrap(err, "bundle load failed on manifest unmarshal") + } + + // For backwards compatibility always write to the old unnamed manifest path + // This will *not* be correct if >1 bundle is in use... + if err := bundle.insert(legacyManifestStoragePath, metadata); err != nil { + return bundle, errors.Wrapf(err, "bundle load failed on %v", legacyRevisionStoragePath) + } + } + + return bundle, nil +} + +func (r *Reader) fullPath(path string) string { + if r.baseDir != "" { + path = filepath.Join(r.baseDir, path) + } + return path +} + +// Write serializes the Bundle and writes it to w. +func Write(w io.Writer, bundle Bundle) error { + gw := gzip.NewWriter(w) + tw := tar.NewWriter(gw) + + var buf bytes.Buffer + + if err := json.NewEncoder(&buf).Encode(bundle.Data); err != nil { + return err + } + + if err := archive.WriteFile(tw, "data.json", buf.Bytes()); err != nil { + return err + } + + for _, module := range bundle.Modules { + if err := archive.WriteFile(tw, module.Path, module.Raw); err != nil { + return err + } + } + + if err := writeWasm(tw, bundle); err != nil { + return err + } + + if err := writeManifest(tw, bundle); err != nil { + return err + } + + if err := tw.Close(); err != nil { + return err + } + + return gw.Close() +} + +func writeWasm(tw *tar.Writer, bundle Bundle) error { + if len(bundle.Wasm) == 0 { + return nil + } + + return archive.WriteFile(tw, WasmFile, bundle.Wasm) +} + +func writeManifest(tw *tar.Writer, bundle Bundle) error { + + var buf bytes.Buffer + + if err := json.NewEncoder(&buf).Encode(bundle.Manifest); err != nil { + return err + } + + return archive.WriteFile(tw, manifestExt, buf.Bytes()) +} + +// ParsedModules returns a map of parsed modules with names that are +// unique and human readable for the given a bundle name. +func (b *Bundle) ParsedModules(bundleName string) map[string]*ast.Module { + + mods := make(map[string]*ast.Module, len(b.Modules)) + + for _, mf := range b.Modules { + mods[modulePathWithPrefix(bundleName, mf.Path)] = mf.Parsed + } + + return mods +} + +// Equal returns true if this bundle's contents equal the other bundle's +// contents. +func (b Bundle) Equal(other Bundle) bool { + if !reflect.DeepEqual(b.Data, other.Data) { + return false + } + if len(b.Modules) != len(other.Modules) { + return false + } + for i := range b.Modules { + if b.Modules[i].Path != other.Modules[i].Path { + return false + } + if !b.Modules[i].Parsed.Equal(other.Modules[i].Parsed) { + return false + } + if !bytes.Equal(b.Modules[i].Raw, other.Modules[i].Raw) { + return false + } + } + if (b.Wasm == nil && other.Wasm != nil) || (b.Wasm != nil && other.Wasm == nil) { + return false + } + + return bytes.Equal(b.Wasm, other.Wasm) +} + +func (b *Bundle) insert(key []string, value interface{}) error { + // Build an object with the full structure for the value + obj, err := mktree(key, value) + if err != nil { + return err + } + + // Merge the new data in with the current bundle data object + merged, ok := merge.InterfaceMaps(b.Data, obj) + if !ok { + return fmt.Errorf("failed to insert data file from path %s", filepath.Join(key...)) + } + + b.Data = merged + + return nil +} + +func mktree(path []string, value interface{}) (map[string]interface{}, error) { + if len(path) == 0 { + // For 0 length path the value is the full tree. + obj, ok := value.(map[string]interface{}) + if !ok { + return nil, fmt.Errorf("root value must be object") + } + return obj, nil + } + + dir := map[string]interface{}{} + for i := len(path) - 1; i > 0; i-- { + dir[path[i]] = value + value = dir + dir = map[string]interface{}{} + } + dir[path[0]] = value + + return dir, nil +} + +// RootPathsOverlap takes in two bundle root paths and returns +// true if they overlap. +func RootPathsOverlap(pathA string, pathB string) bool { + + // Special case for empty prefixes, they always overlap + if pathA == "" || pathB == "" { + return true + } + + aParts := strings.Split(pathA, "/") + bParts := strings.Split(pathB, "/") + + for i := 0; i < len(aParts) && i < len(bParts); i++ { + if aParts[i] != bParts[i] { + // Found diverging path segments, no overlap + return false + } + } + return true +} + +func insertValue(b *Bundle, path string, value interface{}) error { + + // Remove leading / and . characters from the directory path. If the bundle + // was written with OPA then the paths will contain a leading slash. On the + // other hand, if the path is empty, filepath.Dir will return '.'. + // Note: filepath.Dir can return paths with '\' separators, always use + // filepath.ToSlash to keep them normalized. + dirpath := strings.TrimLeft(filepath.ToSlash(filepath.Dir(path)), "/.") + var key []string + if dirpath != "" { + key = strings.Split(dirpath, "/") + } + if err := b.insert(key, value); err != nil { + return errors.Wrapf(err, "bundle load failed on %v", path) + } + + return nil +} + +func dfs(value interface{}, path string, fn func(string, interface{}) (bool, error)) error { + if stop, err := fn(path, value); err != nil { + return err + } else if stop { + return nil + } + obj, ok := value.(map[string]interface{}) + if !ok { + return nil + } + for key := range obj { + if err := dfs(obj[key], path+"/"+key, fn); err != nil { + return err + } + } + return nil +} + +func modulePathWithPrefix(bundleName string, modulePath string) string { + // Default prefix is just the bundle name + prefix := bundleName + + // Bundle names are sometimes just file paths, some of which + // are full urls (file:///foo/). Parse these and only use the path. + parsed, err := url.Parse(bundleName) + if err == nil { + prefix = filepath.Join(parsed.Host, parsed.Path) + } + + return filepath.Join(prefix, modulePath) +} diff --git a/vendor/github.com/open-policy-agent/opa/bundle/file.go b/vendor/github.com/open-policy-agent/opa/bundle/file.go new file mode 100644 index 000000000..ab1d1c72a --- /dev/null +++ b/vendor/github.com/open-policy-agent/opa/bundle/file.go @@ -0,0 +1,166 @@ +package bundle + +import ( + "archive/tar" + "compress/gzip" + "io" + "os" + "path/filepath" + "strings" + "sync" + + "github.com/pkg/errors" +) + +// Descriptor contains information about a file and +// can be used to read the file contents. +type Descriptor struct { + path string + reader io.Reader + closer io.Closer + closeOnce *sync.Once +} + +func newDescriptor(path string, reader io.Reader) *Descriptor { + return &Descriptor{ + path: path, + reader: reader, + } +} + +func (d *Descriptor) withCloser(closer io.Closer) *Descriptor { + d.closer = closer + d.closeOnce = new(sync.Once) + return d +} + +// Path returns the path of the file. +func (d *Descriptor) Path() string { + return d.path +} + +// Read will read all the contents from the file the Descriptor refers to +// into the dest writer up n bytes. Will return an io.EOF error +// if EOF is encountered before n bytes are read. +func (d *Descriptor) Read(dest io.Writer, n int64) (int64, error) { + n, err := io.CopyN(dest, d.reader, n) + return n, err +} + +// Close the file, on some Loader implementations this might be a no-op. +// It should *always* be called regardless of file. +func (d *Descriptor) Close() error { + var err error + if d.closer != nil { + d.closeOnce.Do(func() { + err = d.closer.Close() + }) + } + return err +} + +// DirectoryLoader defines an interface which can be used to load +// files from a directory by iterating over each one in the tree. +type DirectoryLoader interface { + // NextFile must return io.EOF if there is no next value. The returned + // descriptor should *always* be closed when no longer needed. + NextFile() (*Descriptor, error) +} + +type dirLoader struct { + root string + files []string + idx int +} + +// NewDirectoryLoader returns a basic DirectoryLoader implementation +// that will load files from a given root directory path. +func NewDirectoryLoader(root string) DirectoryLoader { + d := dirLoader{ + root: root, + } + return &d +} + +// NextFile iterates to the next file in the directory tree +// and returns a file Descriptor for the file. +func (d *dirLoader) NextFile() (*Descriptor, error) { + // build a list of all files we will iterate over and read, but only one time + if d.files == nil { + d.files = []string{} + err := filepath.Walk(d.root, func(path string, info os.FileInfo, err error) error { + if info != nil && info.Mode().IsRegular() { + d.files = append(d.files, filepath.ToSlash(path)) + } + return nil + }) + if err != nil { + return nil, errors.Wrap(err, "failed to list files") + } + } + + // If done reading files then just return io.EOF + // errors for each NextFile() call + if d.idx >= len(d.files) { + return nil, io.EOF + } + + fileName := d.files[d.idx] + d.idx++ + fh, err := os.Open(fileName) + if err != nil { + return nil, errors.Wrapf(err, "failed to open file %s", fileName) + } + + // Trim off the root directory and return path as if chrooted + cleanedPath := strings.TrimPrefix(fileName, d.root) + if !strings.HasPrefix(cleanedPath, "/") { + cleanedPath = "/" + cleanedPath + } + + f := newDescriptor(cleanedPath, fh).withCloser(fh) + return f, nil +} + +type tarballLoader struct { + r io.Reader + tr *tar.Reader +} + +// NewTarballLoader returns a new DirectoryLoader that reads +// files out of a gzipped tar archive. +func NewTarballLoader(r io.Reader) DirectoryLoader { + l := tarballLoader{ + r: r, + } + return &l +} + +// NextFile iterates to the next file in the directory tree +// and returns a file Descriptor for the file. +func (t *tarballLoader) NextFile() (*Descriptor, error) { + if t.tr == nil { + gr, err := gzip.NewReader(t.r) + if err != nil { + return nil, errors.Wrap(err, "archive read failed") + } + + t.tr = tar.NewReader(gr) + } + + for { + header, err := t.tr.Next() + // Eventually we will get an io.EOF error when finished + // iterating through the archive + if err != nil { + return nil, err + } + + // Keep iterating on the archive until we find a normal file + if header.Typeflag == tar.TypeReg { + // no need to close this descriptor after reading + f := newDescriptor(header.Name, t.tr) + return f, nil + } + } +} diff --git a/vendor/github.com/open-policy-agent/opa/bundle/store.go b/vendor/github.com/open-policy-agent/opa/bundle/store.go new file mode 100644 index 000000000..481ff6ef3 --- /dev/null +++ b/vendor/github.com/open-policy-agent/opa/bundle/store.go @@ -0,0 +1,526 @@ +// Copyright 2019 The OPA Authors. All rights reserved. +// Use of this source code is governed by an Apache2 +// license that can be found in the LICENSE file. + +package bundle + +import ( + "context" + "fmt" + "strings" + + "github.com/open-policy-agent/opa/metrics" + + "github.com/open-policy-agent/opa/ast" + + "github.com/open-policy-agent/opa/storage" + "github.com/open-policy-agent/opa/util" +) + +var bundlesBasePath = storage.MustParsePath("/system/bundles") + +// Note: As needed these helpers could be memoized. + +// ManifestStoragePath is the storage path used for the given named bundle manifest. +func ManifestStoragePath(name string) storage.Path { + return append(bundlesBasePath, name, "manifest") +} + +func namedBundlePath(name string) storage.Path { + return append(bundlesBasePath, name) +} + +func rootsPath(name string) storage.Path { + return append(bundlesBasePath, name, "manifest", "roots") +} + +func revisionPath(name string) storage.Path { + return append(bundlesBasePath, name, "manifest", "revision") +} + +// ReadBundleNamesFromStore will return a list of bundle names which have had their metadata stored. +func ReadBundleNamesFromStore(ctx context.Context, store storage.Store, txn storage.Transaction) ([]string, error) { + value, err := store.Read(ctx, txn, bundlesBasePath) + if err != nil { + return nil, err + } + + bundleMap, ok := value.(map[string]interface{}) + if !ok { + return nil, fmt.Errorf("corrupt manifest roots") + } + + bundles := make([]string, len(bundleMap)) + idx := 0 + for name := range bundleMap { + bundles[idx] = name + idx++ + } + return bundles, nil +} + +// WriteManifestToStore will write the manifest into the storage. This function is called when +// the bundle is activated. +func WriteManifestToStore(ctx context.Context, store storage.Store, txn storage.Transaction, name string, manifest Manifest) error { + return write(ctx, store, txn, ManifestStoragePath(name), manifest) +} + +func write(ctx context.Context, store storage.Store, txn storage.Transaction, path storage.Path, manifest Manifest) error { + var value interface{} = manifest + if err := util.RoundTrip(&value); err != nil { + return err + } + + var dir []string + if len(path) > 1 { + dir = path[:len(path)-1] + } + + if err := storage.MakeDir(ctx, store, txn, dir); err != nil { + return err + } + + return store.Write(ctx, txn, storage.AddOp, path, value) +} + +// EraseManifestFromStore will remove the manifest from storage. This function is called +// when the bundle is deactivated. +func EraseManifestFromStore(ctx context.Context, store storage.Store, txn storage.Transaction, name string) error { + path := namedBundlePath(name) + err := store.Write(ctx, txn, storage.RemoveOp, path, nil) + if err != nil && !storage.IsNotFound(err) { + return err + } + return nil +} + +// ReadBundleRootsFromStore returns the roots in the specified bundle. +// If the bundle is not activated, this function will return +// storage NotFound error. +func ReadBundleRootsFromStore(ctx context.Context, store storage.Store, txn storage.Transaction, name string) ([]string, error) { + value, err := store.Read(ctx, txn, rootsPath(name)) + if err != nil { + return nil, err + } + + sl, ok := value.([]interface{}) + if !ok { + return nil, fmt.Errorf("corrupt manifest roots") + } + + roots := make([]string, len(sl)) + + for i := range sl { + roots[i], ok = sl[i].(string) + if !ok { + return nil, fmt.Errorf("corrupt manifest root") + } + } + + return roots, nil +} + +// ReadBundleRevisionFromStore returns the revision in the specified bundle. +// If the bundle is not activated, this function will return +// storage NotFound error. +func ReadBundleRevisionFromStore(ctx context.Context, store storage.Store, txn storage.Transaction, name string) (string, error) { + return readRevisionFromStore(ctx, store, txn, revisionPath(name)) +} + +func readRevisionFromStore(ctx context.Context, store storage.Store, txn storage.Transaction, path storage.Path) (string, error) { + value, err := store.Read(ctx, txn, path) + if err != nil { + return "", err + } + + str, ok := value.(string) + if !ok { + return "", fmt.Errorf("corrupt manifest revision") + } + + return str, nil +} + +// ActivateOpts defines options for the Activate API call. +type ActivateOpts struct { + Ctx context.Context + Store storage.Store + Txn storage.Transaction + Compiler *ast.Compiler + Metrics metrics.Metrics + Bundles map[string]*Bundle // Optional + ExtraModules map[string]*ast.Module // Optional + + legacy bool +} + +// Activate the bundle(s) by loading into the given Store. This will load policies, data, and record +// the manifest in storage. The compiler provided will have had the polices compiled on it. +func Activate(opts *ActivateOpts) error { + opts.legacy = false + return activateBundles(opts) +} + +// DeactivateOpts defines options for the Deactivate API call +type DeactivateOpts struct { + Ctx context.Context + Store storage.Store + Txn storage.Transaction + BundleNames map[string]struct{} +} + +// Deactivate the bundle(s). This will erase associated data, policies, and the manifest entry from the store. +func Deactivate(opts *DeactivateOpts) error { + erase := map[string]struct{}{} + for name := range opts.BundleNames { + if roots, err := ReadBundleRootsFromStore(opts.Ctx, opts.Store, opts.Txn, name); err == nil { + for _, root := range roots { + erase[root] = struct{}{} + } + } else if !storage.IsNotFound(err) { + return err + } + } + _, err := eraseBundles(opts.Ctx, opts.Store, opts.Txn, opts.BundleNames, erase) + return err +} + +func activateBundles(opts *ActivateOpts) error { + + // Build collections of bundle names, modules, and roots to erase + erase := map[string]struct{}{} + names := map[string]struct{}{} + + for name, b := range opts.Bundles { + names[name] = struct{}{} + + if roots, err := ReadBundleRootsFromStore(opts.Ctx, opts.Store, opts.Txn, name); err == nil { + for _, root := range roots { + erase[root] = struct{}{} + } + } else if !storage.IsNotFound(err) { + return err + } + + // Erase data at new roots to prepare for writing the new data + for _, root := range *b.Manifest.Roots { + erase[root] = struct{}{} + } + } + + // Before changing anything make sure the roots don't collide with any + // other bundles that already are activated or other bundles being activated. + err := hasRootsOverlap(opts.Ctx, opts.Store, opts.Txn, opts.Bundles) + if err != nil { + return err + } + + // Erase data and policies at new + old roots, and remove the old + // manifests before activating a new bundles. + remaining, err := eraseBundles(opts.Ctx, opts.Store, opts.Txn, names, erase) + if err != nil { + return err + } + + for _, b := range opts.Bundles { + // Write data from each new bundle into the store. Only write under the + // roots contained in their manifest. This should be done *before* the + // policies so that path conflict checks can occur. + if err := writeData(opts.Ctx, opts.Store, opts.Txn, *b.Manifest.Roots, b.Data); err != nil { + return err + } + } + + // Write and compile the modules all at once to avoid having to re-do work. + remainingAndExtra := make(map[string]*ast.Module) + for name, mod := range remaining { + remainingAndExtra[name] = mod + } + for name, mod := range opts.ExtraModules { + remainingAndExtra[name] = mod + } + + err = writeModules(opts.Ctx, opts.Store, opts.Txn, opts.Compiler, opts.Metrics, opts.Bundles, remainingAndExtra, opts.legacy) + if err != nil { + return err + } + + for name, b := range opts.Bundles { + // Always write manifests to the named location. If the plugin is in the older style config + // then also write to the old legacy unnamed location. + if err := WriteManifestToStore(opts.Ctx, opts.Store, opts.Txn, name, b.Manifest); err != nil { + return err + } + if opts.legacy { + if err := LegacyWriteManifestToStore(opts.Ctx, opts.Store, opts.Txn, b.Manifest); err != nil { + return err + } + } + } + + return nil +} + +// erase bundles by name and roots. This will clear all policies and data at its roots and remove its +// manifest from storage. +func eraseBundles(ctx context.Context, store storage.Store, txn storage.Transaction, names map[string]struct{}, roots map[string]struct{}) (map[string]*ast.Module, error) { + + if err := eraseData(ctx, store, txn, roots); err != nil { + return nil, err + } + + remaining, err := erasePolicies(ctx, store, txn, roots) + if err != nil { + return nil, err + } + + for name := range names { + if err := EraseManifestFromStore(ctx, store, txn, name); err != nil && !storage.IsNotFound(err) { + return nil, err + } + + if err := LegacyEraseManifestFromStore(ctx, store, txn); err != nil && !storage.IsNotFound(err) { + return nil, err + } + } + + return remaining, nil +} + +func eraseData(ctx context.Context, store storage.Store, txn storage.Transaction, roots map[string]struct{}) error { + for root := range roots { + path, ok := storage.ParsePathEscaped("/" + root) + if !ok { + return fmt.Errorf("manifest root path invalid: %v", root) + } + if len(path) > 0 { + if err := store.Write(ctx, txn, storage.RemoveOp, path, nil); err != nil { + if !storage.IsNotFound(err) { + return err + } + } + } + } + return nil +} + +func erasePolicies(ctx context.Context, store storage.Store, txn storage.Transaction, roots map[string]struct{}) (map[string]*ast.Module, error) { + + ids, err := store.ListPolicies(ctx, txn) + if err != nil { + return nil, err + } + + remaining := map[string]*ast.Module{} + + for _, id := range ids { + bs, err := store.GetPolicy(ctx, txn, id) + if err != nil { + return nil, err + } + module, err := ast.ParseModule(id, string(bs)) + if err != nil { + return nil, err + } + path, err := module.Package.Path.Ptr() + if err != nil { + return nil, err + } + deleted := false + for root := range roots { + if strings.HasPrefix(path, root) { + if err := store.DeletePolicy(ctx, txn, id); err != nil { + return nil, err + } + deleted = true + break + } + } + if !deleted { + remaining[id] = module + } + } + + return remaining, nil +} + +func writeData(ctx context.Context, store storage.Store, txn storage.Transaction, roots []string, data map[string]interface{}) error { + for _, root := range roots { + path, ok := storage.ParsePathEscaped("/" + root) + if !ok { + return fmt.Errorf("manifest root path invalid: %v", root) + } + if value, ok := lookup(path, data); ok { + if len(path) > 0 { + if err := storage.MakeDir(ctx, store, txn, path[:len(path)-1]); err != nil { + return err + } + } + if err := store.Write(ctx, txn, storage.AddOp, path, value); err != nil { + return err + } + } + } + return nil +} + +func writeModules(ctx context.Context, store storage.Store, txn storage.Transaction, compiler *ast.Compiler, m metrics.Metrics, bundles map[string]*Bundle, extraModules map[string]*ast.Module, legacy bool) error { + + m.Timer(metrics.RegoModuleCompile).Start() + defer m.Timer(metrics.RegoModuleCompile).Stop() + + modules := map[string]*ast.Module{} + + // preserve any modules already on the compiler + for name, module := range compiler.Modules { + modules[name] = module + } + + // preserve any modules passed in from the store + for name, module := range extraModules { + modules[name] = module + } + + // include all the new bundle modules + for bundleName, b := range bundles { + if legacy { + for _, mf := range b.Modules { + modules[mf.Path] = mf.Parsed + } + } else { + for name, module := range b.ParsedModules(bundleName) { + modules[name] = module + } + } + } + + if compiler.Compile(modules); compiler.Failed() { + return compiler.Errors + } + for bundleName, b := range bundles { + for _, mf := range b.Modules { + var path string + + // For backwards compatibility, in legacy mode, upsert policies to + // the unprefixed path. + if legacy { + path = mf.Path + } else { + path = modulePathWithPrefix(bundleName, mf.Path) + } + + if err := store.UpsertPolicy(ctx, txn, path, mf.Raw); err != nil { + return err + } + } + } + return nil +} + +func lookup(path storage.Path, data map[string]interface{}) (interface{}, bool) { + if len(path) == 0 { + return data, true + } + for i := 0; i < len(path)-1; i++ { + value, ok := data[path[i]] + if !ok { + return nil, false + } + obj, ok := value.(map[string]interface{}) + if !ok { + return nil, false + } + data = obj + } + value, ok := data[path[len(path)-1]] + return value, ok +} + +func hasRootsOverlap(ctx context.Context, store storage.Store, txn storage.Transaction, bundles map[string]*Bundle) error { + collisions := map[string][]string{} + allBundles, err := ReadBundleNamesFromStore(ctx, store, txn) + if err != nil && !storage.IsNotFound(err) { + return err + } + + allRoots := map[string][]string{} + + // Build a map of roots for existing bundles already in the system + for _, name := range allBundles { + roots, err := ReadBundleRootsFromStore(ctx, store, txn, name) + if err != nil && !storage.IsNotFound(err) { + return err + } + allRoots[name] = roots + } + + // Add in any bundles that are being activated, overwrite existing roots + // with new ones where bundles are in both groups. + for name, bundle := range bundles { + allRoots[name] = *bundle.Manifest.Roots + } + + // Now check for each new bundle if it conflicts with any of the others + for name, bundle := range bundles { + for otherBundle, otherRoots := range allRoots { + if name == otherBundle { + // Skip the current bundle being checked + continue + } + + // Compare the "new" roots with other existing (or a different bundles new roots) + for _, newRoot := range *bundle.Manifest.Roots { + for _, otherRoot := range otherRoots { + if RootPathsOverlap(newRoot, otherRoot) { + collisions[otherBundle] = append(collisions[otherBundle], newRoot) + } + } + } + } + } + + if len(collisions) > 0 { + var bundleNames []string + for name := range collisions { + bundleNames = append(bundleNames, name) + } + return fmt.Errorf("detected overlapping roots in bundle manifest with: %s", bundleNames) + } + return nil +} + +// Helpers for the older single (unnamed) bundle style manifest storage. + +// LegacyManifestStoragePath is the older unnamed bundle path for manifests to be stored. +// Deprecated: Use ManifestStoragePath and named bundles instead. +var legacyManifestStoragePath = storage.MustParsePath("/system/bundle/manifest") +var legacyRevisionStoragePath = append(legacyManifestStoragePath, "revision") + +// LegacyWriteManifestToStore will write the bundle manifest to the older single (unnamed) bundle manifest location. +// Deprecated: Use WriteManifestToStore and named bundles instead. +func LegacyWriteManifestToStore(ctx context.Context, store storage.Store, txn storage.Transaction, manifest Manifest) error { + return write(ctx, store, txn, legacyManifestStoragePath, manifest) +} + +// LegacyEraseManifestFromStore will erase the bundle manifest from the older single (unnamed) bundle manifest location. +// Deprecated: Use WriteManifestToStore and named bundles instead. +func LegacyEraseManifestFromStore(ctx context.Context, store storage.Store, txn storage.Transaction) error { + err := store.Write(ctx, txn, storage.RemoveOp, legacyManifestStoragePath, nil) + if err != nil { + return err + } + return nil +} + +// LegacyReadRevisionFromStore will read the bundle manifest revision from the older single (unnamed) bundle manifest location. +// Deprecated: Use ReadBundleRevisionFromStore and named bundles instead. +func LegacyReadRevisionFromStore(ctx context.Context, store storage.Store, txn storage.Transaction) (string, error) { + return readRevisionFromStore(ctx, store, txn, legacyRevisionStoragePath) +} + +// ActivateLegacy calls Activate for the bundles but will also write their manifest to the older unnamed store location. +// Deprecated: Use Activate with named bundles instead. +func ActivateLegacy(opts *ActivateOpts) error { + opts.legacy = true + return activateBundles(opts) +} diff --git a/vendor/github.com/open-policy-agent/opa/internal/compiler/wasm/opa/opa.go b/vendor/github.com/open-policy-agent/opa/internal/compiler/wasm/opa/opa.go new file mode 100644 index 000000000..4861409a7 --- /dev/null +++ b/vendor/github.com/open-policy-agent/opa/internal/compiler/wasm/opa/opa.go @@ -0,0 +1,25 @@ +// Copyright 2018 The OPA Authors. All rights reserved. +// Use of this source code is governed by an Apache2 +// license that can be found in the LICENSE file. + +// THIS FILE IS GENERATED. DO NOT EDIT. + +// Package opa contains bytecode for the OPA-WASM library. +package opa + +import ( + "bytes" + "compress/gzip" + "io/ioutil" +) + +// Bytes returns the OPA-WASM bytecode. +func Bytes() ([]byte, error) { + gr, err := gzip.NewReader(bytes.NewBuffer(gzipped)) + if err != nil { + return nil, err + } + return ioutil.ReadAll(gr) +} + +var gzipped = []byte("\x1F\x8B\x08\x00\x00\x00\x00\x00\x00\xFF\xD4\xBD\x09\x90\x65\xD7\x59\x18\x7C\x96\xBB\x9F\x7B\xBB\x6F\x8F\x34\x92\xAC\x96\xAD\xEF\x5E\xDB\x30\xB2\x34\x42\xB6\xE5\x91\x35\x2C\xD3\xA7\xAD\x99\x51\x6B\x10\x63\xB0\x0C\x48\x80\x7B\x5A\x3D\x6F\x46\xFD\xBA\xA7\x7B\xFA\xF5\x9B\x91\xC4\x3F\x9E\xD7\xDA\x2C\x2F\xFC\x05\x55\x54\x12\x23\x42\x90\xE5\x24\x22\x4E\x54\x09\x90\x20\x0C\x45\x0C\x45\xC5\x24\x54\x19\x5C\x31\x14\x2E\x70\xC0\x86\xD8\x09\xC1\x18\x8A\xAD\x5C\xA1\x40\xA9\xEF\xFB\xCE\x5D\xDE\xD6\x33\xC6\x14\x4B\xAB\x34\xEF\xBD\x73\xCF\xF2\xED\xDF\x77\xCE\x3D\xE7\x7C\x62\x65\xE7\x9C\x14\x42\xC8\x1F\x94\xD7\x9D\x52\x83\x81\x1C\x9C\xF2\x06\x83\xC1\x40\x9C\xD2\x03\xFA\x25\x07\x82\x4B\xE4\xE0\x94\x3F\x70\x5F\x82\x41\xF5\x4D\x88\x53\x82\x6A\xC9\xC1\x29\x35\xB8\x8C\xFF\x5E\xBA\xC4\x05\x97\xA8\x0B\x71\x4A\xD2\x6F\x7A\x84\x3F\xD5\xE0\xB2\x38\x15\x61\xF3\x4B\xDC\x85\x19\x34\x7F\x72\x70\x4A\x5F\x1E\x70\x67\x97\xE4\x40\x5D\x0E\x75\x67\xF3\x62\x70\xAE\x73\x6E\xAB\xF7\xB8\x12\x0A\x7F\xC5\x5B\xE7\x57\x96\x57\x1E\xDE\xEA\xF5\x85\xC6\xDF\x29\xFE\x7E\xF8\xC2\xDA\x46\x7F\x6D\xF3\x0E\x21\x46\x8B\xDE\xC8\xAD\xDA\x45\x6F\x12\xDE\x68\xD1\x9B\x85\x3F\x5A\x74\xA7\x08\xF4\xD3\xF2\x49\x19\x46\x91\xD6\xB1\x8E\x05\xFE\x25\x82\xFF\x62\xFC\x33\x22\xC5\xAF\x5A\x68\xAD\xB5\x10\x59\x1C\xC7\x33\x62\x36\x16\xB1\x8A\xE3\x3C\x8E\xE2\x58\xE8\x28\x8E\xA2\x7C\x2E\xCF\x45\x2C\x94\xE7\x49\xB9\x6F\xDF\x35\x5E\xAC\xA8\xF9\xB5\x4A\x45\x79\x1E\x87\x71\x1C\x7B\x1A\x0B\x32\x25\x62\x11\x0B\x1D\xC7\x0A\xBF\xED\x4F\x84\x50\xC2\x57\x2A\xF6\x7C\x79\x5E\x06\x41\x50\xF8\x03\x69\x9F\xFF\x80\x67\x06\xA2\xFA\xD8\x8D\xF0\xDF\x7F\x6E\xF0\xDF\x0F\x1A\x13\xFE\xF0\xFE\xC1\xDC\xF2\xF2\xA3\x2B\x3B\xE7\x96\x57\x57\x36\x36\x96\x57\xFB\x5B\xBD\x1D\x11\x98\xE5\xE5\x47\x3A\x2B\xE7\x97\x1F\x5E\xD9\xE9\x68\x99\x2E\x2F\x9F\xDE\xD9\x5A\x7E\x64\x65\xF3\xF4\x46\x47\xAB\x64\x79\xF9\xF4\x4A\x7F\x65\xB9\xB3\x79\x5A\xEB\x1C\x29\x40\x75\xCF\xF7\x7B\xCB\x67\x3B\x7D\x11\x36\x45\xFD\xAD\xF3\x54\x14\x0D\xD7\xDA\xE9\xF4\x45\x3C\x5C\x0B\x8B\x92\x04\x8B\xCE\xAD\x6C\x6C\x6C\xAD\x0A\x13\xE1\x8F\x33\xBD\x4E\x47\xA4\x33\xF8\xF5\xE2\xCA\xC6\x85\xCE\x72\xFF\xF1\xF3\x1D\x91\x5D\xDB\x14\x9C\xED\xF4\x97\xB7\x1E\xEE\x76\x56\xFB\x62\x66\xAE\x29\x5E\xDD\x3A\x77\x7E\xA5\xD7\x11\xB3\xD4\x96\x2B\x10\x28\xF9\xDC\x70\x5B\x1C\x78\xCE\x60\xD9\x4E\x87\x6B\xEC\xBB\x71\xB8\xC6\x4A\xAF\xB7\xF2\xF8\xF2\xE6\x4A\x7F\xED\x62\x47\x5C\x73\xCD\x84\x87\xE2\x5A\x82\x7C\xA5\xBF\xB5\x76\xE8\x4E\xB1\xBC\x0F\x7F\x6C\x5E\x38\xF7\x70\xA7\xB7\xDC\xEF\x3D\xBE\xBC\xB6\xD9\x17\xFB\xB3\xA1\x76\xE2\xBA\xFD\xCD\xEF\xB5\x7E\xA7\x57\xE1\x70\xFD\xBE\x91\x72\x84\xEF\x86\x6B\x47\x0A\x79\xD4\x57\xCD\x0C\x17\x8B\x1B\xAF\x6F\x0A\x36\x3A\x9B\x67\xFB\x8F\x54\xDD\xCE\x5F\x3B\xF6\x04\x3B\xBE\xE9\xBA\xB1\x62\xEE\xFA\xD5\xE3\x3D\xED\xF4\x7B\x6B\x9B\x67\xC5\x6B\xF2\xD1\x27\xE2\xE6\xEB\xC7\xE8\xBE\x7C\x66\x63\x6B\xA5\x2F\xE0\x86\xF1\x27\x4C\x18\x51\x54\x24\x3B\x73\xE8\x4E\x71\xEA\x9A\x16\xC9\x56\x76\x5C\xEB\x72\x42\x6B\x07\xC5\x6B\x99\x63\xFD\xDE\xE6\xEA\xB9\xF3\xE2\x3B\x27\x00\xC0\x78\xBC\x6E\x42\x17\x8E\x24\xAF\x9F\x6D\x49\xC6\x7A\xE7\xF1\x1D\xF1\x35\x44\x4F\x66\x37\x09\xDE\xD7\xEE\x9F\x00\x40\xA7\x2F\x0E\xB4\x08\x4F\x15\x6F\x99\x75\xE0\xAC\x6D\x9E\xE5\x92\x37\xB4\x7B\xA7\x92\x5B\xD3\x4A\xC8\xE8\xE7\x6D\xB3\x4D\x1F\xE7\x3A\xBD\xB3\x1D\x71\xB0\xDD\x84\xF8\x79\xFB\x5C\xBB\x64\x73\xA7\xD3\xEB\x8B\xAF\x7B\x4D\xD3\x6E\xE7\x11\xD4\x95\x47\x97\x57\xB7\xCE\x3F\xBE\xFC\xF0\xD6\xD6\x46\x67\x65\x53\xDC\x41\xA4\xA9\x7E\xBD\xF1\xD5\x53\xAA\x3B\x36\xBC\x29\x6F\x51\x9E\xC9\xFE\xE6\x99\x56\x51\xAF\x73\x46\xDC\xD9\x2E\x40\x61\x7E\xCB\xB4\x4E\x1D\x77\x0E\x25\x0D\x39\xC4\x5D\x37\x4D\xA9\xCC\x1C\x7A\xEB\xB5\x0D\xD1\x1F\x5D\xEB\x3F\xB2\xDC\xD9\xE8\x9C\xDB\x11\x77\x4F\x1B\xC2\x71\xEF\xF0\x8D\xD3\x40\xE8\xF4\xC5\xD7\xD7\xFA\xBC\x72\xFA\xB4\xF8\x86\xEB\x26\x57\x15\xDF\x18\x31\x5E\x1B\x1B\xE2\x9B\x66\x5B\x28\xEE\xAC\x7D\x6F\x47\x1C\xD9\xDF\x62\x69\xBF\xD3\x3B\xB7\xB6\xB9\xD2\xEF\x9C\x16\x0B\x15\x6E\x1B\x9D\x4D\xF1\x1D\xD7\xB6\x2A\x91\xDD\xA2\x3A\x36\x5F\x5E\x6E\xB0\x3A\xDB\xDB\x7A\x54\x2C\xC6\x75\x81\x78\xDB\xBE\x11\x94\x57\x57\xCE\x8B\x7B\x92\x86\xD7\xE2\x68\xE8\x10\x10\xC7\x5A\x22\xE8\x78\x4A\xE5\xC7\x5B\x82\x5D\x81\xDD\xE9\x13\x77\xEE\xCD\x9B\xEE\x57\xCE\x9F\xEF\x6C\x9E\x16\x4B\x2D\xC9\xDE\x41\x47\x78\xDF\xF5\x0C\xA1\x93\x2D\xA4\x39\xC3\x2F\x4E\x5C\xCB\x4F\xB0\xB7\x56\xF1\x37\xD7\xD2\x4B\x92\x79\x3F\xBA\x58\x84\xE3\x5B\xE2\x9D\xCD\xF3\xBD\xB5\xCD\xFE\x99\x65\x71\x32\xB9\xD8\xFC\xF8\xF6\x8A\x4E\xA8\xA0\x0F\x12\x43\xD6\x76\x4E\xAF\x9D\x5D\xEB\x8B\x87\xDC\xAF\x9D\xF3\x2B\xAB\x1D\xF1\x5D\x31\xFF\x7A\xA4\xF3\x98\xF8\x6E\x7A\xD2\xEB\x5C\xEC\xF4\x76\x3A\xE2\x7B\x88\x3F\x6B\xFD\xAD\x15\xF1\xAE\x80\x9A\xEE\x68\x8F\x90\xEB\x5C\x5C\x41\x57\xF5\xD8\xF2\x66\xE7\x51\xB1\x72\xDD\x50\x11\x93\xE1\xFC\x85\xBE\x78\x78\xFF\xD8\x03\xF4\x5B\x62\xF5\xFA\xA1\x72\x34\xE3\xBD\xCE\xCE\x85\x8D\xBE\x38\x3D\xBF\xBC\x7C\x66\xAB\xB7\xDA\x59\x5E\x3B\x77\x7E\xAB\xD7\x5F\x6E\xB9\xF7\x1D\xD1\x21\x2B\xD5\xDD\xD9\xDA\x5C\xDE\xE8\x3C\xB6\xBC\x75\xE6\x0C\x52\xE0\xCC\x75\x43\xA5\xBD\xCE\xB9\x95\xB5\x4D\x14\xFC\xB3\xF9\xD0\x83\xCE\xD6\x19\xF1\xC8\x68\xDD\x95\xD3\x68\x02\xCF\x89\xB5\x1B\xC6\x1F\x30\xC6\xA2\xFB\xAA\xF1\x47\x17\x36\xD7\x56\xB7\x4E\x77\xC4\xFA\x84\x66\x4E\xB1\x37\x26\x3C\x72\x1A\x79\x6E\x6E\xEC\x91\xD8\x1C\x2E\x5B\xDB\x5C\xEB\x8B\xAD\xFD\x75\xD9\xF9\x95\xDE\x4E\x6D\x7C\xCF\x8F\x96\xBB\x21\xB7\xAF\x1D\x29\x67\x71\xEF\x8D\x56\x77\x42\xDE\x9F\x19\x2E\x17\x17\x9A\xF6\x8F\xF6\xC8\xCF\x11\x18\x17\xAF\x1B\x2D\x76\x94\x7F\x74\xFF\xE8\x03\x96\xA8\xC7\xC6\xFA\x21\x15\x7C\xFC\xC6\xD1\xE2\xCE\xB9\xB5\xFE\xF2\xEA\x23\x2B\xBD\x1D\xF1\xBD\xAF\x9A\xFA\x50\xFC\x7F\x93\x9F\x91\xE1\xB8\x74\xD3\xC4\x67\x95\xF9\x7D\xF7\xE4\x31\xD9\xD0\x5E\x9E\xDC\x76\x6D\xB3\xDF\x39\xDB\xE9\x89\xC1\xFC\x94\x61\x89\xD8\xBB\x72\xF2\x63\xC7\xA3\x27\x64\x39\xF1\x31\xDB\x00\x54\xEC\xCE\x66\x5F\x3C\x29\x27\xC3\x47\x76\x45\x3C\x25\x6F\x9E\x4C\x94\xAD\x8D\x8D\xCE\x6A\x7F\x6D\x6B\x53\x3C\x2D\x5F\x3B\xB1\x4A\xCB\xAE\xE0\x38\xCF\x48\x98\x0C\x6C\xA7\xA9\xF3\x1E\x99\xD5\x75\x4E\x5F\x38\x77\x5E\x3C\x2B\xE3\x19\x29\xAC\x34\xFE\xB7\x7E\xDB\x93\xF2\x19\xF9\x1E\x99\x7C\xFC\x57\xE4\x93\x52\x09\x63\x84\x15\x07\xD4\xEE\x7B\x77\x77\xAB\xEF\x4F\xD3\xF7\x4C\x58\x01\xE2\x90\x7B\x52\xFD\xE2\x67\x3F\x22\xA5\x1E\xA8\x05\xB5\x60\xC5\x41\xF1\x5E\x2C\x3A\x9A\x35\x4D\x0B\x99\x4A\x63\x31\x4C\x7E\x7A\xB7\xEA\xC0\x8A\x05\x51\x48\x2B\xAC\x3C\xCC\x0D\xAC\x00\x69\xF3\x7E\x29\xAB\x3E\x6D\x03\x47\xA9\x40\x74\x4B\xED\x9A\xAA\x05\xD0\x20\x97\x32\x01\xC2\xE6\x17\xAD\xEC\x96\x62\x41\xDC\x68\x9B\xF1\xB0\xBC\xDF\xAD\xFA\x01\x65\x94\x30\xA1\x00\x71\x50\x08\x73\x48\x20\x98\x20\x0E\x28\xAF\x44\x20\x35\x7D\x17\x20\xF3\xEF\xDF\x25\xB0\x15\xFE\x8E\x4A\x91\x09\x63\xAC\x98\x35\x54\xD5\xFC\xB0\xE7\x05\x03\x9C\x37\xA9\xCB\x76\x00\x02\x64\xCF\x8A\xE3\x20\xE7\x0B\x85\x9D\x1D\xCD\x04\x48\xC4\x78\x50\x28\x1A\xA6\xD4\x20\xF1\xC3\x5B\xCA\x84\x95\x85\x02\x0F\x34\x7E\x15\x58\xBF\xF9\x0F\xB4\x1D\x74\x4B\x6D\x83\x13\x19\x61\x35\x13\x86\x42\x69\x2F\xF0\x43\x1E\x16\xE4\x01\xE5\xAD\xCF\x1A\xFB\x03\x88\x49\xBE\xBB\x4B\x74\x42\x98\x40\xE6\x1F\xC2\x5F\x0C\x5F\x84\x15\xE9\x1F\xAF\x54\x8C\x9B\x06\x05\x7A\x69\x3E\xFF\xF4\x2E\xD1\x2F\xD3\x0C\x9B\xF5\xBA\x07\x94\x28\x91\xD6\xFC\x4D\x2E\x65\x1A\xBB\x3B\x31\x6B\xD4\x02\x75\x53\xFA\x4C\x81\x00\x7C\x08\x96\xCA\x70\xBE\x54\x88\x9F\x7A\xA9\x88\x40\xDA\xA8\x5B\xC4\xD6\x2B\xF4\xA2\x2C\x12\x08\x0A\x4F\x2F\xB4\x91\x82\x64\x71\x70\xA9\x34\xE0\xBD\xF4\x10\xB1\x87\xC6\x00\x8D\xFF\x16\x0A\x0C\xC4\x6F\x51\xE2\x81\x4C\xA6\xCA\xB8\x1F\x0F\x65\xD2\x80\xAA\xA0\xE1\x9A\xCC\x09\x1C\x55\xA6\x81\x01\x65\xEB\x92\xCC\x37\x90\x40\xF4\x50\x26\x41\x23\x24\x1A\x92\x45\x79\xA9\x48\x40\xD8\x88\xC6\xF0\x52\xE4\xDA\x00\x42\x08\xC0\x5F\x9A\x77\xDC\x2B\x04\xA3\xE6\x95\xC8\x24\x18\x62\x7B\x6B\x3C\x24\x04\xD5\xAE\xEB\x4A\x60\x31\x40\x0E\x8A\x4C\xDA\x01\x0A\xE9\xBC\xA3\xFF\xBF\x72\xF4\x57\xE6\xCD\x5F\xB9\x4C\x5D\xA5\x1C\x7A\xC3\x72\x28\xAE\x72\xA8\xA1\x66\xE6\x80\x94\x03\x66\x12\x88\xB7\xA0\x98\x7C\x07\x12\x81\xC4\xEB\x45\xAB\xFB\x5D\xC4\x59\x21\x1E\x9F\x90\x4A\x0F\xE4\xE5\xD7\xB2\x9C\xE5\xEB\xA5\x7A\x9D\x13\xB9\x42\x23\x05\x0F\x0A\x61\xF5\xF1\xCC\xFD\x26\x40\xE4\x41\x21\x4B\xCF\xEA\x63\x99\x00\xCF\xAA\x63\x99\x06\xCF\xCA\xE3\x99\x44\xA6\xDE\xA2\xA3\xD2\xBF\x4B\x47\xE0\x2F\x8A\x77\x64\x1A\xB9\x5E\xC9\x69\x0A\xCA\x46\xDD\xFC\x77\xB0\xFB\x4C\x81\xA2\xAA\x8B\xE2\x41\x92\x0D\xFB\xCB\x8D\xB8\xA7\xD2\x80\x4F\x70\x3F\x58\x41\xED\xD7\x50\x4B\x50\x36\xEF\x32\x90\x20\x91\x17\xCD\x4F\x6D\xBE\x53\xCA\x81\x03\x53\x20\x98\x8A\xC1\x54\x08\xA6\x02\xC5\x60\x4A\x10\xB7\xE8\xE8\x2E\x2D\x6C\xAD\x4B\xC2\xA6\x5D\xA6\xE9\xEF\x30\x8B\xDB\xF0\x18\x3B\x30\x5F\x52\x7B\xD2\x09\xF9\xD2\xD6\x72\x1E\x5F\x94\x9E\x0D\x99\x4C\xC1\xB1\x4C\x82\x67\xFD\xE3\x59\x58\x13\x35\xC4\xC6\x2D\x72\x7A\x4C\xCE\x90\xC9\x19\x8C\x92\xF3\xC1\xCC\x4F\x9D\x91\x40\x39\x08\xF6\x90\x83\x34\x34\x4D\x45\x7F\x0F\xD9\x4C\x03\xAE\x28\x0A\xD1\x26\xA4\xA8\xB5\x68\xB4\xF4\x0A\xCC\x7C\x47\xA6\x1A\xDE\xC9\x51\xDE\x8D\xF6\x36\x44\xE4\x61\x46\xBE\x4B\xAA\x41\xAD\xC8\x95\xE0\x1D\x65\x73\x8A\xD4\xD6\x23\x48\x15\xA4\xC8\x51\x21\x40\x13\x7E\x99\x48\x3D\x44\x2D\x93\xCC\xE4\xA3\x68\x74\x08\x51\x12\xFB\xBF\x59\xF7\xDE\xD5\x76\xFF\xBE\x29\xD2\xD2\x12\x0F\x37\x9A\xAE\xA5\x41\x8F\x48\x83\x1C\x52\x2E\xAF\x2D\x0D\xA9\x32\x6C\x10\xA2\xA3\x4E\x41\x70\xE4\x3D\x34\xC3\x4E\x65\x9D\x24\xD6\x15\x3E\x5A\x64\x7F\x51\x5E\x6A\x29\xAF\x98\xA6\x84\xA2\xD0\xC3\xDC\x1A\xE6\xE4\xD0\xB8\xE6\xB7\xF5\x57\xA8\x38\x7B\x2A\x4F\x8C\x74\x6B\x68\x16\x8F\xD0\x2C\x60\x9A\xC5\x4C\xB3\x68\x88\x66\x61\x2D\xD2\x6C\xE5\x8F\x66\xD1\x28\x93\xBD\xAA\x86\x57\x31\x39\x36\xED\x56\x24\x1A\xE1\xC4\x56\x11\xB7\xF2\xA8\x55\x64\x48\x24\x82\xE9\x9A\x15\x1D\xCD\xFC\x9A\x71\xE3\x15\xA8\xA3\x69\x8F\xF4\x94\x47\x7B\xA8\xE6\x44\xFE\xCA\x69\x8A\x3A\xDE\xF7\x1E\x8A\x7A\x90\x2C\x2E\x1B\x1B\x27\xD3\xE4\xA1\xAC\xEC\xB2\x57\x8D\x4A\x99\x09\x43\xBD\x88\x2B\xD7\xF6\xDA\xB5\x43\x12\xBC\xC8\x7D\x7A\xE6\x09\x29\x15\xBA\x35\xE9\x24\xC4\x5E\xEA\xA2\x85\x3F\xD1\x88\x10\xA8\x19\x4F\x48\xA5\x05\xB3\xCD\x11\x7B\xB6\x31\x86\x12\x9B\xEB\x05\x90\x38\xA2\x6C\xEC\xA0\x6A\x19\x4C\x31\x56\xC7\x39\x57\x90\x66\x56\x70\x2C\x78\x1A\xFF\x59\x9D\x37\x9F\xF7\xB4\x1A\xA8\xCB\xEA\x92\x13\x70\x98\xA0\xEB\x93\xFE\x63\xF7\xA4\xD9\x3D\x69\x36\xFB\x1A\x85\x56\x81\x72\xEE\x29\x67\xC1\xD6\x28\xE8\x32\xD5\x23\x9E\x0A\x99\xE0\x78\xEC\xD7\x15\x8F\xA1\xF9\xA5\xDE\x3C\xEE\x4D\x57\x2A\xE0\xDD\xA5\x23\x3B\x40\xC6\xDE\xA2\xF3\xD2\x07\xEF\xC1\x4C\xA5\xE9\xB0\xDB\xD5\x95\x0C\xD5\x43\x34\x62\xD4\x6E\x4B\xC6\xDE\x7B\x47\x96\xA0\x28\x40\x25\x0A\x3E\x78\xEF\x1C\x95\x14\xE4\x2B\x85\xC6\x56\x1E\xA3\x00\x0E\x21\x44\x00\x8F\x67\x88\xCF\xAD\x3A\x2A\x02\x06\x5E\x60\x0D\x9D\x7A\x86\x90\xFF\x68\xAB\xF8\x38\x45\x11\xCE\xDE\xB5\xD0\xBF\xA1\x9B\x7F\x76\xD7\xCD\x3A\xFE\x60\x48\x3C\x6F\xD5\x37\x0C\xB5\x57\x38\x15\xF9\xBE\xF7\xD5\x55\xEE\x71\xEF\x10\xC4\x50\x2D\x69\x88\x52\x1F\x2D\xC2\x54\x1A\x66\x11\x73\x47\x20\xB8\x12\x24\x42\x1B\x32\x28\x23\x64\xDA\x1B\x94\x70\xEA\xF0\x61\x9B\x80\x76\x00\x01\x84\xA7\xF1\x9F\xD5\xF9\xD9\x21\xD2\x0A\xF3\x71\xA9\xD5\x40\x5E\x6A\xDB\x51\xD9\x16\xB3\x2A\xDA\x91\x1C\xED\x90\xFB\x50\x08\xB7\x62\x2A\x6B\x9C\x68\xB5\xEC\x35\x51\xB9\xF0\xDA\xA5\xDE\x47\x67\xC7\xA8\x8C\xB1\xFF\x14\xD4\xE4\xA4\x6E\xDB\x68\xB6\x1E\xD5\x18\x9B\x77\x48\x35\xA8\x06\xB9\xE2\xFC\x45\x4C\x9F\xBF\x08\x9A\xBF\xB0\xB3\xFD\x82\x0C\x3C\x9A\xA7\xC9\xCB\x0A\xFF\x77\x53\x1A\x17\xE4\x70\xA7\xA5\x37\x5F\xFA\xE8\x2C\xFD\x97\x90\xE9\x38\x91\x08\xAD\x57\xF8\x8B\xB2\x88\x40\x17\xB1\x5A\xA0\x49\x4D\x52\xAB\x67\xC4\x93\x9A\xF8\xA5\x07\x32\x01\x06\xC2\x66\x12\x53\xCD\x70\x7C\xB2\x94\x89\x7B\x48\x93\x9A\xA4\x9E\xD4\xF8\xAD\x49\x4D\x72\x94\xD5\x37\xA9\x27\x35\x09\xAA\x69\x04\xC1\x43\x99\x02\x1F\x61\xF1\x21\xC2\x49\x4D\x54\x4F\x6A\x62\x9C\xD4\x40\x32\x8B\x13\x1B\x0F\x34\xA8\xA5\x79\xF3\x39\x4F\x66\x03\x10\xF9\xBF\x26\xAF\xA3\xAA\x38\x82\x7F\x97\x1A\x91\xF5\x40\xF1\x6C\xCE\xC3\xA9\x50\x19\xD0\x24\xCE\x7A\x88\xAC\x28\xD0\xDD\x85\x08\x5C\x64\x75\xBF\x8C\x31\x22\x03\x6D\xBD\x6E\x99\x60\x21\xFD\x76\xF0\x99\x4C\x59\x51\xA4\x56\x14\x59\x63\xAF\x1C\xF1\x0D\xF6\x19\xD6\x2D\x8A\x19\xBD\x00\x06\x7F\xCE\x34\xD1\xA5\x41\x20\x4C\x86\xF3\xF9\x22\x33\x35\xDF\x8C\x9B\x05\x62\xB3\xAC\x29\xA5\xD8\x29\x69\xF5\x88\xCC\x18\xE9\x53\xD6\x7D\x92\xA5\xA6\x2E\x52\x03\x19\xA4\x0D\xC8\x10\x59\xD9\x2D\x23\x08\x96\xC8\x56\x7B\xE0\xD3\x9C\xA4\x42\xDF\xA0\x74\x23\xE0\xE5\x0C\xC5\x1C\x64\xDE\xCA\xF8\x80\xA2\xA9\xFC\x0C\xF2\xEB\xFB\x58\xA1\x00\x0B\xAD\x5A\xAA\x88\x17\xD9\xA8\x98\xB1\xB2\x88\xF5\x02\x44\x04\x56\xB7\xAE\x0D\x33\xC8\xBF\x19\x88\x71\xEC\x98\xFA\x3C\xA0\x04\x43\x80\x83\x15\x33\x06\x66\xB8\xAA\x01\x55\x8F\xA0\x51\xDE\x1D\x1C\xFA\x2A\xE0\xD0\x5F\x21\x1C\xBA\x0D\x87\x66\x4A\x0F\x81\xA2\x5D\x33\x83\x22\x66\x25\x52\xEB\xC4\xBC\xF9\x6B\x5F\x1A\x72\xB0\xD5\xDC\x91\xD4\xD2\xC5\xC5\x1A\x49\x26\xBB\x85\x02\x92\x35\x4D\xD3\xCD\x3C\xFF\x00\xD1\xDF\xB3\xFE\x61\x21\xC0\x03\x75\x48\xA5\xE0\x2D\x8A\xBB\x94\x57\xB9\xE4\xA3\x14\xC4\xA5\x5D\x6C\xDE\x3F\x44\x16\xD3\xEB\x73\x3B\x72\xE5\x1E\x12\xC0\x67\x02\x50\x00\xEE\x21\xB6\x01\xC6\x76\xE8\x85\x41\x75\x21\x20\x43\xDB\xBD\x45\x89\xBB\x9C\x63\x42\x30\x10\x57\x0D\x7E\x8D\x28\xB6\x2B\x55\x43\x13\x05\x12\x87\xF3\x29\x2E\xB7\x1E\x06\xF1\x4E\xE8\x42\x9E\x74\x8E\x15\x93\x2C\x22\xAC\x45\x44\x60\x15\xA8\x4A\xBA\x5B\x04\x4E\x77\xD0\x52\x80\x22\xEA\x97\x1A\x19\x13\x21\x14\x7D\x9B\x80\x9E\x2F\x15\xE3\xA6\x2B\xDC\x90\x85\x2D\xB4\x1C\x3E\xE2\xEA\xF1\x09\x1A\x54\x02\x10\x0D\x2A\x38\xC5\x42\x56\x1C\x52\xA2\x66\x80\x06\xF5\x12\x86\x76\xDA\x3E\xA5\xBE\x5E\x0A\x6A\x5B\x4A\x02\xA7\x5B\x2A\xD0\x04\x1B\xC4\x87\x30\xCE\xE3\xA5\x99\x10\xB9\x18\xE2\x40\xA4\x0C\x24\x67\x44\x79\x61\xA3\x22\xB1\x92\x0C\x86\x64\x6B\xA1\x98\x41\x85\xA1\x02\x5A\xC0\xE2\x39\x1D\x76\xE6\x95\xB1\x0B\x85\x29\x58\x12\x18\x22\x74\x8B\x80\xCD\x93\x95\xF7\x66\x02\x92\x42\x41\x44\x41\x55\x80\xB0\x77\xED\x63\x15\xEE\x82\x29\x61\x29\xF0\x84\xAA\xA3\xB0\xEA\x88\xC6\x29\x02\x50\xF6\x31\x24\x91\xB4\x03\x0C\xC9\x34\x7E\x68\x08\xAD\xB8\x0F\xA3\x45\x1E\x2F\x95\x06\xC7\x30\x10\x80\x21\x5A\x81\xC4\xC6\x8C\xB3\xB3\x0D\x1E\xEB\xC5\xC9\x4C\x42\x82\x44\x4F\x38\x96\x96\x64\x6E\x3D\xF3\x7D\x52\x7A\x43\xF1\x69\x6B\x9A\x25\xDB\x9A\x89\x76\x9A\x24\x8D\xC9\x86\xF2\x53\x68\x1B\x15\xD2\xCA\xC2\xD3\xA4\xDF\x02\x64\x4B\x3D\xC9\xE5\xE0\x24\x46\x76\xD9\x4A\x3B\x16\x3B\xC9\x43\xB0\x65\xA5\x9B\xAE\x8D\x79\x58\xCA\xE1\x49\xEA\xD5\x2C\x68\xA9\x3D\x17\xB4\x48\xBE\x15\xE9\xBC\x00\x39\x6F\x7E\x4D\x39\x84\x31\x9A\x1E\x74\x4B\xE9\xD6\x25\x47\x82\x55\x39\x13\x86\x61\x88\x81\x35\xCE\xA2\x0E\x0A\xE9\x26\x37\x11\x43\x9A\x06\xA6\x26\x99\xFF\xF7\x40\xB2\xD4\x6F\xC6\xF7\xD8\xFB\x7B\x47\x33\x0A\xE1\x5D\x5F\x5E\xA9\x70\x7E\x4C\x4F\xB2\xC6\x12\x53\xBC\xE9\x1A\x6A\x17\x8B\x0C\x37\x54\xA5\x87\x30\xD0\x93\x0C\x43\xE4\x66\x50\x65\x40\x16\x9E\x01\x6F\x94\x6F\x06\x84\x8B\xC8\xDC\xF4\x3C\x1A\xAD\xF1\xAD\xB4\x74\xDE\x16\xB3\x3A\x6A\x61\xBE\x36\xC3\x6B\x14\x77\x7A\xE2\x7C\x9F\xC4\xD9\xB7\xBE\xDA\x1E\xBD\xA9\x3D\x7A\x57\xE8\xF1\x09\x5F\x86\x83\x6A\xA1\x97\x05\x24\x38\x8E\x4C\xAD\xBE\xD9\xC8\x99\x1E\x65\x05\xAA\x98\xB2\xC1\x61\x21\xDA\x5E\x83\xDD\xA7\x47\x66\x04\xD9\x9B\xFF\x53\x52\x6E\x9F\x86\x67\x03\x31\x22\x68\x14\xA3\x05\xE0\xE7\x2F\x51\xCF\xC1\xD1\x2C\xA5\x96\x45\x88\x3C\x10\x65\xC4\xDC\x89\xC8\x1A\x41\x58\x85\x07\x64\x86\xA2\x32\x72\x1C\x1A\x69\xE1\x4D\x68\x11\x36\x2D\x02\x83\x8E\xC0\x37\x36\x75\xF8\x44\x10\x1E\xA2\x30\x49\x1C\x52\x11\xC6\x69\x64\x34\x8F\xD2\x84\x39\xC2\x22\xE7\xFF\x82\x34\x44\xDB\x43\x8F\x5D\x91\x8F\xAA\xE0\x61\x2D\x51\x15\x79\xA9\xFF\x37\xEC\x5F\x61\xD4\x38\xDC\x3F\x05\x50\xC3\xFD\xD3\xBC\x74\x98\xE2\x52\x13\xCB\x5A\x34\xC7\x47\xED\x2E\x98\x01\x8E\x50\x8D\x9A\x3B\x82\x89\x09\x04\xD3\x0D\xC1\xD4\xDF\x94\x60\xD3\x10\x92\x63\x08\x29\x9A\xFF\xEF\x63\x23\x40\x1A\x0F\x72\xFE\x80\x12\xE6\x02\xAF\x00\x54\x6B\xD1\x24\x6A\x1E\x43\xEC\x95\x7A\x64\x95\x53\x57\xFA\xDE\x80\xEA\x71\x48\xE0\x31\xA8\x14\xAE\x78\xA0\x11\x54\x0D\xDE\x21\x5A\x2A\xD0\x54\x38\xCB\x86\x06\x0B\x85\x79\x6D\x6D\x89\x65\x2D\xF5\x02\x7B\xF2\x70\x76\x88\x81\x8F\x30\x37\x49\x39\xA8\x1F\x4A\x20\x95\x90\xFC\x50\x9A\xE7\xA4\xA2\xF9\x49\x6B\xB2\x26\x79\xB2\x26\x79\xB2\x26\xDD\x64\x0D\xA7\x65\xAA\x76\xEA\x02\x14\xFA\x74\x61\x9F\xF2\xD0\xA7\x8B\x6A\xE2\x36\xA9\x86\x9A\x56\xC3\x3E\x15\xD0\x23\x57\x73\xD6\xD8\x7F\xD1\x7E\xB5\x64\x5E\x8D\x80\xE7\x0D\xE0\x77\x6B\x9C\xDB\xF2\x88\xD2\xBC\xAE\xFD\x54\x01\x91\x4D\xB9\x2E\x29\xBE\x4A\x41\x8D\x75\x71\x17\x77\xA1\xB8\x8B\xD7\x4A\x79\x19\xC4\x2D\x44\xBD\xB4\x86\x8A\x31\x02\x79\x17\x12\x91\x28\x5C\x3F\x54\x4C\x5A\xC5\x75\xAA\x41\xCD\x0F\x49\x19\x10\x1B\xD2\x26\xC6\x92\x95\x18\xA4\xF5\xEA\x2D\x39\x16\x9A\xC2\x16\x1E\xCA\x69\x15\x43\xEA\x49\x31\x97\x8F\xDE\xC5\x67\x53\x81\x13\x07\x27\xDC\xEC\xD0\xA2\xC2\x6F\xA1\xEE\x33\xEA\x14\xE7\x2A\x14\xE2\x14\x2A\x40\xCD\x1B\xDA\x04\xD0\x58\x8A\x91\x18\x55\xD5\x1C\x12\x6B\x96\x08\x6D\x7E\x0F\xB1\x68\x04\xC5\x3A\x41\xA9\x6C\xA7\x15\xF9\x73\xBB\xEE\x5D\x18\x4F\x79\xAB\xA8\xD1\x59\x4F\xAF\x79\x93\x48\x21\x12\x0B\xBF\x5F\x72\x48\xD9\xA8\x6B\x40\x33\xC1\x16\x46\x35\x79\x7D\xA0\x48\xD5\x67\x1D\xF0\x51\xC4\x3D\x08\x50\x07\x02\x46\x12\xFB\x79\xAE\x8A\x27\x50\x67\x03\xAE\xD3\x2E\x67\x33\x82\xF5\xC5\x50\x39\x2F\x93\xFD\xAA\x44\xE7\x31\x86\x64\xB8\x27\x92\xC3\x88\x79\xCC\x50\x9F\x34\xBB\x8D\x5C\x49\xEB\xAC\x1E\xC6\x7E\x0A\x82\xC2\x47\x4C\x3D\x52\x73\x2B\x68\x81\x06\x3F\x10\x63\x0C\x08\x83\x22\x04\x1F\x91\x8F\x6A\x03\xA0\x6A\x03\xE0\x41\x80\x2A\xAE\x21\x98\x27\x3D\x07\x0F\x0D\x19\xC6\x04\x63\x08\x7D\x00\x23\xC3\x2A\x8A\x1F\x36\x3D\x08\x24\x4D\x59\x87\x0D\x10\x3E\xD3\x0C\xA6\x04\xED\x9C\x5E\x05\xA6\x42\xD8\x34\xCA\x5F\xA1\xDB\xB0\xC9\x16\x6C\x38\x6D\x63\x1B\xA4\x6B\xF0\xD0\x32\x19\xF3\xB3\x5A\xAB\x81\xBC\xEC\x0D\x1A\xD7\x3C\x35\x76\x6B\x22\xB8\x54\x48\xAD\x3C\x3F\x75\x2B\xDA\xAA\x65\xAC\x54\xDB\x58\xCD\x9A\x96\x81\x0A\xD8\x40\x85\x6C\xA0\x22\x36\x2E\xBA\xA5\xEB\xFA\xAE\xB6\xB9\xE0\x95\xD6\x96\x7E\x7A\x4E\x37\xB5\xA5\xD9\x01\x72\xDA\x67\xDD\x0C\xDC\x9C\xD2\x03\xD9\xA5\x29\x09\xC8\x4A\x37\x5D\xE4\x87\x53\x9C\x52\xF1\x4C\x26\x44\xDD\xF4\x31\xB4\x42\x3B\xA2\x6B\x59\x96\x6C\x23\x10\x0E\x44\x01\xC7\xCF\x7F\x66\xD7\xBD\x25\xCF\x7F\x96\xBF\x59\x5A\x6A\x23\x85\x56\x2D\xD0\xD5\x21\x5A\xC5\x23\x2D\x95\xC8\xF9\x14\x24\x0B\xBA\xAC\xCC\xE8\x24\x4C\x55\x8D\xE9\x48\x0D\x67\x15\xAB\x9A\xC3\x86\x16\x81\xC0\xE8\xCA\xEC\x47\x53\x21\x6B\x33\x28\xD9\x7B\xBC\x66\xC4\x82\xBE\x34\x6C\x42\x0F\xA2\xDB\x1B\xC1\x59\xE4\xBF\xC1\x2B\x41\x20\xD9\x42\x56\x7C\x64\x9B\x3D\x66\x4C\x9F\x56\x43\xD6\xF4\x59\x94\x68\xA0\xF8\x34\x45\x5E\xE3\x24\x56\xCE\x97\x64\xD1\x65\xC3\x40\xD5\xC4\xF3\x0D\x0B\x91\x9B\x18\x55\x29\xE4\x9D\x9E\xC4\x3B\x17\xB5\x8B\xF6\x3A\x04\x36\x2D\x65\x2B\xC0\x27\x55\x64\xD3\xDF\x78\x2B\x62\x87\x40\x15\x48\x41\xD0\x3A\x02\x08\xF3\xA2\x94\x7E\x8B\x3E\x15\xCF\xA8\x92\x74\x8B\x0D\x6E\x4F\x86\x4D\x51\x6B\x78\xA1\x41\x34\x0B\x0D\x8A\x26\xA3\x51\x97\x74\xD4\xBD\x79\x60\xCB\x53\x39\x09\x05\xA2\x4B\x8E\x00\x44\x3D\x39\x45\x64\x44\xE5\x24\x74\x8D\x09\xB6\x2B\x45\x83\x89\x60\x4C\x40\x0E\x47\x01\x82\x15\x59\x90\x71\x1F\x0D\x11\xEA\x87\x21\x3F\x8C\xC9\x1B\xA2\x7E\xE7\xEC\x17\xC9\xAD\xCB\xC3\x42\xA2\xCB\xF0\xAB\x65\x53\x5E\xD7\x4C\x79\xF5\x01\x15\x69\x8F\x05\x08\xAF\x9A\x76\xB5\x17\x57\x44\x7B\x71\xC5\xBB\xFA\xC5\x08\x31\xB6\xB8\xE2\xB5\x57\x24\x70\x9C\x2B\x2E\x4A\x54\x6B\xA7\xD5\xAA\x04\x4B\x26\x2D\x4E\x98\xEF\x57\xD2\x54\x48\xB6\xA6\x88\x64\x70\xDD\x14\x51\xF1\x9A\x80\x67\x75\xBF\x0C\xBA\x25\x2D\xC8\x90\x03\x43\xF3\x17\x57\x2B\x12\x21\xDA\xD9\x04\xE4\x1C\xE9\x1D\x4E\x37\xA8\x5D\x08\x41\xB5\x28\xE1\xF3\xA2\x04\x1A\x65\xAF\x30\xB4\x28\x11\x82\x5F\x13\x80\x17\x70\xED\x25\x02\xB5\xD5\x51\x5A\x75\x14\xBB\x45\x09\xDF\x3E\x86\xE6\xCC\xD8\x41\xB7\x30\x10\xE3\x47\x0C\xA9\x5B\x94\xC0\x1E\x91\x94\xE4\x86\xBC\xC2\xA0\x17\xA5\x30\x37\x04\x43\xEB\x12\x09\x1B\x79\xDA\xD3\x32\x41\x5D\x46\x43\x22\xC1\xCC\x65\xED\xAD\xE2\x8E\xF1\xA0\x13\x85\x9A\xBD\x2D\x87\xCF\xDE\x50\xF8\x7C\x1B\xBF\x7F\x23\xB7\xCB\x12\xA0\x81\x2C\x10\xB2\xBB\xD0\xA0\x90\x92\xEC\xF3\x84\xB9\x5B\xCA\x41\xF3\xBE\xC1\xAB\xDE\x13\x20\xBB\x52\x20\xF5\x05\x9D\x7F\x72\x97\x5F\x8F\x7B\xED\x97\x82\xBF\x79\x93\x97\xBA\xAD\x53\xCD\x7B\x31\xD7\x81\x7D\x82\xFE\x7D\x92\x15\x67\xBE\xF0\xD1\x55\x83\xB2\xAA\x8B\x33\x11\x50\x07\x45\x35\x07\x51\x0B\x10\xD9\xD7\x63\x90\x1C\xD9\x1B\xFA\xF6\x86\x0B\x80\x73\x43\x09\xFE\x1C\x75\x21\x20\x44\xA0\x43\x08\xF0\x23\x70\xEE\x82\x9A\x57\xAB\x7B\xB4\xAE\x0F\x21\x22\x95\xDC\x86\x0F\xEC\xA9\x6E\x69\x6C\x7E\x22\x93\x56\xB6\xF7\x65\x81\x99\x99\x13\x41\x20\x03\xFE\x53\x81\x0E\x02\x4F\x18\x1B\x15\x18\xFF\xD8\xBC\x50\xD8\xA1\x47\x7E\xDB\x2A\x94\x77\x37\x76\x0C\xAA\xC7\xAB\xF5\xAD\xCE\x22\xFB\xF6\xAE\x7D\x45\x6E\xDB\xF8\x7E\xA7\x74\x91\x7D\xC3\xF1\x4C\x42\x6C\x55\x0F\x62\x8E\x6F\x22\x2B\xEE\x9D\x2F\x62\x88\x20\xB2\x37\x5F\x28\x55\x17\xD4\x0E\xE9\x9D\xB3\xAF\x06\x42\xC2\x85\x3A\xBA\xFD\x18\x05\x62\xF5\x32\x6C\xB2\x51\xD1\xA4\x6B\xDF\x8E\xEA\x9A\x1C\x14\xB2\x88\x20\x41\x1A\x84\x45\x32\x0C\x82\xA1\xEE\x5C\x4F\xFC\x3E\x0C\x12\x5E\x14\x6F\x4A\xA5\xC3\x29\x81\xD8\xEE\x46\x3D\xA4\x9D\x5A\x40\x18\x64\x39\xD4\x99\x6B\x41\xF8\x54\x88\xE0\xD4\xEF\xBE\x79\x04\x19\xD9\x98\x3A\x24\xB0\xB1\x2A\x22\x86\x1B\xD5\xCC\x8C\xC2\x6D\x86\xE0\x4E\x47\xE1\xA6\xC9\x7C\x52\xA4\x08\x6F\x58\xA4\xA6\x45\x60\xEA\xC3\x7E\x70\xD0\xB5\x37\x3F\x5A\x86\x36\x6E\xBF\x03\x0E\x67\x12\xA9\xB4\xE7\x79\xC2\xF3\x84\x34\x90\x22\x5A\x29\xA2\x45\xBC\xD2\x06\x52\xC4\x2A\xB4\xFF\x5B\x1C\xCF\x34\xA4\x0C\x73\x6C\x7F\x5E\xE2\x63\xD5\x6E\xE0\x61\x89\xAC\x1B\x7C\x51\xE0\x24\xAD\x6E\xB0\x1B\xF4\x8A\x98\x1E\x72\xE0\xD9\x6A\x28\xAB\x86\x43\x63\x9B\xBD\x5F\x0A\x37\x18\x30\x7E\x65\x62\xBF\xAB\x5B\x46\xF6\x53\x62\x62\x60\x07\xD1\xCC\x03\x51\xBC\xF7\x9F\x54\x72\xB4\x48\x34\xDF\x8C\xC0\xE7\x5C\x20\xD2\x38\xCE\xF0\xBB\x88\x90\x8F\xAF\xC8\xED\x32\xB2\x3F\x86\x9A\x63\xE1\x44\xA6\x6C\x54\x84\x36\x2F\x32\xB5\x00\xC9\x4C\xE1\xE9\xEA\x4F\xE8\xD6\x9F\x5F\x7D\xF1\x3C\x54\x93\x30\xF5\xE8\xA5\x8B\x85\x1E\xF1\xF4\x7F\x89\xE3\x99\x40\x5A\xE4\x4C\x0B\x14\x17\x7E\xF2\x4B\xEE\x09\x10\x41\x7D\xA7\xE8\xDA\x86\x5D\xFB\xD8\x76\x19\xDE\xAA\x05\x18\x50\x10\xE7\x9F\x22\x73\x13\x40\xC8\x76\x33\x25\x9D\x27\xDB\xE1\x2C\x47\x76\x63\x3A\x3F\xDE\x43\x54\xF7\x60\x01\x35\x90\xF0\xB3\x1F\x17\xC7\xE6\xF3\x5F\x73\x3D\x46\x53\x7B\x7C\x55\x7A\x23\x5A\xF7\xAD\xED\x22\xB6\x49\x91\x99\xAA\x93\xC8\x7E\x5A\x1C\x23\x15\xB6\xFF\x53\xE0\x24\x3E\x2B\x42\x88\xEC\xEF\x0B\x7A\x0D\x1F\xDB\x9D\x6D\x96\x81\x0C\xA3\xCF\xD8\x5E\xDE\x86\x98\x74\x6B\x7B\xBE\x79\xD1\x48\xC2\x8C\x2D\x04\xF5\x42\x6A\x98\xD8\x5D\x6F\xDB\x69\x17\xE9\xC6\xAE\xDA\xCE\x14\x24\xF6\xE7\xC5\x76\x86\x2A\x92\xF7\x6D\x7E\x01\xE8\x89\xDC\x9E\x2F\x23\x7B\xF3\xC5\x62\x06\x12\x48\xEC\xD6\x36\x44\xF3\xC5\x2C\x44\x47\x33\x93\xA6\x55\x57\x6A\xB8\xAB\xCC\x75\xD5\x34\xB1\xAF\xBC\xA2\xB7\x5B\x1D\xBA\x2E\x66\xD3\xDC\x54\x75\x2A\x4A\xEA\x5B\xB4\x28\xF3\x45\xF1\x6D\x58\x2B\x83\x7C\xF1\xC8\x7B\x5F\x2C\xE6\x18\x11\x32\x76\x11\x64\x88\x23\xD9\x93\x97\x8A\x7D\x90\x63\x9D\x67\xCB\x6B\x2E\xC1\x35\xCF\x14\x39\x64\x16\xB6\xED\xE7\xC4\x8E\xFD\x0B\xD9\x2D\x62\xF7\x5A\xD2\x83\xA8\x6B\xEF\x80\x18\x2B\xC3\xBE\xDD\xF2\x1A\xD8\x77\xF9\xDD\x2F\x96\x09\xAA\xC1\xEE\xEE\xEE\x27\xC5\xBD\xF3\x90\x74\xD1\x4F\xF1\x3A\xBE\xBD\xF9\x44\x26\xB1\xEA\x43\x45\x02\xD7\x14\x39\x24\xEE\x45\x17\xB1\xB0\x62\xBE\x07\x11\xCC\xA1\xBB\x05\x05\x59\xFE\xEB\x8E\xCF\x13\x18\x7C\x5D\x7A\x3D\x1A\xC2\x51\x2A\xB6\x90\x4A\x60\xB6\x42\xCA\xD9\xEB\xA4\x0B\xC9\x4E\x11\xC1\x6C\x1B\x9F\x39\xAC\x4A\xF8\x24\x88\xCF\x1C\x56\x86\x70\xB3\xCC\x20\xDC\x58\x2F\xE3\x36\x3A\x31\xA1\x43\x76\x2F\x61\x74\x22\x08\x4F\x16\x31\x64\x45\x04\x71\xFD\xDE\x6E\x08\x9D\x04\x66\x1C\x3A\xB3\x7B\xA1\xB3\x3F\xBD\xEE\x0A\x6C\xDB\x83\x5D\x7F\xB7\xFC\xB1\xE2\x6A\xF8\x73\x6D\xBA\xDF\xD8\xD7\x8F\x78\x7F\x72\x62\x23\x65\xC6\xC5\x02\x69\x6E\xAC\x44\x25\x43\x8F\xBB\x5D\xC6\x99\xB0\xAA\x48\xAA\x97\x5C\x18\xE4\x14\x11\xBF\xCC\x4C\x70\x6E\xA8\x17\x2C\x50\x4F\x61\xB7\x1D\x5E\x44\xE4\x10\xCB\xF0\x38\xBD\x31\x80\xA0\x8B\xED\x02\x03\xFA\x36\x0C\xD8\x87\x43\x11\x17\x83\x38\x96\xC5\x47\xB3\x59\x48\x40\x9D\xCC\x66\x41\x41\xB2\x8E\x41\x0A\xA8\x2E\xD2\xD2\x8D\x34\xDE\x94\xA2\x94\x10\x87\x8A\xB0\xC5\x44\x42\xE4\xE9\x1C\x5A\x92\xA2\x57\x86\x10\x12\x7B\x29\x20\x75\xF6\x04\x42\xC7\xD1\xC4\x71\x74\x88\x7B\xF6\x2E\x08\xED\xEC\x76\xA9\x6C\xB2\x34\x0F\x6A\x8C\x5B\xA1\x9D\x3D\x51\x28\x08\xAD\x77\x11\x07\x9C\x2C\x81\x91\x15\x36\x07\x63\x23\x48\xF6\x62\x18\x19\x0E\x82\x6D\x96\x3C\xC6\x2C\x16\x27\x38\x03\x91\x18\x2D\x83\x99\x47\x92\xCC\x12\x74\x08\x43\x91\x41\x78\x94\x60\xA0\xA0\x2A\xC2\xA0\x60\x8E\x34\x61\x0E\x0D\x35\xC1\x92\xC1\xEC\x7A\x89\xC2\x12\x82\x59\x9A\x87\x90\x4D\x69\x99\xD9\xE4\xE2\x3C\xAA\x0C\x33\x3A\x81\x08\xC9\x1E\x83\x82\x68\x1D\x23\xFE\xAB\x61\x70\x17\x50\x2E\x0C\x4B\x44\x04\x49\x11\x38\x88\x69\x70\x32\xA4\x2E\xE2\x98\x68\x6B\x99\xD2\xFF\xA8\x4C\x44\x92\x1A\x73\x65\x53\xF7\x77\x0A\x78\x6D\x0A\xF6\x04\x3C\x4E\x13\xD3\x72\x5B\x18\xA4\x4C\xE6\xC1\x3F\x40\xD8\xE9\xE5\x10\x89\x7B\x84\x9E\xF9\x15\xB9\x7D\x34\x93\x18\x06\x65\x38\x1D\x9B\x25\xD9\xC7\x28\xF9\x28\x39\xFB\x89\x33\x1E\x67\x2B\xDC\x8C\x13\x27\x08\x45\x52\x69\x4D\xE2\x5E\xE7\xD5\x1D\xED\xDD\xC7\x58\xE3\x06\xB3\xF8\x28\xED\x6C\x50\x27\xD1\x58\x42\x54\x9B\x2F\x75\x55\xE6\x8B\x5A\x4C\x24\x80\xAC\x62\xE4\x91\x72\x3A\xB7\xC0\xFD\xDA\x41\x17\x3F\x96\xE6\xDB\x43\x78\xCD\x56\xB8\xC0\x5C\x47\x7B\x77\x40\x23\x70\xB4\xAE\x2F\x0E\x0B\x9C\x7E\x09\xF3\x73\x51\x28\x69\x93\x9C\xBC\x84\x9F\xFE\xA0\x99\x7B\x46\x43\x9B\xE5\xBC\x7B\x5E\xA1\xBF\x3F\x7A\x65\x35\xC3\x09\xAD\xF7\x30\x7E\x54\x85\x83\xD3\x56\xEE\xD0\x9E\x95\xE7\xF0\x59\xBD\x79\x6D\x75\xBE\x8C\x7F\xAE\x4C\x16\xB9\xDA\x2B\xE1\x53\x8B\xBB\xFC\xF7\xE5\x23\x4F\x7F\xCC\xD5\xFA\xF2\xC7\x9E\xBF\x67\xE5\x6D\x5B\xAB\x2F\x3E\xFB\xDF\x8F\xBC\x00\xC9\xE2\x9D\xEF\x7D\xD1\xBE\x32\xBB\x6D\x9F\x78\xAC\xFB\xF2\x3D\x7F\xF9\xF8\x8F\xBE\xFD\x9A\x7B\x3E\x75\xE4\x85\x7B\x7E\xF2\xBF\x9E\xFA\xC0\x81\xF7\x7F\xFC\xC8\xF3\xCF\x97\xE6\x87\x5C\xD3\xCF\xDA\xD5\x4C\xA2\x18\xEE\x3E\x56\xA4\xB4\xD1\x8C\x67\xD8\xE0\x81\x8F\x96\xCB\x05\xAA\x3E\x44\xAD\x3D\x97\xB3\x06\xCC\x47\x8A\xD4\x40\xFA\x72\x69\xEE\xB9\xEE\x9D\xFF\xE9\x67\x7F\x7C\x5D\x2D\xBC\xE0\x3C\xBC\xB9\x67\xFB\xC3\x8F\xC7\x27\x7F\x30\x59\x78\xA1\x1A\xE4\xC8\xC8\x88\xC2\x8D\x38\x83\xEE\xD4\x7C\xA4\x98\xC1\xD8\x1F\xED\xEA\xAC\x4D\x2E\x16\x39\x7A\x50\xC8\x60\xE6\xE5\x7B\xFE\xE8\xEE\xFF\xFB\xD7\x8B\xB7\x7F\xE1\x63\x2F\x3C\x5F\x1A\x30\xCF\x57\x54\x59\x00\xF3\x21\x30\x60\x5E\xC0\x52\x30\xAE\xF8\xB6\x85\x0F\xBB\x6F\xAF\x5B\x78\xBE\xFA\x7A\xC3\xC2\xF3\x1F\xAE\x7F\xFC\xF1\x91\xE7\x61\xC6\xBE\x12\x76\x5F\x5A\xBC\xF3\x3D\x1F\x7B\xA1\x34\xAB\x56\xEE\x64\xA2\xEE\xE1\x75\x0B\x1F\x2E\x50\x56\x06\x5D\xC4\xCE\xB7\x01\xE4\xF3\xC5\x8C\xF5\xAC\x0F\xA9\xFD\x3D\xD1\xB5\xBF\x24\x97\xE6\x31\xEE\x44\x70\xF3\x6D\xDE\xAA\x11\xDF\x73\xF0\x6D\x37\xFD\xC1\xEF\x1E\xBA\xF1\xC8\x19\xEE\x2D\xA6\xDE\x76\x9F\xBE\xDD\xBA\xEE\x67\x20\xBD\xAF\xC0\x2E\x06\x3B\x45\x6E\x45\x91\xC2\x0C\xE4\x5D\x2B\xC0\x9F\x2F\x66\x08\xF3\x1E\xCD\x8B\xE7\x68\x0F\x28\xCC\xD0\x96\x2C\x98\xB5\xE2\xF8\x3A\x91\xC8\x8A\x62\x86\x5E\x7E\xD6\xDC\x89\xC1\x7C\x18\x62\x48\xE7\x4B\xF3\x1C\x98\x21\x71\x81\x19\x8B\x32\x3D\x07\x73\x10\x2C\xCD\xC3\xDC\x7A\x99\xC3\xDC\x3C\xE4\x38\xED\xD9\x2E\xF7\x59\x79\x71\x1E\x27\x22\x5B\xDB\x8E\xB5\xB3\x6A\x01\xE6\x68\x7B\x1D\xDA\xAE\xCF\x8B\x1D\x90\x30\x0B\x1A\x44\xA5\x07\x29\xA4\xE8\x57\xC2\x2E\x84\x3B\x45\x48\x6B\x1B\x11\xF8\x5D\xC0\x60\x20\xD9\x2C\x67\x6C\xB2\xB1\x6E\xEF\xE8\xA1\x01\xE3\x25\xCB\xCA\xB3\xC7\x27\x8A\x1C\x66\x8A\x10\x72\x52\x73\xDF\xC2\x12\x2D\xB6\xF8\x30\x67\x2F\x77\xCB\x19\xD4\x25\xAA\x2C\xA8\x43\x7B\x87\x15\x56\xC1\xDC\x3A\xF8\xDD\x32\x04\xDF\x9E\xEA\x96\x39\x84\x90\x9F\x98\x5F\xCF\x7F\x05\x61\xB9\x51\x2F\x54\x43\x60\x57\xE0\xC3\xCC\xC9\x4C\x52\xC7\xA4\xCD\x50\xCC\x20\x6B\x8E\x56\x1D\xDE\x6A\x0F\x22\xD1\xEF\x9B\xAF\x60\x43\x41\x43\x43\x55\xE6\x85\x8F\x90\x11\x2A\x21\x46\x58\xB7\xD1\xCA\xAE\xDF\xC6\x9B\x1A\xF8\x2D\x13\x83\x4C\x2B\x66\x61\x1F\xD9\x4D\x50\xEB\x10\x9C\xA4\xC3\x89\x6A\xBD\xF0\xC9\x46\x8D\x10\xCE\xE7\xB1\x66\xBB\xBC\x39\x90\xAD\xD9\xAC\x69\xAB\xD2\xAC\x79\x5F\xAE\x83\x81\xBE\xA4\xA6\x99\x0D\x34\x1C\x23\xA6\xE2\x15\x12\x2B\x49\x0C\x2D\x62\x50\x85\xAF\x16\x20\xB0\x3E\x45\x99\x04\x31\x06\x2F\xF4\x35\xCE\xD0\x54\x5E\xEA\x62\x18\x5A\x03\x39\x09\xCB\x84\x56\xAB\x2A\x28\xB1\x85\x6F\xEC\xC1\x91\xCA\xF6\xF7\x89\x48\x56\x76\xDB\x85\x5F\xE2\x42\x35\x54\xF8\x05\x2E\xD4\xDD\xE1\xB1\xE8\x45\x4C\x4C\xEF\xF5\x1D\x01\x35\x13\x30\x99\x04\x5B\xE2\xB8\xCD\x04\xA4\x75\x99\x11\x9C\xBD\xA9\x38\xBF\xFB\x2B\xC6\xF9\xDD\x84\xF3\x97\x46\xE5\xC0\x7E\xEE\x6A\x91\x26\x7C\x19\x3F\x55\xE3\xA7\xAE\x1E\x3F\x45\x7B\xE2\x86\x3D\x42\x26\xAC\x67\x35\xA2\x7D\xD1\xCA\xED\x32\x76\xCB\x5D\xDB\x45\x0A\x8A\x82\xDC\x1A\xF1\x0C\x52\xF7\x69\x48\x30\x21\x00\xB3\xFE\x15\xD2\x00\x5B\x64\xC6\xEE\xBE\x1F\x25\xF1\x19\xFC\x17\xE2\x79\x8A\x42\x7C\x30\x14\xCB\xC4\x18\xCB\xEC\xA5\x2E\x4D\x87\x19\x18\x2C\x48\xBF\x1A\x6A\x38\xC7\x89\x96\xAD\xF3\x89\x4F\xD8\xD3\xA4\x06\xEE\xD7\x2F\xAC\xB2\xC7\xAC\x6D\x1F\x78\x1F\x1A\x71\x9D\xC5\x0C\x6F\xAF\x26\x53\x4E\x1E\x66\xBE\x34\x36\xA1\x2D\x94\x64\x6E\x62\x30\xEB\xA5\x6F\x4F\xA1\xB5\x69\xCC\x8C\x95\xEE\xDD\xAA\xCF\x4B\x93\x18\xF8\x70\x3B\x36\x36\x45\x0C\x09\x0B\x1A\xC6\xB3\xE8\x2D\x12\x03\x33\x93\x5C\xAA\x3F\xC9\xA5\x7E\x6A\xB2\x4B\x9D\xF9\x48\xE1\xF3\x11\x80\x19\xF0\x5F\xFE\x10\xAD\xD7\xDB\xFF\xF0\xBE\xDD\x5D\xD1\xBD\x55\x8B\x72\xF6\x85\x32\xAF\xDC\x98\x5D\x85\xBC\x46\xF3\xCC\x36\x45\xD0\xB4\xE8\x93\xFF\xDB\xB1\xE5\x3E\xC8\x21\xFE\xE9\x0F\xD5\x6D\x3F\x7B\x84\xC5\x0A\x66\xDD\xAE\xDD\x9F\xEE\xB0\x35\x61\xFE\x61\x47\x60\xF8\xFD\x7B\xD3\x64\x95\x96\xC9\xE4\x36\xC4\x47\x7B\x10\x77\x8B\xD8\x00\x6F\xA0\x8E\x20\xE1\x38\x37\xC3\xC8\x0F\xB2\x6E\x99\xA2\x41\x57\x30\x07\x59\x97\x16\x9E\x92\xCD\x32\x6D\x79\x88\x8C\xE7\x5D\x31\x3A\x87\x7D\x90\x16\x31\xEC\x43\x59\xA1\xA6\x31\x92\x76\x1F\x4D\x9F\xC9\x29\x18\xC8\x8E\x65\x82\x9E\x15\x29\xF5\x58\xEC\xC3\xC2\x75\xA4\xF8\x1C\x73\x56\x2F\xC0\x3E\x0A\xAF\xAB\xF8\xB9\x88\x21\xC5\x02\x89\x1C\x23\x47\x34\x07\xC9\x71\x1C\xB2\x48\x20\x23\x8E\x19\x1A\x07\x52\x87\xC9\x3E\x14\xD0\x08\xE2\xAE\xBD\x1D\x3B\x89\x89\xE7\xBC\x67\xC8\x67\x56\x34\x74\xB0\x72\x67\xBB\xCB\x5C\x4D\x8B\x04\x29\x1E\x17\x89\x21\x30\xEA\xB8\x3D\xE9\x62\x4B\x9B\x9C\x2B\x63\x46\xBC\x8E\xED\x0B\x14\xF1\x98\xF0\x2F\x7C\xC8\xEC\xBE\x13\x6E\x1F\x6C\x68\xF5\xB6\x95\xC7\xDD\x06\x3A\xDA\x40\x9F\x40\xD0\x16\x65\x08\x6D\xBA\x6D\xC5\xF1\xDE\x7A\x19\x2C\x55\x2F\x1A\x02\x9C\x03\x9C\xE4\xC5\x3B\xE7\x46\x13\x72\xA3\xD8\x7A\xBD\x44\x65\x3C\xD5\x2D\x63\xF0\x21\x1E\xF2\xA1\x3C\xCF\x40\x7D\x94\x90\x38\xE7\x49\x86\x84\x97\xDF\x09\x08\xEE\xB1\xD6\x3E\x02\x82\x05\x87\x06\x39\xD8\xC2\x89\xEC\x51\xC8\xB6\xF7\x28\x89\x0E\xA1\xE4\x6D\x53\x50\x11\xE1\x7C\x83\x1B\xC1\x9E\x8D\xB8\xCE\xAD\x4D\x1D\x33\x54\x89\xF6\x90\xA2\xA5\x77\x66\x3E\x71\x56\x8E\xBD\xC1\x5E\x26\x8E\xDE\x91\x95\xF1\xF1\xDA\xCA\x25\xEB\x58\x4C\x02\xEC\x93\x2E\x67\x14\x08\x64\x8D\x65\x8B\xDB\xFD\xC4\x2C\x56\x19\xF6\x92\xF1\xCE\xDF\x04\x7D\x05\xAF\x1E\x54\xA6\x4D\x5E\xB5\x69\x33\x2E\x14\xF0\x87\x42\x01\xDF\xFC\x85\xCF\xDB\x62\x13\x0C\xD4\x4C\x46\x64\xD8\x2E\x52\x54\x08\x94\x88\x14\xFF\x89\x21\xB1\xE9\x36\x04\x3D\x0A\x0E\x69\x39\xDB\x87\xA8\x15\x49\x79\x2E\x92\xC2\xE2\xF5\x32\xE3\x30\x6A\x06\xA3\xE9\x09\x61\x14\xB6\x6C\xE2\x27\x1E\xC1\x87\xB8\xE9\x8E\x66\xAF\xBE\x53\x2F\xD7\x26\x6E\xB7\x69\xC9\x8B\x8B\x87\x5B\x96\x87\x57\x84\x04\xEF\xAF\xF4\x21\x3A\x46\xE4\x8A\x8F\xF1\x5B\x9F\xFC\x38\x4F\xCD\x7C\x7B\xB9\x0B\x3E\x12\x37\x82\x68\x9E\x96\x47\xF2\x63\xF3\xE4\xD3\x73\x57\x95\x8E\x42\xB9\xB8\x92\xE0\xF9\x5D\xD1\x0E\x37\xEF\xCF\x14\xAF\xF9\x27\x16\xB6\x71\x66\x37\x44\x8C\x2F\x4F\xA8\x0B\xB4\x53\xB8\x5D\xEB\xD3\x75\x2D\xE4\x0B\x3D\xA1\x4D\x8F\x6D\xE4\xEB\x27\xA2\xD2\x52\x7A\x7E\xB0\x79\x0E\x0A\x67\xB5\x16\xE7\xD8\x4E\x0F\x12\xD6\x83\xC4\xE9\x01\x35\x80\xA9\x0D\xF8\xF9\xAD\xED\x01\xAB\x0A\x2C\xFF\x21\x18\x92\xFF\xB0\xE2\x53\x0C\x3E\x7B\xF9\xC8\x09\x5E\xD4\x16\xBC\xC8\xAD\x2D\xD5\x4E\x39\x46\xF9\xF7\xD7\x8B\xD0\x54\xFB\x5E\xC3\x22\x02\xBF\x59\xA7\xA8\x34\x60\xEF\x6E\x42\xF0\xBB\xDC\x07\x9F\xCC\x41\x1D\x88\xEB\x68\xB8\x02\x25\x1C\xEA\xA3\x5E\xF6\x8A\x97\x2A\x48\x8A\xD0\x40\xE8\x76\x1E\x34\xEF\x75\xCD\xCD\xB4\xF1\x91\x36\xDD\xBB\x03\x91\xA5\x3C\x28\x04\xB6\x92\x20\xD6\xCD\x37\xD3\x79\xAE\xFA\x98\x87\x5E\x18\xBD\x0A\x83\xBC\xED\x89\x4C\xD7\x87\x29\xF9\xA4\x67\xF3\x9E\xD9\x8A\x59\x63\x07\xB3\xC6\x4A\xF3\x21\x29\x43\x37\x5A\x75\xD8\x84\xDE\xE7\xD2\x68\xB4\xE3\x5F\x2F\xB8\x7D\x09\x5C\x5A\x6D\x17\x03\xB9\x5E\x7A\xA0\x40\xAC\xBB\xA3\x4F\x65\x30\x5F\x86\xCD\xB1\x15\x01\xAA\xEB\xC0\x72\xDF\x22\x82\xCC\xAD\x8F\xB8\xB5\xBD\xFB\x32\x5D\x6D\x39\x0A\x79\x16\x33\xA0\xD3\x0A\xDE\xD2\x7C\x0D\xE2\x8C\x00\x51\xBD\xA1\x4C\x96\xCC\x2D\xEE\x38\xAB\xBD\xD8\x2D\xA5\xBD\xEE\x04\x2D\x70\xCA\xBE\x7D\xDF\xEE\xAE\xDA\xA6\xE5\x4E\xDA\x46\x1A\x1F\xC3\xD0\x43\xD8\xDF\x91\xDB\xF6\x63\x03\x6E\x1D\x2C\xB5\x7B\xEA\x99\x33\xD2\x9F\x42\x69\xB5\x80\xB4\xB6\x83\x9D\x6E\x75\x64\xB6\xC2\x89\xE8\x50\x78\x40\xB7\x79\x74\x4B\xFF\xA0\x60\xD5\x01\x8F\xF6\x28\x32\x2E\x92\xF6\x86\xB9\x3D\x1C\xE6\xD7\xA4\xD2\x97\x79\xDF\xCF\xE2\x91\x67\x4B\x7D\x09\xF4\x33\x85\x06\xF5\xEF\x0A\x0F\xA4\x23\xBB\x38\xA0\x3E\xF8\x01\x7A\x91\x8F\xCC\x1B\x20\x4D\x2F\xBF\xFB\xC5\xAE\xEB\xDD\x9D\x01\xF2\x0B\x0D\xFE\xA2\x78\x27\x03\x28\x16\x07\xEF\xA4\x95\xF5\x83\x4D\x15\xBA\xC6\x83\xF6\xE9\x4C\xE2\x28\x2D\xE2\x48\xC2\x4A\x31\x56\x01\x6D\xD4\x0C\xBA\x25\xAD\x4C\x45\x10\x12\xB7\xCA\xD8\x8D\x1B\x43\x74\xD8\x2D\x37\x95\x81\x13\x20\xB7\x1B\xC7\x3C\x21\x95\xBC\xCC\xB7\xD1\xB8\x13\x27\x8B\xEE\x78\xF9\x41\x21\xEC\xC1\x63\xA5\x07\xF2\x5B\xB8\x10\x3C\xDA\xEC\x23\xC0\x27\x41\x08\xEA\xF7\xCD\x28\xA4\x7A\x31\xB9\x0C\xC1\x4B\x8B\x6F\x7D\xCF\xE2\x5B\x9F\xBD\xB4\xF8\xF6\x4B\x88\x24\xDB\x5A\xEC\xD4\x18\x54\x8D\xC5\xC1\xA2\x04\x6F\xFE\xB2\xBB\x40\x82\x24\xE3\xB2\xF9\x09\x2F\x94\x03\x45\x0B\x4F\xF8\x7F\xA5\x16\x2D\x90\xAA\x90\xF1\x63\xF5\x12\x48\x0D\x9E\x9E\x2F\x1A\xE7\x4E\x13\x2A\x4D\x00\xB7\xCA\x08\x66\x8D\x26\xC1\xBE\xBD\x5B\x06\x15\xCC\x12\xFC\x7A\xC5\xE4\x05\x08\x5E\x7E\xBE\xF0\xAB\xAD\x3B\x0E\x62\x0F\xFC\x17\x0A\xD4\x7F\xC9\x56\x1E\xE1\xE5\xDD\xE1\x8E\x00\xB7\x1F\x6F\x0F\x14\xAB\x85\xBA\x03\x84\xE0\xB9\x1F\xC2\xBF\x8F\x1E\x29\xFC\x56\x9D\xA9\xC0\x60\x84\x13\xBC\xFC\xC2\xF3\x45\xDC\x82\xEB\xC3\x63\x30\x85\xE0\x41\xFC\xC2\xF3\x0C\x16\xFA\x22\x07\x4D\x11\x18\x7B\xB9\x88\x20\xA8\xDE\x18\x7F\x5E\x1C\xCF\x14\xBF\xCA\x51\x6E\x4F\x3B\xC5\x93\x0E\xF4\x5B\x8F\xB1\x03\xB0\x07\x8F\xF3\x31\xEA\x6E\x61\xEC\xA0\x48\x38\x24\x33\x38\x5C\x1D\xA0\xE1\xD3\xD2\x20\x52\xB4\x8C\x43\xD7\x67\x80\x41\x1C\x4A\xDD\x12\x02\x09\x81\x4D\x36\x40\xD3\xFE\x84\x00\x0C\x0D\x57\xC1\x6D\x90\x88\x08\x8F\xA6\x51\xB1\x14\x7F\x90\x59\x4A\x36\x0A\x0D\x01\x8A\x65\xC0\x3B\x80\x50\x58\x10\xC1\x64\xE3\xE5\x17\xEE\xD6\xF4\xBE\x4A\x41\x78\xB7\xA6\xED\x18\x91\x81\x88\x77\xEF\xA4\xED\xFD\x60\x11\x6F\x79\x6B\xEF\x07\x6B\xEF\x0C\xAB\x0E\xC4\xDF\x2F\xEC\xC0\x8A\xFC\x09\x9E\x23\x0D\xAC\xB0\x22\x7F\xB2\xF5\xC3\x8A\xFC\xA9\xA1\x9F\x56\xE4\x4F\x8F\x14\x58\x91\x3F\x43\x45\x86\x0F\xA9\xE4\x74\x98\x68\xDD\x5C\xE3\x6E\x7B\xC0\xA2\x75\x2A\xEB\xBA\x32\x57\x83\x1E\x9F\x34\xF7\xBA\xAB\x56\x9A\xDA\xA5\xE7\xEA\x83\xBA\x97\x16\x67\x3D\x50\x7C\x98\xF7\x28\xD1\x9C\x6C\x61\xDE\x2D\xEB\x43\x5F\xB4\xE9\xDF\x3C\x54\x1F\x5D\xC9\x4B\xD9\x1E\x82\x6F\x7B\xEA\xD6\x7B\xBA\xD1\x3E\xDC\x26\x44\xFE\x9B\xAE\x4B\x50\xB4\x21\x8B\x4C\x25\x6D\x7C\x72\x8D\xEB\x7D\x7A\xE6\x19\xBA\x30\x40\xF2\x85\x01\x35\x98\xCA\x81\x69\xBD\x7B\xE9\x8C\xB1\x04\x85\xDD\x7E\x66\xD7\x1D\x94\x20\x20\xB1\xE7\xDB\x84\x6C\x4A\xB9\x40\x8D\x54\x13\x54\xAA\x5B\xA5\xD4\xB7\xF5\x68\x1F\x1C\x1F\x52\x32\x7F\xEE\x91\x63\x73\x28\x1E\x52\x91\x72\xF7\xE8\x88\x52\xB1\xE8\x62\x5F\xCE\xE8\x73\x3B\x05\xC3\xA4\x90\x54\xBD\x50\x3C\x2D\x55\x24\xAD\x77\x8C\xB7\x74\x94\xAA\x4E\xB0\x2B\xF6\x6C\x15\xC5\x44\xB3\x39\xDE\x61\x38\x42\x70\x47\x69\x6F\x84\xD2\x6A\x1A\xA5\x3D\xA6\x74\xAA\x4D\xC3\x5E\x3E\x9A\xC5\x4E\x8B\xF6\xEB\xAD\xBB\xFD\xDC\x27\x49\x57\x1D\xD6\xB7\x1F\xAF\x09\xDD\x40\x5F\x99\x4B\xF0\xD7\x4B\x0D\x4E\x00\x3C\x3E\x7D\x79\x15\xF0\x94\xD4\x8C\xB6\x0B\x7B\xEE\xE4\x2B\x78\x4B\xEE\xEE\x04\x86\xAC\x6C\x80\xAA\xCE\x7C\x22\xDA\xA6\xA2\x2E\xA8\xB6\xE5\x69\x53\x57\x57\x7C\xD1\x08\x5D\x80\xD0\xF1\xD9\x03\xF4\xC1\x65\x68\x0F\x1E\xA3\x39\xD6\xAD\xB4\x83\xC3\x35\x53\xDC\xAC\xD5\xC4\xF1\x2F\x80\x46\xBA\xDD\x96\x5A\xF4\x8F\x7A\x08\x47\x49\xE7\x71\xAA\xB1\x41\x3B\x9A\x37\x07\xE1\x34\x5D\xB2\xD6\x22\x3A\x9B\x89\xD4\xFA\x7C\x30\xFE\xB3\x9A\x4F\x70\xC9\x4A\xBB\xD0\xF5\xDA\x92\xF0\x72\xBE\x98\x2C\x0E\x72\x41\xF3\xB6\x78\xC5\xE4\x61\x45\xF6\x4A\xFF\x24\x7B\x66\x04\x9A\xF6\xE3\xA1\xA2\x94\xA1\xFD\x8C\x60\x6C\xCB\x63\x6E\x63\x77\x68\xE1\xDE\xCC\x83\xD0\xBE\x22\xE8\xEE\x89\xBA\x7B\xDA\x27\xBA\x0E\xFE\x52\x75\x60\x80\x1E\x84\x4E\x39\xD0\x11\xAC\x83\x7F\x32\xD3\x10\xA2\x33\x79\x57\xB7\x0C\x69\xB7\x93\x76\x1B\xB9\x1E\x10\x7E\xEB\x6F\xE8\xC7\x95\xFE\x44\xFD\xAF\x68\x95\x08\x5F\x48\xC1\x70\x28\x06\xD0\xCA\x22\x18\x07\xD2\x3D\xC4\x58\xAA\x0B\x6A\x1D\x4D\x85\x1E\x32\x13\x9A\xA4\xBA\x65\x21\xAA\x02\x35\x5A\xA0\x5B\x05\x54\x44\x1B\x69\x9B\x91\x6B\x8A\x3B\x81\xF4\x97\x9A\x8B\xBC\xF8\xEC\x08\x89\x53\xB5\xB7\x36\xB4\x01\x04\xF3\x6C\x58\x5E\x46\xC3\xD2\xD8\xB7\xBC\x74\xBD\x69\xC7\xBF\x46\xCA\x7C\xA7\x49\x93\x37\x9C\x35\xBC\x95\xCC\x83\xDF\x12\x38\x1D\x0B\x67\xBE\x5B\x57\x4E\x5F\x78\xA2\xFD\xE7\x8B\x2B\xFC\x05\x22\xAC\xBE\x46\xEE\x73\xE8\x06\x6F\xC3\xF7\x8D\xB0\xB8\xBB\xDD\xD9\xA4\x43\x0A\x5D\x47\xFE\x5B\x4E\x0B\x10\x39\xF0\x69\x97\x71\x4B\xA2\x2A\x24\x2B\x5D\x97\x69\x86\xD1\x6D\xFE\xC5\x6A\xBF\x3F\xED\xFE\xFF\xC3\xEA\x1C\x40\x9B\x7C\xE9\x68\x41\x36\x5A\x90\x8C\x16\x18\x3A\x41\x20\x51\x2E\xD6\xC1\xEB\x5A\xFF\xDE\xCC\xB3\xFF\xF2\xFD\x7C\xD5\x91\xCF\x1E\x2E\xF3\x58\x8F\x9C\xD2\xFA\xD4\xD0\x1B\x69\x88\x42\x64\x9F\x73\x0D\x3D\xD7\x50\x0F\x35\x64\xBF\xA1\xC6\x1B\x2A\xFB\xA3\x23\x0D\xD5\x84\x86\x7A\x14\xF6\x68\xB4\x20\x9E\x45\xC1\x79\x8D\xA8\xF7\x10\xBB\x8D\xC5\xF8\x91\x83\xC2\xE8\x23\x32\x3F\xA5\xDD\xB1\x60\x77\xE4\x13\x54\xFE\x32\x13\xB3\x9A\xEC\x71\xEC\x23\x68\xA7\xAA\xE0\x6D\xC4\x20\x30\x2A\x43\xDB\x50\xB2\x93\x41\xC8\x25\xBA\x7B\x0C\x07\x28\x74\x5E\x77\x1B\xD3\x7D\xEE\x40\xD2\x26\x7C\x9F\x0F\x38\xE1\x9C\xC1\x9B\x74\x72\x85\xFB\x2D\x03\xFB\x19\xC1\xC0\x78\x14\xB4\x05\x28\xA9\x1E\x59\x8B\x48\x2D\x80\x37\xF3\x80\x98\xBC\xCB\xF1\x2A\xFE\xB8\xB2\x8A\xE3\x58\xBB\x12\x2F\x8E\x63\x3F\x0E\x22\xB2\x15\xC1\x61\x41\x67\x9C\xF9\x5B\xB5\x57\xDA\x63\xCC\xA2\x94\x77\x36\x61\x1D\x0F\xBF\xA5\xF8\x4D\xE3\xB7\x04\xBF\xD1\x8D\x3C\xD9\x61\xDE\xB0\xA4\x6D\x7C\x58\x60\x9F\xDC\x85\x40\x3B\x43\xBD\xA8\xD4\x37\xF6\x23\xEF\x6F\x1D\x17\xF9\xE5\xE6\x07\x30\x09\xE9\xB5\x8B\xAA\x48\x68\xE8\xEC\x10\xAD\x6E\xF9\xEB\xF9\x2F\x50\xA3\x94\x22\xC5\xFC\x3F\xD2\x8F\x47\xF8\xC4\x92\xCC\x7F\xB1\x52\x31\x74\xA3\xD5\xE9\xA5\xFC\x4B\xBB\x7C\x6F\x43\xCA\x77\x6C\x19\xBE\x64\x88\xC5\x8E\xB7\x25\x08\xF0\xF2\x3F\xE5\x9D\xE5\x14\x00\x81\xCC\xFF\x1B\xBD\x19\x93\xE4\x78\x68\x37\x32\xFA\x99\x3F\x94\x52\x57\xB7\x0D\x4A\x7B\xB9\x5B\x6A\x1B\x8D\x6D\x58\x05\x3D\x13\x4B\xA5\x3D\x21\xFC\x30\x90\x06\x64\x75\x3D\x8B\x38\xA0\x52\x94\x8C\x3F\x66\x01\xCB\xFF\xB3\xD3\x61\x99\xFF\x78\xA5\xCD\xD5\x37\x6E\x22\xB9\x89\x5C\x67\x3C\x49\xD5\xFF\x8C\x51\xE4\x37\xB4\x35\xC6\x84\xAF\x37\x84\xAF\x66\x7C\xF5\x10\xBE\x5E\x85\xAF\xBE\x02\xBE\x74\xF5\x63\x75\x14\x4C\xE6\xFF\x65\x88\xB0\x53\x06\x8A\xF7\x1A\x88\xCE\xEC\xBA\x26\x36\x63\xCF\xEC\x7E\x0E\x83\xA2\x41\xE6\xFF\x7E\x02\x30\xDF\xDE\xEC\xD1\x6F\xAE\x9E\xAA\x0F\x66\xBA\xB3\x3E\xF8\x71\x03\xE9\xB6\xCE\xE9\x7C\x07\x5F\xED\xD4\x1A\x86\x96\x64\x5A\xF7\x0D\xE5\xEE\x08\x4E\x35\x1F\x31\x59\x33\x2B\xA8\xE7\x08\x11\x15\x55\x73\x84\x0D\xBE\xDA\xA7\x3A\xD9\xA4\xDD\x25\x2B\x3C\xB7\xA7\x03\xFE\xE8\x4A\x29\xF2\xF7\xEA\xB5\x04\x77\xF0\xE1\x38\xAF\x4E\xF0\xB9\x49\xBA\x91\xC1\x43\x22\xF1\xD9\x42\xBE\x7A\x0F\xBC\xF5\xEE\x21\xE5\xF1\x8C\xDB\x7C\x96\x2F\x6A\xA8\x0F\x9B\x57\xB1\x11\x1D\xA2\xA0\x83\x8F\xF7\x93\x54\xCB\x7E\x19\x34\x00\x49\xB7\xD2\x36\x1D\x20\x7F\x18\xA0\x61\x48\x90\x70\x41\x15\x64\x97\xA4\x78\xD4\x78\x9D\x23\x34\xD3\x42\xD7\x9D\x74\xAB\x06\x91\xAD\x11\xE8\xFE\xBC\x51\xCC\xF9\x18\xA2\xE2\x49\x03\x61\xF8\x93\xC3\x18\xB2\xCF\x5B\xE7\x59\x34\x61\xE8\xDD\x4F\xE2\x25\xFB\xA5\x3F\x8C\xA1\xD7\x2C\x12\x69\xDA\x1F\xC3\x93\x2D\x37\xBE\xE7\x42\xB3\x16\x86\x6A\x08\x43\xBF\xC6\x10\xE3\x38\x3A\x0F\xA5\xD6\xD9\xF3\xD2\xB5\xAD\x7C\x3C\x0B\x43\x98\x2A\x06\x6F\x80\xFE\xF5\x61\xA0\xDD\x34\x81\x2B\xB3\xBA\xDF\x4F\xAB\x44\xB2\x5F\x7A\xD5\xC1\x16\x06\x5A\x37\xD7\xAE\xD1\xA1\x32\xBE\xD8\xA2\x02\x5A\xBB\x39\x42\x0B\x68\xE9\x0E\x5D\x31\xD0\x5E\x0D\x34\xDD\xF7\x44\x8D\xD7\x79\x72\x60\x40\xDA\x2F\x89\x1A\x6A\x72\x96\x7F\x2E\x0E\x0B\xE7\x37\xBF\x28\x0E\x0B\xD5\x7C\xD5\x30\xE4\x4F\x19\xAD\x9F\xD2\x52\x0F\x86\xF8\xD0\x5C\x86\x7A\x40\x79\xD5\x35\x3E\x5E\xCD\x1B\x59\xF3\xC6\x6B\x78\xA3\x5A\x77\x4C\x4E\xE6\x8D\xBC\x12\x6F\x1A\x34\x31\x36\x90\x23\xBC\xB1\x7F\xD6\xA0\x49\xAC\xF9\x13\x42\x53\x55\x18\x3B\x86\x7D\x9E\xD0\x74\x3F\x6A\x34\x09\x03\xFF\xEF\x1B\x83\x2F\x8C\x60\xF0\xB9\x06\x83\x2F\x36\x18\xFC\x29\x63\x50\x21\xE3\x55\xC8\xF8\x0D\xCF\x88\x6F\x3F\xA1\x64\x50\x59\xC7\x3B\x86\xAC\xE3\xDD\x9A\x6F\xD0\xB3\x60\xFF\x0F\x87\x55\xF9\xAF\xD2\x62\x88\x8B\x6E\xF2\x2E\x1F\xAE\x64\x72\x78\xA5\x33\x7C\xDD\xFA\xA2\xE4\xFB\xE9\xA5\x85\xEC\x97\x61\x75\x76\x97\xE5\x38\x68\x9D\x5A\x06\xDF\x91\xC7\xAF\xC8\x13\xB8\xD5\xC4\x16\x79\x7C\x74\xB3\x35\x79\xC2\x9A\x3C\x01\x78\x7C\x30\xCF\x5F\xEF\x96\x9E\x33\x2F\x4E\x4B\xFC\x66\x5A\x4C\x83\x20\xBC\xAD\x41\xD8\x6A\xE9\xE1\xB1\xE8\x2E\x03\xD0\x5D\x3E\x45\x76\x47\xFB\x62\xD0\xD6\xCF\x81\xF9\xD1\x16\xCD\x3E\x29\x1A\xA2\x21\x67\x93\xFC\xB7\x5B\x44\xFA\x47\x45\xA1\xAF\x8C\x3C\x9F\x14\xC3\xF4\x69\x7E\x0F\xCC\xCF\x28\x3A\x8C\x59\xE9\xFF\xF4\x7B\xF7\xE8\x8A\xBC\xFC\xAF\xAB\x18\x85\x6E\x17\xCC\x5F\x71\x3F\xE5\x01\x15\xF1\x0A\x75\xCB\xAA\x54\x57\x0A\x96\xA4\x1E\x8D\x1F\x93\x6D\x3F\xA6\x58\x0F\xFD\x3D\x0D\xA6\x7F\x25\x83\xD9\xF6\x63\xCA\x1D\xBF\xAE\xF4\xB0\xF1\x21\xB2\xF1\x63\x7C\xD4\xB7\x35\x02\xEB\xB6\x37\x3C\x10\x92\x51\x41\x63\x3B\xFF\xF0\xFD\x43\xB7\x28\xFF\x89\x2F\xE3\xF6\xC2\xF8\x3F\x28\xD7\x66\xCB\xDA\xF6\x0C\x4F\x38\x1B\x33\x4F\x0B\xC9\x7C\x7B\x20\x4D\x99\xE9\x1E\xBC\x80\xDF\x28\xD1\xC2\x0A\xBD\xD5\x68\xA3\x63\xA3\x6E\x49\xB7\x85\x35\x38\xC5\x0D\x4E\xE1\x57\x85\x53\x04\x31\x2F\x51\xB8\x40\xA4\x8D\xCB\x67\x9C\x18\x0C\x4D\x9C\x1B\x70\x8B\xC8\x4C\x04\x35\x1E\x06\x35\x69\x40\xF5\xBE\x2A\x50\x63\x48\xA6\x81\xCA\xAF\x75\x46\x20\x75\x6F\x79\xDC\x1D\x81\x4B\xD5\x25\x65\xA3\xF0\x0E\x43\xDB\x12\x16\xFD\x77\x28\x2C\xE3\x71\x50\xFB\x7F\x73\xAD\x7B\xA7\xCA\xD7\x77\xE4\x4F\x3D\x81\xBA\xF0\x67\xCA\x25\x4F\x18\x9E\xEA\xE6\xFF\x3F\x1B\x99\x6E\xA9\xAA\x04\x01\x6A\x86\xF2\x03\xF8\x41\x28\xCC\x84\xB8\xCA\x69\x8C\x9A\xE4\xAE\xA3\x26\x20\x95\x3C\x45\x1F\x35\x13\x6A\xB2\x99\x98\xE4\xAE\x71\x0A\xA2\xFE\x56\xE3\xAA\xCB\xEE\x52\xFD\xBF\x6A\x2C\x64\xBE\xFB\x44\xF3\xFD\x89\xE6\xBB\xFD\x2D\x61\xFF\x87\xB0\xF4\x4E\x23\x7F\xBA\x55\xFE\x97\xC2\xFE\x95\xB0\x4F\x8F\x95\x73\xFD\x67\xDA\xE5\x76\x60\x3E\xEE\xC9\x60\x30\x6C\x7C\xFC\x46\xA4\xF8\x46\xB3\x32\xBC\x9F\x76\x2D\xCA\x7E\x59\x9D\xAB\x0E\xD8\xF8\x84\x8D\xE3\x0A\xC6\x1D\x57\x38\xD9\x71\x05\x35\x29\xA3\x9A\x94\x21\x04\xAC\x8A\xE8\xB8\x7C\x22\xA5\x0F\xAA\xA6\x64\x44\xD7\x0B\x56\xF2\x44\xAE\x2C\xEA\xBA\xA3\xCC\x12\xFC\xE6\xE6\x94\x6A\x3A\xA3\xF8\x3A\xC8\x61\x5C\x48\x93\x1B\x5C\x92\x06\x17\xFF\xAB\xC2\xC5\x69\x72\x04\x01\xDF\x41\xD9\xC2\xC1\xDE\x46\x67\x8A\x5A\xE0\xF3\x3E\x21\xDE\x52\xE0\xCD\xD1\x1B\x2A\xBA\x64\x51\x18\x08\xF9\x2D\x64\x15\xE9\x8D\x30\xC2\xE9\xF6\x44\x5E\xE8\xBF\x4D\x5E\xD0\xCD\x8A\xA3\xBC\xD0\xAC\x1B\xD3\xD4\xDA\x7C\xBA\xBA\x26\x10\x14\xEB\x33\xCF\xE3\xF5\xAC\x19\x9B\x92\xBA\x73\xFD\x93\xA7\xA4\xDE\xD0\x94\x94\xEF\x63\xD0\x0E\x19\xDD\xB8\x72\x7E\x77\xDA\x42\x46\x0F\x45\x44\x6D\x57\xCE\x97\x3B\x82\xAE\xA6\xA4\xA0\xED\xE1\x5A\xB0\xDC\xEA\x33\x61\x34\x6E\x93\x08\xB3\x54\x34\x48\x99\x7F\xA3\xA4\x37\x18\xBB\x70\x98\xEF\x17\x88\xE8\xDA\x0D\x2D\xD4\x82\xDD\x8D\x9A\xC8\x44\xB8\x3B\x36\xEC\x6E\xC4\xEF\x65\xF8\x46\x12\x0E\x3C\x24\xB8\xA1\xB2\x3A\x39\x07\x5B\x08\x47\x25\x7A\x4D\x24\x6A\x2A\x79\xC3\xBE\x5F\x8C\x98\x73\x39\x64\xCE\xC5\x90\x39\x97\xB5\xED\x92\x6C\xD8\xA5\xBB\x13\x91\x4D\x9B\xA4\x1D\x13\xCE\x9A\x7B\xCD\x3E\x05\xAA\x46\xF4\xF1\xEA\x86\xCD\xB5\x06\x8A\xC3\xB6\xE6\x7A\xC2\xD6\x2D\xF7\xE6\xC7\xB4\xA6\x64\x90\x71\xB3\x02\x6E\x7F\x20\x32\xCF\xEA\xB5\x8D\x8D\xCE\xD9\x95\x0D\xE0\x14\x40\x7B\x26\x8F\x3B\x0C\x55\xE5\x5E\xE7\x7C\x4F\x8C\x67\x3D\x9C\x5E\xA1\xCA\xF1\xD7\xAE\x71\xE6\x4A\x15\xCE\xF7\x6A\x58\xFF\xF8\x48\xF5\xB2\x9E\x3F\x1F\xE4\xCF\x85\xEF\xE7\xCF\xF7\xFE\x22\x7F\x76\xBF\x4C\x9F\xBB\x4F\xDF\x6E\xF1\xF3\x93\xFB\x56\xE9\xF3\xE9\xDF\xF8\x67\xF4\xD9\xF9\xC4\x27\xEC\x99\xCD\xB5\x5B\xC5\x99\xCD\x35\x71\xC7\x1B\xDF\xF4\xE6\x3B\xDF\x72\xE8\xAE\xB7\xDE\xBD\xF2\xF0\xEA\xE9\xCE\x19\x41\xF9\x9D\xFA\xBD\x0B\x1D\x71\x66\x65\x63\xA7\x23\x36\xB7\xFA\xB0\x76\xEE\x3C\xA7\x2B\xEA\x9C\x3E\x0C\xEF\x7C\xE0\xD8\xC1\x37\x1E\x82\xF3\x2B\xBD\x9D\xB5\xCD\xB3\xA2\x02\x94\x33\x30\x41\x67\x67\x75\xE5\x7C\x07\x56\x1F\x59\xE9\xAD\xAC\xF6\x3B\x3D\xF1\xFA\xB3\x62\x8F\x84\x4E\xA3\x88\xDA\x0F\x1A\xE3\xC5\xBE\x10\xE2\x47\xF6\x7B\x9B\x2B\xE7\x3A\xF2\x9F\xEC\x7F\xAF\x14\x4D\x32\x53\x39\x94\xC7\x54\x0D\xA5\x30\xD5\x43\xD9\x4B\xBD\xA1\xC4\xA5\xFE\x50\xCE\xD2\x60\x3C\xFB\x67\x38\x96\xD2\x33\x1A\xCB\xE8\x19\x8F\x25\xF4\x4C\xC6\xF2\x79\x9A\x56\x3A\xCF\xB4\xCE\xE6\x99\x8D\x24\xF3\x9C\x99\x98\xCB\x73\x76\x3C\x95\x67\x3E\x92\xC9\x73\x6E\x3C\x91\xE7\xBE\x76\x1E\xCF\x6B\xF6\x48\xE3\x79\xED\xA4\x2C\x9E\xFB\x27\xE4\xED\xBC\x6E\x38\x6D\xE7\xF5\x93\xB3\x76\xDE\x30\x21\x69\xE7\xAB\x26\xE6\xEC\xBC\x71\x24\x65\xE7\xFC\xB4\x8C\x9D\x37\x4D\x4C\xD8\xF9\xEA\x29\xF9\x3A\x5F\x33\x2D\x5D\xE7\xCD\x63\xD9\x3A\x61\x5A\xB2\xCE\x62\x6A\xAE\xCE\x72\x52\x76\xCE\xD7\x4E\x4D\xCE\xF9\xBA\x69\xD9\x38\x5F\x3F\x35\x19\xE7\xD7\x8C\xE6\xE2\xFC\xDA\x91\x54\x9C\x07\x26\x67\xE2\xBC\x65\x24\x11\xE7\x1B\x46\xF3\x70\xDE\x3A\x9A\x86\xF3\xB6\xA1\x2C\x9C\x07\x47\x93\x70\xDE\x3E\x9A\x83\xF3\xEB\xC6\x53\x70\xDE\x71\x85\x0C\x9C\x6F\x6C\x27\xE0\x7C\xD3\xDE\xF9\x37\xDF\x3C\x96\x7E\xF3\xCE\x91\xEC\x9B\x6F\x19\x49\xBE\x79\x68\xEF\xDC\x9B\x77\xB5\x52\x6F\xBE\x75\xCF\xCC\x9B\x77\x4F\x4C\xBC\x79\x78\xEF\xBC\x9B\x5F\xBF\x47\xDA\xCD\x6F\x68\x67\xDD\xFC\xC6\x29\x49\x37\xBF\xA9\xCE\xB9\x79\x64\x34\xE5\xE6\xC2\xE4\x8C\x9B\x76\x62\x8E\xCD\xC5\xB1\x14\x9B\x6F\x6B\x32\x6C\xDE\x33\x21\xC1\xE6\xD1\x56\x7E\xCD\x63\x55\x7A\xCD\xE3\x93\xB3\x6B\xDE\x3B\x35\xB9\xE6\xD2\x58\x6E\xCD\xFB\x46\x52\x6B\x9E\x98\x96\x59\xF3\x9B\x27\x26\xD6\xBC\x7F\x28\xAF\xE6\xB7\xB8\xB4\x9A\x27\x9B\xAC\x9A\x6F\x4F\x96\xEB\xAC\x9A\xDF\x6A\x96\xB7\x2E\xF4\x97\x1F\xBE\x70\xE6\x4C\xA7\xF7\x6D\x31\xFD\x40\x52\xBE\xC3\x5F\xEE\xF4\xB7\x56\x1E\xF0\x97\xCF\xF4\xB7\x56\xDE\x99\x2E\x6F\xF6\xB7\x56\x96\xCF\x6C\xF5\xCE\xAD\xF4\xBF\xBD\x95\x94\xF3\x3B\x5A\xB9\x4B\xBF\xB3\x9D\x41\xF7\xC1\x56\xB2\xCE\x87\xDA\xB9\x3A\xBF\xAB\x9D\xAA\xF3\xBB\x9B\x4C\x9D\xDF\xD3\x4E\xD4\xF9\xAE\x3A\x4F\xE7\x72\x2B\x0F\xF2\xA9\x56\x82\xDF\x95\xB1\xAC\x9D\x0F\x4F\x49\xDA\xB9\x3A\x39\x67\xE7\xE9\x69\x29\x3B\x3B\x7B\x65\xEC\x3C\x33\x29\x61\xE7\xD9\x29\xF9\x3A\x1F\x19\x4B\xD7\xB9\x36\x25\x5B\x67\x77\x6A\xB2\xCE\xF5\xE9\xB9\x3A\x27\xE5\xE3\x64\xD9\x3A\x37\x35\x53\xE7\xE6\x78\xA2\xCE\xAD\xF1\x3C\x9D\x63\xE9\x38\xB9\xF1\xF6\xE4\x2C\x9D\xBD\x89\x49\x3A\x77\x46\x4B\xFB\x5B\xEB\x9D\xCD\xFE\xE4\xCC\x9D\x17\x46\x12\x77\x5E\x9C\x98\xB7\xF3\xD1\x29\x69\x3B\x1F\x9B\x9C\xB5\xF3\xF1\x89\x49\x3B\xBF\x77\x8F\x9C\x9D\x53\xD2\x72\xE2\xB3\x4B\xD3\x33\x76\xBE\x7B\xCF\x84\x9D\x97\xF7\xC8\xD7\x39\xD8\x33\x5D\xE7\xB4\x7C\x9C\x4C\xF5\x27\xF6\xCC\xD6\xF9\xE4\xD5\x24\xEB\x7C\x6A\xAF\x5C\x9D\x4F\x5F\x31\x55\xE7\x33\x57\x95\xA9\xF3\x3D\x57\x4E\xD4\xF9\xEC\x48\x9E\xCE\xFF\x17\x00\x00\xFF\xFF\x78\x9C\x11\x2A\x12\x80\x00\x00") diff --git a/vendor/github.com/open-policy-agent/opa/internal/compiler/wasm/opa/opa.wasm b/vendor/github.com/open-policy-agent/opa/internal/compiler/wasm/opa/opa.wasm new file mode 100644 index 000000000..6b9592b50 Binary files /dev/null and b/vendor/github.com/open-policy-agent/opa/internal/compiler/wasm/opa/opa.wasm differ diff --git a/vendor/github.com/open-policy-agent/opa/internal/compiler/wasm/wasm.go b/vendor/github.com/open-policy-agent/opa/internal/compiler/wasm/wasm.go new file mode 100644 index 000000000..51b2ed16b --- /dev/null +++ b/vendor/github.com/open-policy-agent/opa/internal/compiler/wasm/wasm.go @@ -0,0 +1,1013 @@ +// Copyright 2018 The OPA Authors. All rights reserved. +// Use of this source code is governed by an Apache2 +// license that can be found in the LICENSE file. + +// Package wasm contains an IR->WASM compiler backend. +package wasm + +import ( + "bytes" + "fmt" + + "github.com/pkg/errors" + + "github.com/open-policy-agent/opa/internal/compiler/wasm/opa" + "github.com/open-policy-agent/opa/internal/ir" + "github.com/open-policy-agent/opa/internal/wasm/encoding" + "github.com/open-policy-agent/opa/internal/wasm/instruction" + "github.com/open-policy-agent/opa/internal/wasm/module" + "github.com/open-policy-agent/opa/internal/wasm/types" +) + +const ( + opaTypeNull int32 = iota + 1 + opaTypeBoolean + opaTypeNumber + opaTypeString + opaTypeArray + opaTypeObject +) + +const ( + opaFuncPrefix = "opa_" + opaAbort = "opa_abort" + opaJSONParse = "opa_json_parse" + opaNull = "opa_null" + opaBoolean = "opa_boolean" + opaNumberInt = "opa_number_int" + opaNumberFloat = "opa_number_float" + opaNumberRef = "opa_number_ref" + opaNumberSize = "opa_number_size" + opaArrayWithCap = "opa_array_with_cap" + opaArrayAppend = "opa_array_append" + opaObject = "opa_object" + opaObjectInsert = "opa_object_insert" + opaSet = "opa_set" + opaSetAdd = "opa_set_add" + opaStringTerminated = "opa_string_terminated" + opaValueBooleanSet = "opa_value_boolean_set" + opaValueNumberSetInt = "opa_value_number_set_int" + opaValueCompare = "opa_value_compare" + opaValueGet = "opa_value_get" + opaValueIter = "opa_value_iter" + opaValueLength = "opa_value_length" + opaValueMerge = "opa_value_merge" + opaValueShallowCopy = "opa_value_shallow_copy" + opaValueType = "opa_value_type" +) + +var builtins = [...]string{ + "opa_builtin0", + "opa_builtin1", + "opa_builtin2", + "opa_builtin3", + "opa_builtin4", +} + +// Compiler implements an IR->WASM compiler backend. +type Compiler struct { + stages []func() error // compiler stages to execute + errors []error // compilation errors encountered + + policy *ir.Policy // input policy to compile + module *module.Module // output WASM module + code *module.CodeEntry // output WASM code + + builtinStringAddrs map[int]uint32 // addresses of built-in string constants + builtinFuncNameAddrs map[string]int32 // addresses of built-in function names for listing + builtinFuncs map[string]int32 // built-in function ids + stringOffset int32 // null-terminated string data base offset + stringAddrs []uint32 // null-terminated string constant addresses + funcs map[string]uint32 // maps imported and exported function names to function indices + + nextLocal uint32 + locals map[ir.Local]uint32 + lctx uint32 // local pointing to eval context + lrs uint32 // local pointing to result set +} + +const ( + errVarAssignConflict int = iota + errObjectInsertConflict + errObjectMergeConflict + errWithConflict +) + +var errorMessages = [...]struct { + id int + message string +}{ + {errVarAssignConflict, "var assignment conflict"}, + {errObjectInsertConflict, "object insert conflict"}, + {errObjectMergeConflict, "object merge conflict"}, + {errWithConflict, "with target conflict"}, +} + +// New returns a new compiler object. +func New() *Compiler { + c := &Compiler{} + c.stages = []func() error{ + c.initModule, + c.compileStrings, + c.compileBuiltinDecls, + c.compileFuncs, + c.compilePlan, + } + return c +} + +// WithPolicy sets the policy to compile. +func (c *Compiler) WithPolicy(p *ir.Policy) *Compiler { + c.policy = p + return c +} + +// Compile returns a compiled WASM module. +func (c *Compiler) Compile() (*module.Module, error) { + + for _, stage := range c.stages { + if err := stage(); err != nil { + return nil, err + } else if len(c.errors) > 0 { + return nil, c.errors[0] // TODO(tsandall) return all errors. + } + } + + return c.module, nil +} + +// initModule instantiates the module from the pre-compiled OPA binary. The +// module is then updated to include declarations for all of the functions that +// are about to be compiled. +func (c *Compiler) initModule() error { + + bs, err := opa.Bytes() + if err != nil { + return err + } + + c.module, err = encoding.ReadModule(bytes.NewReader(bs)) + if err != nil { + return err + } + + c.funcs = make(map[string]uint32) + var funcidx uint32 + + for _, imp := range c.module.Import.Imports { + if imp.Descriptor.Kind() == module.FunctionImportType { + c.funcs[imp.Name] = funcidx + funcidx++ + } + } + + for i := range c.module.Export.Exports { + exp := &c.module.Export.Exports[i] + if exp.Descriptor.Type == module.FunctionExportType { + c.funcs[exp.Name] = exp.Descriptor.Index + } + } + + for _, fn := range c.policy.Funcs.Funcs { + + params := make([]types.ValueType, len(fn.Params)) + for i := 0; i < len(params); i++ { + params[i] = types.I32 + } + + tpe := module.FunctionType{ + Params: params, + Results: []types.ValueType{types.I32}, + } + + c.emitFunctionDecl(fn.Name, tpe, false) + } + + c.emitFunctionDecl("eval", module.FunctionType{ + Params: []types.ValueType{types.I32}, + Results: []types.ValueType{types.I32}, + }, true) + + c.emitFunctionDecl("builtins", module.FunctionType{ + Params: nil, + Results: []types.ValueType{types.I32}, + }, true) + + return nil +} + +// compileStrings compiles string constants into the data section of the module. +// The strings are indexed for lookups in later stages. +func (c *Compiler) compileStrings() error { + + c.stringOffset = 2048 + c.stringAddrs = make([]uint32, len(c.policy.Static.Strings)) + var buf bytes.Buffer + + for i, s := range c.policy.Static.Strings { + addr := uint32(buf.Len()) + uint32(c.stringOffset) + buf.WriteString(s.Value) + buf.WriteByte(0) + c.stringAddrs[i] = addr + } + + c.builtinFuncNameAddrs = make(map[string]int32, len(c.policy.Static.BuiltinFuncs)) + + for _, decl := range c.policy.Static.BuiltinFuncs { + addr := int32(buf.Len()) + int32(c.stringOffset) + buf.WriteString(decl.Name) + buf.WriteByte(0) + c.builtinFuncNameAddrs[decl.Name] = addr + } + + c.builtinStringAddrs = make(map[int]uint32, len(errorMessages)) + + for i := range errorMessages { + addr := uint32(buf.Len()) + uint32(c.stringOffset) + buf.WriteString(errorMessages[i].message) + buf.WriteByte(0) + c.builtinStringAddrs[errorMessages[i].id] = addr + } + + c.module.Data.Segments = append(c.module.Data.Segments, module.DataSegment{ + Index: 0, + Offset: module.Expr{ + Instrs: []instruction.Instruction{ + instruction.I32Const{ + Value: c.stringOffset, + }, + }, + }, + Init: buf.Bytes(), + }) + + return nil +} + +// compileBuiltinDecls generates a function that lists the built-ins required by +// the policy. The host environment should invoke this function obtain the list +// of built-in function identifiers (represented as integers) that will be used +// when calling out. +func (c *Compiler) compileBuiltinDecls() error { + + c.code = &module.CodeEntry{} + c.nextLocal = 0 + c.locals = map[ir.Local]uint32{} + + lobj := c.genLocal() + + c.appendInstr(instruction.Call{Index: c.function(opaObject)}) + c.appendInstr(instruction.SetLocal{Index: lobj}) + c.builtinFuncs = make(map[string]int32, len(c.policy.Static.BuiltinFuncs)) + + for index, decl := range c.policy.Static.BuiltinFuncs { + c.appendInstr(instruction.GetLocal{Index: lobj}) + c.appendInstr(instruction.I32Const{Value: c.builtinFuncNameAddrs[decl.Name]}) + c.appendInstr(instruction.Call{Index: c.function(opaStringTerminated)}) + c.appendInstr(instruction.I64Const{Value: int64(index)}) + c.appendInstr(instruction.Call{Index: c.function(opaNumberInt)}) + c.appendInstr(instruction.Call{Index: c.function(opaObjectInsert)}) + c.builtinFuncs[decl.Name] = int32(index) + } + + c.appendInstr(instruction.GetLocal{Index: lobj}) + + c.code.Func.Locals = []module.LocalDeclaration{ + { + Count: c.nextLocal, + Type: types.I32, + }, + } + + return c.emitFunction("builtins", c.code) +} + +// compileFuncs compiles the policy functions and emits them into the module. +func (c *Compiler) compileFuncs() error { + + for _, fn := range c.policy.Funcs.Funcs { + if err := c.compileFunc(fn); err != nil { + return errors.Wrapf(err, "func %v", fn.Name) + } + } + + return nil +} + +// compilePlan compiles the policy plan and emits the resulting function into +// the module. +func (c *Compiler) compilePlan() error { + + c.code = &module.CodeEntry{} + c.nextLocal = 0 + c.locals = map[ir.Local]uint32{} + c.lctx = c.genLocal() + c.lrs = c.genLocal() + + // Initialize the input and data locals. + c.appendInstr(instruction.GetLocal{Index: c.lctx}) + c.appendInstr(instruction.I32Load{Offset: 0, Align: 2}) + c.appendInstr(instruction.SetLocal{Index: c.local(ir.Input)}) + + c.appendInstr(instruction.GetLocal{Index: c.lctx}) + c.appendInstr(instruction.I32Load{Offset: 4, Align: 2}) + c.appendInstr(instruction.SetLocal{Index: c.local(ir.Data)}) + + // Initialize the result set. + c.appendInstr(instruction.Call{Index: c.function(opaSet)}) + c.appendInstr(instruction.SetLocal{Index: c.lrs}) + c.appendInstr(instruction.GetLocal{Index: c.lctx}) + c.appendInstr(instruction.GetLocal{Index: c.lrs}) + c.appendInstr(instruction.I32Store{Offset: 8, Align: 2}) + + for i := range c.policy.Plan.Blocks { + + instrs, err := c.compileBlock(c.policy.Plan.Blocks[i]) + if err != nil { + return errors.Wrapf(err, "block %d", i) + } + + c.appendInstr(instruction.Block{Instrs: instrs}) + } + + c.appendInstr(instruction.I32Const{Value: int32(0)}) + + c.code.Func.Locals = []module.LocalDeclaration{ + { + Count: c.nextLocal, + Type: types.I32, + }, + } + + return c.emitFunction("eval", c.code) +} + +func (c *Compiler) compileFunc(fn *ir.Func) error { + + if len(fn.Params) == 0 { + return fmt.Errorf("illegal function: zero args") + } + + c.nextLocal = 0 + c.locals = map[ir.Local]uint32{} + + for _, a := range fn.Params { + _ = c.local(a) + } + + _ = c.local(fn.Return) + + c.code = &module.CodeEntry{} + + for i := range fn.Blocks { + instrs, err := c.compileBlock(fn.Blocks[i]) + if err != nil { + return errors.Wrapf(err, "block %d", i) + } + if i < len(fn.Blocks)-1 { + c.appendInstr(instruction.Block{Instrs: instrs}) + } else { + c.appendInstrs(instrs) + } + } + + c.code.Func.Locals = []module.LocalDeclaration{ + { + Count: c.nextLocal, + Type: types.I32, + }, + } + + var params []types.ValueType + + for i := 0; i < len(fn.Params); i++ { + params = append(params, types.I32) + } + + return c.emitFunction(fn.Name, c.code) +} + +func (c *Compiler) compileBlock(block *ir.Block) ([]instruction.Instruction, error) { + + var instrs []instruction.Instruction + + for _, stmt := range block.Stmts { + switch stmt := stmt.(type) { + case *ir.ResultSetAdd: + instrs = append(instrs, instruction.GetLocal{Index: c.lrs}) + instrs = append(instrs, instruction.GetLocal{Index: c.local(stmt.Value)}) + instrs = append(instrs, instruction.Call{Index: c.function(opaSetAdd)}) + case *ir.ReturnLocalStmt: + instrs = append(instrs, instruction.GetLocal{Index: c.local(stmt.Source)}) + instrs = append(instrs, instruction.Return{}) + case *ir.BlockStmt: + for i := range stmt.Blocks { + block, err := c.compileBlock(stmt.Blocks[i]) + if err != nil { + return nil, err + } + instrs = append(instrs, instruction.Block{Instrs: block}) + } + case *ir.BreakStmt: + instrs = append(instrs, instruction.Br{Index: stmt.Index}) + case *ir.CallStmt: + if err := c.compileCallStmt(stmt, &instrs); err != nil { + return nil, err + } + case *ir.WithStmt: + if err := c.compileWithStmt(stmt, &instrs); err != nil { + return instrs, err + } + case *ir.AssignVarStmt: + instrs = append(instrs, instruction.GetLocal{Index: c.local(stmt.Source)}) + instrs = append(instrs, instruction.SetLocal{Index: c.local(stmt.Target)}) + case *ir.AssignVarOnceStmt: + instrs = append(instrs, instruction.Block{ + Instrs: []instruction.Instruction{ + instruction.Block{ + Instrs: []instruction.Instruction{ + instruction.GetLocal{Index: c.local(stmt.Target)}, + instruction.I32Eqz{}, + instruction.BrIf{Index: 0}, + instruction.GetLocal{Index: c.local(stmt.Target)}, + instruction.GetLocal{Index: c.local(stmt.Source)}, + instruction.Call{Index: c.function(opaValueCompare)}, + instruction.I32Eqz{}, + instruction.BrIf{Index: 1}, + instruction.I32Const{Value: c.builtinStringAddr(errVarAssignConflict)}, + instruction.Call{Index: c.function(opaAbort)}, + instruction.Unreachable{}, + }, + }, + instruction.GetLocal{Index: c.local(stmt.Source)}, + instruction.SetLocal{Index: c.local(stmt.Target)}, + }, + }) + case *ir.AssignBooleanStmt: + instrs = append(instrs, instruction.GetLocal{Index: c.local(stmt.Target)}) + if stmt.Value { + instrs = append(instrs, instruction.I32Const{Value: 1}) + } else { + instrs = append(instrs, instruction.I32Const{Value: 0}) + } + instrs = append(instrs, instruction.Call{Index: c.function(opaValueBooleanSet)}) + case *ir.AssignIntStmt: + instrs = append(instrs, instruction.GetLocal{Index: c.local(stmt.Target)}) + instrs = append(instrs, instruction.I64Const{Value: stmt.Value}) + instrs = append(instrs, instruction.Call{Index: c.function(opaValueNumberSetInt)}) + case *ir.ScanStmt: + if err := c.compileScan(stmt, &instrs); err != nil { + return nil, err + } + case *ir.NotStmt: + if err := c.compileNot(stmt, &instrs); err != nil { + return nil, err + } + case *ir.DotStmt: + instrs = append(instrs, instruction.GetLocal{Index: c.local(stmt.Source)}) + instrs = append(instrs, instruction.GetLocal{Index: c.local(stmt.Key)}) + instrs = append(instrs, instruction.Call{Index: c.function(opaValueGet)}) + instrs = append(instrs, instruction.SetLocal{Index: c.local(stmt.Target)}) + instrs = append(instrs, instruction.GetLocal{Index: c.local(stmt.Target)}) + instrs = append(instrs, instruction.I32Eqz{}) + instrs = append(instrs, instruction.BrIf{Index: 0}) + case *ir.LenStmt: + instrs = append(instrs, instruction.GetLocal{Index: c.local(stmt.Source)}) + instrs = append(instrs, instruction.Call{Index: c.function(opaValueLength)}) + instrs = append(instrs, instruction.Call{Index: c.function(opaNumberSize)}) + instrs = append(instrs, instruction.SetLocal{Index: c.local(stmt.Target)}) + case *ir.EqualStmt: + instrs = append(instrs, instruction.GetLocal{Index: c.local(stmt.A)}) + instrs = append(instrs, instruction.GetLocal{Index: c.local(stmt.B)}) + instrs = append(instrs, instruction.Call{Index: c.function(opaValueCompare)}) + instrs = append(instrs, instruction.BrIf{Index: 0}) + case *ir.LessThanStmt: + instrs = append(instrs, instruction.GetLocal{Index: c.local(stmt.A)}) + instrs = append(instrs, instruction.GetLocal{Index: c.local(stmt.B)}) + instrs = append(instrs, instruction.Call{Index: c.function(opaValueCompare)}) + instrs = append(instrs, instruction.I32Const{Value: 0}) + instrs = append(instrs, instruction.I32GeS{}) + instrs = append(instrs, instruction.BrIf{Index: 0}) + case *ir.LessThanEqualStmt: + instrs = append(instrs, instruction.GetLocal{Index: c.local(stmt.A)}) + instrs = append(instrs, instruction.GetLocal{Index: c.local(stmt.B)}) + instrs = append(instrs, instruction.Call{Index: c.function(opaValueCompare)}) + instrs = append(instrs, instruction.I32Const{Value: 0}) + instrs = append(instrs, instruction.I32GtS{}) + instrs = append(instrs, instruction.BrIf{Index: 0}) + case *ir.GreaterThanStmt: + instrs = append(instrs, instruction.GetLocal{Index: c.local(stmt.A)}) + instrs = append(instrs, instruction.GetLocal{Index: c.local(stmt.B)}) + instrs = append(instrs, instruction.Call{Index: c.function(opaValueCompare)}) + instrs = append(instrs, instruction.I32Const{Value: 0}) + instrs = append(instrs, instruction.I32LeS{}) + instrs = append(instrs, instruction.BrIf{Index: 0}) + case *ir.GreaterThanEqualStmt: + instrs = append(instrs, instruction.GetLocal{Index: c.local(stmt.A)}) + instrs = append(instrs, instruction.GetLocal{Index: c.local(stmt.B)}) + instrs = append(instrs, instruction.Call{Index: c.function(opaValueCompare)}) + instrs = append(instrs, instruction.I32Const{Value: 0}) + instrs = append(instrs, instruction.I32LtS{}) + instrs = append(instrs, instruction.BrIf{Index: 0}) + case *ir.NotEqualStmt: + instrs = append(instrs, instruction.GetLocal{Index: c.local(stmt.A)}) + instrs = append(instrs, instruction.GetLocal{Index: c.local(stmt.B)}) + instrs = append(instrs, instruction.Call{Index: c.function(opaValueCompare)}) + instrs = append(instrs, instruction.I32Eqz{}) + instrs = append(instrs, instruction.BrIf{Index: 0}) + case *ir.MakeNullStmt: + instrs = append(instrs, instruction.Call{Index: c.function(opaNull)}) + instrs = append(instrs, instruction.SetLocal{Index: c.local(stmt.Target)}) + case *ir.MakeBooleanStmt: + instr := instruction.I32Const{} + if stmt.Value { + instr.Value = 1 + } else { + instr.Value = 0 + } + instrs = append(instrs, instr) + instrs = append(instrs, instruction.Call{Index: c.function(opaBoolean)}) + instrs = append(instrs, instruction.SetLocal{Index: c.local(stmt.Target)}) + case *ir.MakeNumberFloatStmt: + instrs = append(instrs, instruction.F64Const{Value: stmt.Value}) + instrs = append(instrs, instruction.Call{Index: c.function(opaNumberFloat)}) + instrs = append(instrs, instruction.SetLocal{Index: c.local(stmt.Target)}) + case *ir.MakeNumberIntStmt: + instrs = append(instrs, instruction.I64Const{Value: stmt.Value}) + instrs = append(instrs, instruction.Call{Index: c.function(opaNumberInt)}) + instrs = append(instrs, instruction.SetLocal{Index: c.local(stmt.Target)}) + case *ir.MakeNumberRefStmt: + instrs = append(instrs, instruction.I32Const{Value: c.stringAddr(stmt.Index)}) + instrs = append(instrs, instruction.I32Const{Value: int32(len(c.policy.Static.Strings[stmt.Index].Value))}) + instrs = append(instrs, instruction.Call{Index: c.function(opaNumberRef)}) + instrs = append(instrs, instruction.SetLocal{Index: c.local(stmt.Target)}) + case *ir.MakeStringStmt: + instrs = append(instrs, instruction.I32Const{Value: c.stringAddr(stmt.Index)}) + instrs = append(instrs, instruction.Call{Index: c.function(opaStringTerminated)}) + instrs = append(instrs, instruction.SetLocal{Index: c.local(stmt.Target)}) + case *ir.MakeArrayStmt: + instrs = append(instrs, instruction.I32Const{Value: stmt.Capacity}) + instrs = append(instrs, instruction.Call{Index: c.function(opaArrayWithCap)}) + instrs = append(instrs, instruction.SetLocal{Index: c.local(stmt.Target)}) + case *ir.MakeObjectStmt: + instrs = append(instrs, instruction.Call{Index: c.function(opaObject)}) + instrs = append(instrs, instruction.SetLocal{Index: c.local(stmt.Target)}) + case *ir.MakeSetStmt: + instrs = append(instrs, instruction.Call{Index: c.function(opaSet)}) + instrs = append(instrs, instruction.SetLocal{Index: c.local(stmt.Target)}) + case *ir.IsArrayStmt: + instrs = append(instrs, instruction.GetLocal{Index: c.local(stmt.Source)}) + instrs = append(instrs, instruction.Call{Index: c.function(opaValueType)}) + instrs = append(instrs, instruction.I32Const{Value: opaTypeArray}) + instrs = append(instrs, instruction.I32Ne{}) + instrs = append(instrs, instruction.BrIf{Index: 0}) + case *ir.IsObjectStmt: + instrs = append(instrs, instruction.GetLocal{Index: c.local(stmt.Source)}) + instrs = append(instrs, instruction.Call{Index: c.function(opaValueType)}) + instrs = append(instrs, instruction.I32Const{Value: opaTypeObject}) + instrs = append(instrs, instruction.I32Ne{}) + instrs = append(instrs, instruction.BrIf{Index: 0}) + case *ir.IsUndefinedStmt: + instrs = append(instrs, instruction.GetLocal{Index: c.local(stmt.Source)}) + instrs = append(instrs, instruction.I32Const{Value: 0}) + instrs = append(instrs, instruction.I32Ne{}) + instrs = append(instrs, instruction.BrIf{Index: 0}) + case *ir.IsDefinedStmt: + instrs = append(instrs, instruction.GetLocal{Index: c.local(stmt.Source)}) + instrs = append(instrs, instruction.I32Eqz{}) + instrs = append(instrs, instruction.BrIf{Index: 0}) + case *ir.ArrayAppendStmt: + instrs = append(instrs, instruction.GetLocal{Index: c.local(stmt.Array)}) + instrs = append(instrs, instruction.GetLocal{Index: c.local(stmt.Value)}) + instrs = append(instrs, instruction.Call{Index: c.function(opaArrayAppend)}) + case *ir.ObjectInsertStmt: + instrs = append(instrs, instruction.GetLocal{Index: c.local(stmt.Object)}) + instrs = append(instrs, instruction.GetLocal{Index: c.local(stmt.Key)}) + instrs = append(instrs, instruction.GetLocal{Index: c.local(stmt.Value)}) + instrs = append(instrs, instruction.Call{Index: c.function(opaObjectInsert)}) + case *ir.ObjectInsertOnceStmt: + tmp := c.genLocal() + instrs = append(instrs, instruction.Block{ + Instrs: []instruction.Instruction{ + instruction.Block{ + Instrs: []instruction.Instruction{ + instruction.GetLocal{Index: c.local(stmt.Object)}, + instruction.GetLocal{Index: c.local(stmt.Key)}, + instruction.Call{Index: c.function(opaValueGet)}, + instruction.SetLocal{Index: tmp}, + instruction.GetLocal{Index: tmp}, + instruction.I32Eqz{}, + instruction.BrIf{Index: 0}, + instruction.GetLocal{Index: tmp}, + instruction.GetLocal{Index: c.local(stmt.Value)}, + instruction.Call{Index: c.function(opaValueCompare)}, + instruction.I32Eqz{}, + instruction.BrIf{Index: 1}, + instruction.I32Const{Value: c.builtinStringAddr(errObjectInsertConflict)}, + instruction.Call{Index: c.function(opaAbort)}, + instruction.Unreachable{}, + }, + }, + instruction.GetLocal{Index: c.local(stmt.Object)}, + instruction.GetLocal{Index: c.local(stmt.Key)}, + instruction.GetLocal{Index: c.local(stmt.Value)}, + instruction.Call{Index: c.function(opaObjectInsert)}, + }, + }) + case *ir.ObjectMergeStmt: + instrs = append(instrs, instruction.GetLocal{Index: c.local(stmt.A)}) + instrs = append(instrs, instruction.GetLocal{Index: c.local(stmt.B)}) + instrs = append(instrs, instruction.Call{Index: c.function(opaValueMerge)}) + instrs = append(instrs, instruction.SetLocal{Index: c.local(stmt.Target)}) + instrs = append(instrs, instruction.Block{ + Instrs: []instruction.Instruction{ + instruction.GetLocal{Index: c.local(stmt.Target)}, + instruction.BrIf{Index: 0}, + instruction.I32Const{Value: c.builtinStringAddr(errObjectMergeConflict)}, + instruction.Call{Index: c.function(opaAbort)}, + instruction.Unreachable{}, + }, + }) + case *ir.SetAddStmt: + instrs = append(instrs, instruction.GetLocal{Index: c.local(stmt.Set)}) + instrs = append(instrs, instruction.GetLocal{Index: c.local(stmt.Value)}) + instrs = append(instrs, instruction.Call{Index: c.function(opaSetAdd)}) + default: + var buf bytes.Buffer + ir.Pretty(&buf, stmt) + return instrs, fmt.Errorf("illegal statement: %v", buf.String()) + } + } + + return instrs, nil +} + +func (c *Compiler) compileScan(scan *ir.ScanStmt, result *[]instruction.Instruction) error { + var instrs = *result + instrs = append(instrs, instruction.I32Const{Value: 0}) + instrs = append(instrs, instruction.SetLocal{Index: c.local(scan.Key)}) + body, err := c.compileScanBlock(scan) + if err != nil { + return err + } + instrs = append(instrs, instruction.Block{ + Instrs: []instruction.Instruction{ + instruction.Loop{Instrs: body}, + }, + }) + *result = instrs + return nil +} + +func (c *Compiler) compileScanBlock(scan *ir.ScanStmt) ([]instruction.Instruction, error) { + var instrs []instruction.Instruction + + // Execute iterator. + instrs = append(instrs, instruction.GetLocal{Index: c.local(scan.Source)}) + instrs = append(instrs, instruction.GetLocal{Index: c.local(scan.Key)}) + instrs = append(instrs, instruction.Call{Index: c.function(opaValueIter)}) + + // Check for emptiness. + instrs = append(instrs, instruction.SetLocal{Index: c.local(scan.Key)}) + instrs = append(instrs, instruction.GetLocal{Index: c.local(scan.Key)}) + instrs = append(instrs, instruction.I32Eqz{}) + instrs = append(instrs, instruction.BrIf{Index: 1}) + + // Load value. + instrs = append(instrs, instruction.GetLocal{Index: c.local(scan.Source)}) + instrs = append(instrs, instruction.GetLocal{Index: c.local(scan.Key)}) + instrs = append(instrs, instruction.Call{Index: c.function(opaValueGet)}) + instrs = append(instrs, instruction.SetLocal{Index: c.local(scan.Value)}) + + // Loop body. + nested, err := c.compileBlock(scan.Block) + if err != nil { + return nil, err + } + + // Continue. + instrs = append(instrs, nested...) + instrs = append(instrs, instruction.Br{Index: 0}) + + return instrs, nil +} + +func (c *Compiler) compileNot(not *ir.NotStmt, result *[]instruction.Instruction) error { + var instrs = *result + + // generate and initialize condition variable + cond := c.genLocal() + instrs = append(instrs, instruction.I32Const{Value: 1}) + instrs = append(instrs, instruction.SetLocal{Index: cond}) + + nested, err := c.compileBlock(not.Block) + if err != nil { + return err + } + + // unset condition variable if end of block is reached + nested = append(nested, instruction.I32Const{Value: 0}) + nested = append(nested, instruction.SetLocal{Index: cond}) + instrs = append(instrs, instruction.Block{Instrs: nested}) + + // break out of block if condition variable was unset + instrs = append(instrs, instruction.GetLocal{Index: cond}) + instrs = append(instrs, instruction.I32Eqz{}) + instrs = append(instrs, instruction.BrIf{Index: 0}) + + *result = instrs + return nil +} + +func (c *Compiler) compileWithStmt(with *ir.WithStmt, result *[]instruction.Instruction) error { + + var instrs = *result + save := c.genLocal() + instrs = append(instrs, instruction.GetLocal{Index: c.local(with.Local)}) + instrs = append(instrs, instruction.SetLocal{Index: save}) + + if len(with.Path) == 0 { + instrs = append(instrs, instruction.GetLocal{Index: c.local(with.Value)}) + instrs = append(instrs, instruction.SetLocal{Index: c.local(with.Local)}) + } else { + instrs = c.compileUpsert(with.Local, with.Path, with.Value, instrs) + } + + undefined := c.genLocal() + instrs = append(instrs, instruction.I32Const{Value: 1}) + instrs = append(instrs, instruction.SetLocal{Index: undefined}) + + nested, err := c.compileBlock(with.Block) + if err != nil { + return err + } + + nested = append(nested, instruction.I32Const{Value: 0}) + nested = append(nested, instruction.SetLocal{Index: undefined}) + instrs = append(instrs, instruction.Block{Instrs: nested}) + instrs = append(instrs, instruction.GetLocal{Index: save}) + instrs = append(instrs, instruction.SetLocal{Index: c.local(with.Local)}) + instrs = append(instrs, instruction.GetLocal{Index: undefined}) + instrs = append(instrs, instruction.BrIf{Index: 0}) + + *result = instrs + + return nil +} + +func (c *Compiler) compileUpsert(local ir.Local, path []int, value ir.Local, instrs []instruction.Instruction) []instruction.Instruction { + + lcopy := c.genLocal() // holds copy of local + instrs = append(instrs, instruction.GetLocal{Index: c.local(local)}) + instrs = append(instrs, instruction.SetLocal{Index: lcopy}) + + // Shallow copy the local if defined otherwise initialize to an empty object. + instrs = append(instrs, instruction.Block{ + Instrs: []instruction.Instruction{ + instruction.Block{Instrs: []instruction.Instruction{ + instruction.GetLocal{Index: lcopy}, + instruction.I32Eqz{}, + instruction.BrIf{Index: 0}, + instruction.GetLocal{Index: lcopy}, + instruction.Call{Index: c.function(opaValueShallowCopy)}, + instruction.SetLocal{Index: lcopy}, + instruction.GetLocal{Index: lcopy}, + instruction.SetLocal{Index: c.local(local)}, + instruction.Br{Index: 1}, + }}, + instruction.Call{Index: c.function(opaObject)}, + instruction.SetLocal{Index: lcopy}, + instruction.GetLocal{Index: lcopy}, + instruction.SetLocal{Index: c.local(local)}, + }, + }) + + // Initialize the locals that specify the path of the upsert operation. + lpath := make(map[int]uint32, len(path)) + + for i := 0; i < len(path); i++ { + lpath[i] = c.genLocal() + instrs = append(instrs, instruction.I32Const{Value: c.stringAddr(path[i])}) + instrs = append(instrs, instruction.Call{Index: c.function(opaStringTerminated)}) + instrs = append(instrs, instruction.SetLocal{Index: lpath[i]}) + } + + // Generate a block that traverses the path of the upsert operation, + // shallowing copying values at each step as needed. Stop before the final + // segment that will only be inserted. + var inner []instruction.Instruction + ltemp := c.genLocal() + + for i := 0; i < len(path)-1; i++ { + + // Lookup the next part of the path. + inner = append(inner, instruction.GetLocal{Index: lcopy}) + inner = append(inner, instruction.GetLocal{Index: lpath[i]}) + inner = append(inner, instruction.Call{Index: c.function(opaValueGet)}) + inner = append(inner, instruction.SetLocal{Index: ltemp}) + + // If the next node is missing, break. + inner = append(inner, instruction.GetLocal{Index: ltemp}) + inner = append(inner, instruction.I32Eqz{}) + inner = append(inner, instruction.BrIf{Index: uint32(i)}) + + // If the next node is not an object, generate a conflict error. + inner = append(inner, instruction.Block{ + Instrs: []instruction.Instruction{ + instruction.GetLocal{Index: ltemp}, + instruction.Call{Index: c.function(opaValueType)}, + instruction.I32Const{Value: opaTypeObject}, + instruction.I32Eq{}, + instruction.BrIf{Index: 0}, + instruction.I32Const{Value: c.builtinStringAddr(errWithConflict)}, + instruction.Call{Index: c.function(opaAbort)}, + }, + }) + + // Otherwise, shallow copy the next node node and insert into the copy + // before continuing. + inner = append(inner, instruction.GetLocal{Index: ltemp}) + inner = append(inner, instruction.Call{Index: c.function(opaValueShallowCopy)}) + inner = append(inner, instruction.SetLocal{Index: ltemp}) + inner = append(inner, instruction.GetLocal{Index: lcopy}) + inner = append(inner, instruction.GetLocal{Index: lpath[i]}) + inner = append(inner, instruction.GetLocal{Index: ltemp}) + inner = append(inner, instruction.Call{Index: c.function(opaObjectInsert)}) + inner = append(inner, instruction.GetLocal{Index: ltemp}) + inner = append(inner, instruction.SetLocal{Index: lcopy}) + } + + inner = append(inner, instruction.Br{Index: uint32(len(path) - 1)}) + + // Generate blocks that handle missing nodes during traversal. + var block []instruction.Instruction + lval := c.genLocal() + + for i := 0; i < len(path)-1; i++ { + block = append(block, instruction.Block{Instrs: inner}) + block = append(block, instruction.Call{Index: c.function(opaObject)}) + block = append(block, instruction.SetLocal{Index: lval}) + block = append(block, instruction.GetLocal{Index: lcopy}) + block = append(block, instruction.GetLocal{Index: lpath[i]}) + block = append(block, instruction.GetLocal{Index: lval}) + block = append(block, instruction.Call{Index: c.function(opaObjectInsert)}) + block = append(block, instruction.GetLocal{Index: lval}) + block = append(block, instruction.SetLocal{Index: lcopy}) + inner = block + block = nil + } + + // Finish by inserting the statement's value into the shallow copied node. + instrs = append(instrs, instruction.Block{Instrs: inner}) + instrs = append(instrs, instruction.GetLocal{Index: lcopy}) + instrs = append(instrs, instruction.GetLocal{Index: lpath[len(path)-1]}) + instrs = append(instrs, instruction.GetLocal{Index: c.local(value)}) + instrs = append(instrs, instruction.Call{Index: c.function(opaObjectInsert)}) + + return instrs +} + +func (c *Compiler) compileCallStmt(stmt *ir.CallStmt, result *[]instruction.Instruction) error { + + if index, ok := c.funcs[stmt.Func]; ok { + return c.compileInternalCall(stmt, index, result) + } + + if id, ok := c.builtinFuncs[stmt.Func]; ok { + return c.compileBuiltinCall(stmt, id, result) + } + + c.errors = append(c.errors, fmt.Errorf("undefined function: %q", stmt.Func)) + + return nil +} + +func (c *Compiler) compileInternalCall(stmt *ir.CallStmt, index uint32, result *[]instruction.Instruction) error { + instrs := *result + + for _, arg := range stmt.Args { + instrs = append(instrs, instruction.GetLocal{Index: c.local(arg)}) + } + + instrs = append(instrs, instruction.Call{Index: index}) + instrs = append(instrs, instruction.SetLocal{Index: c.local(stmt.Result)}) + instrs = append(instrs, instruction.GetLocal{Index: c.local(stmt.Result)}) + instrs = append(instrs, instruction.I32Eqz{}) + instrs = append(instrs, instruction.BrIf{Index: 0}) + *result = instrs + return nil +} + +func (c *Compiler) compileBuiltinCall(stmt *ir.CallStmt, id int32, result *[]instruction.Instruction) error { + + if len(stmt.Args) >= len(builtins) { + c.errors = append(c.errors, fmt.Errorf("too many built-in call arguments: %q", stmt.Func)) + return nil + } + + instrs := *result + instrs = append(instrs, instruction.I32Const{Value: id}) + instrs = append(instrs, instruction.I32Const{Value: 0}) // unused context parameter + + for _, arg := range stmt.Args { + instrs = append(instrs, instruction.GetLocal{Index: c.local(arg)}) + } + + instrs = append(instrs, instruction.Call{Index: c.funcs[builtins[len(stmt.Args)]]}) + instrs = append(instrs, instruction.SetLocal{Index: c.local(stmt.Result)}) + instrs = append(instrs, instruction.GetLocal{Index: c.local(stmt.Result)}) + instrs = append(instrs, instruction.I32Eqz{}) + instrs = append(instrs, instruction.BrIf{Index: 0}) + *result = instrs + return nil +} + +func (c *Compiler) emitFunctionDecl(name string, tpe module.FunctionType, export bool) { + + typeIndex := c.emitFunctionType(tpe) + c.module.Function.TypeIndices = append(c.module.Function.TypeIndices, typeIndex) + c.module.Code.Segments = append(c.module.Code.Segments, module.RawCodeSegment{}) + c.funcs[name] = uint32((len(c.module.Function.TypeIndices) - 1) + c.functionImportCount()) + + if export { + c.module.Export.Exports = append(c.module.Export.Exports, module.Export{ + Name: name, + Descriptor: module.ExportDescriptor{ + Type: module.FunctionExportType, + Index: c.funcs[name], + }, + }) + } + +} + +func (c *Compiler) emitFunctionType(tpe module.FunctionType) uint32 { + for i, other := range c.module.Type.Functions { + if tpe.Equal(other) { + return uint32(i) + } + } + c.module.Type.Functions = append(c.module.Type.Functions, tpe) + return uint32(len(c.module.Type.Functions) - 1) +} + +func (c *Compiler) emitFunction(name string, entry *module.CodeEntry) error { + var buf bytes.Buffer + if err := encoding.WriteCodeEntry(&buf, entry); err != nil { + return err + } + index := c.function(name) - uint32(c.functionImportCount()) + c.module.Code.Segments[index].Code = buf.Bytes() + return nil +} + +func (c *Compiler) functionImportCount() int { + var count int + + for _, imp := range c.module.Import.Imports { + if imp.Descriptor.Kind() == module.FunctionImportType { + count++ + } + } + + return count +} + +func (c *Compiler) stringAddr(index int) int32 { + return int32(c.stringAddrs[index]) +} + +func (c *Compiler) builtinStringAddr(code int) int32 { + return int32(c.builtinStringAddrs[code]) +} + +func (c *Compiler) local(l ir.Local) uint32 { + var u32 uint32 + var exist bool + if u32, exist = c.locals[l]; !exist { + u32 = c.nextLocal + c.locals[l] = u32 + c.nextLocal++ + } + return u32 +} + +func (c *Compiler) genLocal() uint32 { + l := c.nextLocal + c.nextLocal++ + return l +} + +func (c *Compiler) function(name string) uint32 { + return c.funcs[name] +} + +func (c *Compiler) appendInstr(instr instruction.Instruction) { + c.code.Func.Expr.Instrs = append(c.code.Func.Expr.Instrs, instr) +} + +func (c *Compiler) appendInstrs(instrs []instruction.Instruction) { + for _, instr := range instrs { + c.appendInstr(instr) + } +} diff --git a/vendor/github.com/open-policy-agent/opa/internal/file/archive/tarball.go b/vendor/github.com/open-policy-agent/opa/internal/file/archive/tarball.go new file mode 100644 index 000000000..6b8ba48d1 --- /dev/null +++ b/vendor/github.com/open-policy-agent/opa/internal/file/archive/tarball.go @@ -0,0 +1,42 @@ +package archive + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "strings" +) + +// MustWriteTarGz write the list of file names and content +// into a tarball. +func MustWriteTarGz(files [][2]string) *bytes.Buffer { + var buf bytes.Buffer + gw := gzip.NewWriter(&buf) + defer gw.Close() + tw := tar.NewWriter(gw) + defer tw.Close() + for _, file := range files { + if err := WriteFile(tw, file[0], []byte(file[1])); err != nil { + panic(err) + } + } + return &buf +} + +// WriteFile adds a file header with content to the given tar writer +func WriteFile(tw *tar.Writer, path string, bs []byte) error { + + hdr := &tar.Header{ + Name: "/" + strings.TrimLeft(path, "/"), + Mode: 0600, + Typeflag: tar.TypeReg, + Size: int64(len(bs)), + } + + if err := tw.WriteHeader(hdr); err != nil { + return err + } + + _, err := tw.Write(bs) + return err +} diff --git a/vendor/github.com/open-policy-agent/opa/internal/file/url/url.go b/vendor/github.com/open-policy-agent/opa/internal/file/url/url.go new file mode 100644 index 000000000..aacc57183 --- /dev/null +++ b/vendor/github.com/open-policy-agent/opa/internal/file/url/url.go @@ -0,0 +1,42 @@ +// Copyright 2019 The OPA Authors. All rights reserved. +// Use of this source code is governed by an Apache2 +// license that can be found in the LICENSE file. + +// Package url contains helpers for dealing with file paths and URLs. +package url + +import ( + "fmt" + "net/url" + "runtime" + "strings" +) + +var goos = runtime.GOOS + +// Clean returns a cleaned file path that may or may not be a URL. +func Clean(path string) (string, error) { + + if strings.Contains(path, "://") { + + url, err := url.Parse(path) + if err != nil { + return "", err + } + + if url.Scheme != "file" { + return "", fmt.Errorf("unsupported URL scheme: %v", path) + } + + path = url.Path + + // Trim leading slash on Windows if present. The url.Path field returned + // by url.Parse has leading slash that causes CreateFile() calls to fail + // on Windows. See https://github.com/golang/go/issues/6027 for details. + if goos == "windows" && len(path) >= 1 && path[0] == '/' { + path = path[1:] + } + } + + return path, nil +} diff --git a/vendor/github.com/open-policy-agent/opa/internal/ir/ir.go b/vendor/github.com/open-policy-agent/opa/internal/ir/ir.go new file mode 100644 index 000000000..14d67022f --- /dev/null +++ b/vendor/github.com/open-policy-agent/opa/internal/ir/ir.go @@ -0,0 +1,395 @@ +// Copyright 2018 The OPA Authors. All rights reserved. +// Use of this source code is governed by an Apache2 +// license that can be found in the LICENSE file. + +// Package ir defines an intermediate representation (IR) for Rego. +// +// The IR specifies an imperative execution model for Rego policies similar to a +// query plan in traditional databases. +package ir + +import ( + "fmt" +) + +type ( + // Policy represents a planned policy query. + Policy struct { + Static *Static + Plan *Plan + Funcs *Funcs + } + + // Static represents a static data segment that is indexed into by the policy. + Static struct { + Strings []*StringConst + BuiltinFuncs []*BuiltinFunc + } + + // BuiltinFunc represents a built-in function that may be required by the + // policy. + BuiltinFunc struct { + Name string + } + + // Funcs represents a collection of planned functions to include in the + // policy. + Funcs struct { + Funcs []*Func + } + + // Func represents a named plan (function) that can be invoked. Functions + // accept one or more parameters and return a value. By convention, the + // input document and data documents are always passed as the first and + // second arguments (respectively). + Func struct { + Name string + Params []Local + Return Local + Blocks []*Block // TODO(tsandall): should this be a plan? + } + + // Plan represents an ordered series of blocks to execute. Plan execution + // stops when a return statement is reached. Blocks are executed in-order. + Plan struct { + Blocks []*Block + } + + // Block represents an ordered sequence of statements to execute. Blocks are + // executed until a return statement is encountered, a statement is undefined, + // or there are no more statements. If all statements are defined but no return + // statement is encountered, the block is undefined. + Block struct { + Stmts []Stmt + } + + // Stmt represents an operation (e.g., comparison, loop, dot, etc.) to execute. + Stmt interface { + } + + // Local represents a plan-scoped variable. + // + // TODO(tsandall): should this be int32 for safety? + Local int + + // Const represents a constant value from the policy. + Const interface { + typeMarker() + } + + // NullConst represents a null value. + NullConst struct{} + + // BooleanConst represents a boolean value. + BooleanConst struct { + Value bool + } + + // StringConst represents a string value. + StringConst struct { + Value string + } + + // IntConst represents an integer constant. + IntConst struct { + Value int64 + } + + // FloatConst represents a floating-point constant. + FloatConst struct { + Value float64 + } +) + +const ( + // Input is the local variable that refers to the global input document. + Input Local = iota + + // Data is the local variable that refers to the global data document. + Data + + // Unused is the free local variable that can be allocated in a plan. + Unused +) + +func (a *Policy) String() string { + return "Policy" +} + +func (a *Static) String() string { + return fmt.Sprintf("Static (%d strings)", len(a.Strings)) +} + +func (a *Funcs) String() string { + return fmt.Sprintf("Funcs (%d funcs)", len(a.Funcs)) +} + +func (a *Func) String() string { + return fmt.Sprintf("%v (%d params: %v, %d blocks)", a.Name, len(a.Params), a.Params, len(a.Blocks)) +} + +func (a *Plan) String() string { + return fmt.Sprintf("Plan (%d blocks)", len(a.Blocks)) +} + +func (a *Block) String() string { + return fmt.Sprintf("Block (%d statements)", len(a.Stmts)) +} + +func (a *BooleanConst) typeMarker() {} +func (a *NullConst) typeMarker() {} +func (a *IntConst) typeMarker() {} +func (a *FloatConst) typeMarker() {} +func (a *StringConst) typeMarker() {} + +// ReturnLocalStmt represents a return statement that yields a local value. +type ReturnLocalStmt struct { + Source Local +} + +// CallStmt represents a named function call. The result should be stored in the +// result local. +type CallStmt struct { + Func string + Args []Local + Result Local +} + +// BlockStmt represents a nested block. Nested blocks and break statements can +// be used to short-circuit execution. +type BlockStmt struct { + Blocks []*Block +} + +func (a *BlockStmt) String() string { + return fmt.Sprintf("BlockStmt (%d blocks)", len(a.Blocks)) +} + +// BreakStmt represents a jump out of the current block. The index specifies how +// many blocks to jump starting from zero (the current block). Execution will +// continue from the end of the block that is jumped to. +type BreakStmt struct { + Index uint32 +} + +// DotStmt represents a lookup operation on a value (e.g., array, object, etc.) +// The source of a DotStmt may be a scalar value in which case the statement +// will be undefined. +type DotStmt struct { + Source Local + Key Local + Target Local +} + +// LenStmt represents a length() operation on a local variable. The +// result is stored in the target local variable. +type LenStmt struct { + Source Local + Target Local +} + +// ScanStmt represents a linear scan over a composite value. The +// source may be a scalar in which case the block will never execute. +type ScanStmt struct { + Source Local + Key Local + Value Local + Block *Block +} + +// NotStmt represents a negated statement. +type NotStmt struct { + Block *Block +} + +// AssignBooleanStmt represents an assignment of a boolean value to a local variable. +type AssignBooleanStmt struct { + Value bool + Target Local +} + +// AssignIntStmt represents an assignment of an integer value to a +// local variable. +type AssignIntStmt struct { + Value int64 + Target Local +} + +// AssignVarStmt represents an assignment of one local variable to another. +type AssignVarStmt struct { + Source Local + Target Local +} + +// AssignVarOnceStmt represents an assignment of one local variable to another. +// If the target is defined, execution aborts with a conflict error. +// +// TODO(tsandall): is there a better name for this? +type AssignVarOnceStmt struct { + Target Local + Source Local +} + +// MakeStringStmt constructs a local variable that refers to a string constant. +type MakeStringStmt struct { + Index int + Target Local +} + +// MakeNullStmt constructs a local variable that refers to a null value. +type MakeNullStmt struct { + Target Local +} + +// MakeBooleanStmt constructs a local variable that refers to a boolean value. +type MakeBooleanStmt struct { + Value bool + Target Local +} + +// MakeNumberFloatStmt constructs a local variable that refers to a +// floating-point number value. +type MakeNumberFloatStmt struct { + Value float64 + Target Local +} + +// MakeNumberIntStmt constructs a local variable that refers to an integer value. +type MakeNumberIntStmt struct { + Value int64 + Target Local +} + +// MakeNumberRefStmt constructs a local variable that refers to a number stored as a string. +type MakeNumberRefStmt struct { + Index int + Target Local +} + +// MakeArrayStmt constructs a local variable that refers to an array value. +type MakeArrayStmt struct { + Capacity int32 + Target Local +} + +// MakeObjectStmt constructs a local variable that refers to an object value. +type MakeObjectStmt struct { + Target Local +} + +// MakeSetStmt constructs a local variable that refers to a set value. +type MakeSetStmt struct { + Target Local +} + +// EqualStmt represents an value-equality check of two local variables. +type EqualStmt struct { + A Local + B Local +} + +// LessThanStmt represents a < check of two local variables. +type LessThanStmt struct { + A Local + B Local +} + +// LessThanEqualStmt represents a <= check of two local variables. +type LessThanEqualStmt struct { + A Local + B Local +} + +// GreaterThanStmt represents a > check of two local variables. +type GreaterThanStmt struct { + A Local + B Local +} + +// GreaterThanEqualStmt represents a >= check of two local variables. +type GreaterThanEqualStmt struct { + A Local + B Local +} + +// NotEqualStmt represents a != check of two local variables. +type NotEqualStmt struct { + A Local + B Local +} + +// IsArrayStmt represents a dynamic type check on a local variable. +type IsArrayStmt struct { + Source Local +} + +// IsObjectStmt represents a dynamic type check on a local variable. +type IsObjectStmt struct { + Source Local +} + +// IsDefinedStmt represents a check of whether a local variable is defined. +type IsDefinedStmt struct { + Source Local +} + +// IsUndefinedStmt represents a check of whether local variable is undefined. +type IsUndefinedStmt struct { + Source Local +} + +// ArrayAppendStmt represents a dynamic append operation of a value +// onto an array. +type ArrayAppendStmt struct { + Value Local + Array Local +} + +// ObjectInsertStmt represents a dynamic insert operation of a +// key/value pair into an object. +type ObjectInsertStmt struct { + Key Local + Value Local + Object Local +} + +// ObjectInsertOnceStmt represents a dynamic insert operation of a key/value +// pair into an object. If the key already exists and the value differs, +// execution aborts with a conflict error. +type ObjectInsertOnceStmt struct { + Key Local + Value Local + Object Local +} + +// ObjectMergeStmt performs a recursive merge of two object values. If either of +// the locals refer to non-object values this operation will abort with a +// conflict error. Overlapping object keys are merged recursively. +type ObjectMergeStmt struct { + A Local + B Local + Target Local +} + +// SetAddStmt represents a dynamic add operation of an element into a set. +type SetAddStmt struct { + Value Local + Set Local +} + +// WithStmt replaces the Local or a portion of the document referred to by the +// Local with the Value and executes the contained block. If the Path is +// non-empty, the Value is upserted into the Local. If the intermediate nodes in +// the Local referred to by the Path do not exist, they will be created. When +// the WithStmt finishes the Local is reset to it's original value. +type WithStmt struct { + Local Local + Path []int + Value Local + Block *Block +} + +// ResultSetAdd adds a value into the result set returned by the query plan. +type ResultSetAdd struct { + Value Local +} diff --git a/vendor/github.com/open-policy-agent/opa/internal/ir/pretty.go b/vendor/github.com/open-policy-agent/opa/internal/ir/pretty.go new file mode 100644 index 000000000..a23bad729 --- /dev/null +++ b/vendor/github.com/open-policy-agent/opa/internal/ir/pretty.go @@ -0,0 +1,44 @@ +// Copyright 2018 The OPA Authors. All rights reserved. +// Use of this source code is governed by an Apache2 +// license that can be found in the LICENSE file. + +package ir + +import ( + "fmt" + "io" + "strings" +) + +// Pretty writes a human-readable representation of an IR object to w. +func Pretty(w io.Writer, x interface{}) { + + pp := &prettyPrinter{ + depth: -1, + w: w, + } + Walk(pp, x) +} + +type prettyPrinter struct { + depth int + w io.Writer +} + +func (pp *prettyPrinter) Before(x interface{}) { + pp.depth++ +} + +func (pp *prettyPrinter) After(x interface{}) { + pp.depth-- +} + +func (pp *prettyPrinter) Visit(x interface{}) (Visitor, error) { + pp.writeIndent("%T %+v", x, x) + return pp, nil +} + +func (pp *prettyPrinter) writeIndent(f string, a ...interface{}) { + pad := strings.Repeat("| ", pp.depth) + fmt.Fprintf(pp.w, pad+f+"\n", a...) +} diff --git a/vendor/github.com/open-policy-agent/opa/internal/ir/walk.go b/vendor/github.com/open-policy-agent/opa/internal/ir/walk.go new file mode 100644 index 000000000..0e97099c5 --- /dev/null +++ b/vendor/github.com/open-policy-agent/opa/internal/ir/walk.go @@ -0,0 +1,84 @@ +// Copyright 2018 The OPA Authors. All rights reserved. +// Use of this source code is governed by an Apache2 +// license that can be found in the LICENSE file. + +package ir + +// Visitor defines the interface for visiting IR nodes. +type Visitor interface { + Before(x interface{}) + Visit(x interface{}) (Visitor, error) + After(x interface{}) +} + +// Walk invokes the visitor for nodes under x. +func Walk(vis Visitor, x interface{}) error { + impl := walkerImpl{ + vis: vis, + } + impl.walk(x) + return impl.err +} + +type walkerImpl struct { + vis Visitor + err error +} + +func (w *walkerImpl) walk(x interface{}) { + + if x == nil { + return + } + + prev := w.vis + w.vis.Before(x) + defer w.vis.After(x) + w.vis, w.err = w.vis.Visit(x) + if w.err != nil { + return + } else if w.vis == nil { + w.vis = prev + return + } + + switch x := x.(type) { + case *Policy: + w.walk(x.Static) + w.walk(x.Plan) + w.walk(x.Funcs) + case *Static: + for _, s := range x.Strings { + w.walk(s) + } + for _, f := range x.BuiltinFuncs { + w.walk(f) + } + case *Funcs: + for _, fn := range x.Funcs { + w.walk(fn) + } + case *Func: + for _, b := range x.Blocks { + w.walk(b) + } + case *Plan: + for _, b := range x.Blocks { + w.walk(b) + } + case *Block: + for _, s := range x.Stmts { + w.walk(s) + } + case *BlockStmt: + for _, b := range x.Blocks { + w.walk(b) + } + case *ScanStmt: + w.walk(x.Block) + case *NotStmt: + w.walk(x.Block) + case *WithStmt: + w.walk(x.Block) + } +} diff --git a/vendor/github.com/open-policy-agent/opa/internal/leb128/leb128.go b/vendor/github.com/open-policy-agent/opa/internal/leb128/leb128.go new file mode 100644 index 000000000..24ddc9095 --- /dev/null +++ b/vendor/github.com/open-policy-agent/opa/internal/leb128/leb128.go @@ -0,0 +1,170 @@ +// Copyright 2018 The OPA Authors. All rights reserved. +// Use of this source code is governed by an Apache2 +// license that can be found in the LICENSE file. + +// Package leb128 implements LEB128 integer encoding. +package leb128 + +import ( + "io" +) + +// MustReadVarInt32 returns an int32 from r or panics. +func MustReadVarInt32(r io.Reader) int32 { + i32, err := ReadVarInt32(r) + if err != nil { + panic(err) + } + return i32 +} + +// MustReadVarInt64 returns an int64 from r or panics. +func MustReadVarInt64(r io.Reader) int64 { + i64, err := ReadVarInt64(r) + if err != nil { + panic(err) + } + return i64 +} + +// MustReadVarUint32 returns an uint32 from r or panics. +func MustReadVarUint32(r io.Reader) uint32 { + u32, err := ReadVarUint32(r) + if err != nil { + panic(err) + } + return u32 +} + +// MustReadVarUint64 returns an uint64 from r or panics. +func MustReadVarUint64(r io.Reader) uint64 { + u64, err := ReadVarUint64(r) + if err != nil { + panic(err) + } + return u64 +} + +// Copied rom http://dwarfstd.org/doc/Dwarf3.pdf. + +// ReadVarUint32 tries to read a uint32 from r. +func ReadVarUint32(r io.Reader) (uint32, error) { + u64, err := ReadVarUint64(r) + if err != nil { + return 0, err + } + return uint32(u64), nil +} + +// ReadVarUint64 tries to read a uint64 from r. +func ReadVarUint64(r io.Reader) (uint64, error) { + var result uint64 + var shift uint64 + buf := make([]byte, 1) + for { + if _, err := r.Read(buf); err != nil { + return 0, err + } + v := uint64(buf[0]) + result |= (v & 0x7F) << shift + if v&0x80 == 0 { + return result, nil + } + shift += 7 + } + +} + +// ReadVarInt32 tries to read a int32 from r. +func ReadVarInt32(r io.Reader) (int32, error) { + i64, err := ReadVarInt64(r) + if err != nil { + return 0, err + } + return int32(i64), nil +} + +// ReadVarInt64 tries to read a int64 from r. +func ReadVarInt64(r io.Reader) (int64, error) { + var result int64 + var shift uint64 + size := uint64(32) + buf := make([]byte, 1) + for { + if _, err := r.Read(buf); err != nil { + return 0, err + } + v := int64(buf[0]) + result |= (v & 0x7F) << shift + shift += 7 + if v&0x80 == 0 { + if (shift < size) && (v&0x40 != 0) { + result |= (^0 << shift) + } + return result, nil + } + } +} + +// WriteVarUint32 writes u to w. +func WriteVarUint32(w io.Writer, u uint32) error { + var b []byte + _, err := w.Write(appendUleb128(b, uint64(u))) + return err +} + +// WriteVarUint64 writes u to w. +func WriteVarUint64(w io.Writer, u uint64) error { + var b []byte + _, err := w.Write(appendUleb128(b, u)) + return err +} + +// WriteVarInt32 writes u to w. +func WriteVarInt32(w io.Writer, i int32) error { + var b []byte + _, err := w.Write(appendSleb128(b, int64(i))) + return err +} + +// WriteVarInt64 writes u to w. +func WriteVarInt64(w io.Writer, i int64) error { + var b []byte + _, err := w.Write(appendSleb128(b, i)) + return err +} + +// Copied from https://github.com/golang/go/blob/master/src/cmd/internal/dwarf/dwarf.go. + +// appendUleb128 appends v to b using DWARF's unsigned LEB128 encoding. +func appendUleb128(b []byte, v uint64) []byte { + for { + c := uint8(v & 0x7f) + v >>= 7 + if v != 0 { + c |= 0x80 + } + b = append(b, c) + if c&0x80 == 0 { + break + } + } + return b +} + +// appendSleb128 appends v to b using DWARF's signed LEB128 encoding. +func appendSleb128(b []byte, v int64) []byte { + for { + c := uint8(v & 0x7f) + s := uint8(v & 0x40) + v >>= 7 + if (v != -1 || s == 0) && (v != 0 || s != 0) { + c |= 0x80 + } + b = append(b, c) + if c&0x80 == 0 { + break + } + } + return b +} diff --git a/vendor/github.com/open-policy-agent/opa/internal/merge/merge.go b/vendor/github.com/open-policy-agent/opa/internal/merge/merge.go new file mode 100644 index 000000000..fa53236d0 --- /dev/null +++ b/vendor/github.com/open-policy-agent/opa/internal/merge/merge.go @@ -0,0 +1,40 @@ +// Copyright 2017 The OPA Authors. All rights reserved. +// Use of this source code is governed by an Apache2 +// license that can be found in the LICENSE file. + +// Package merge contains helpers to merge data structures +// frequently encountered in OPA. +package merge + +// InterfaceMaps returns the result of merging a and b. If a and b cannot be +// merged because of conflicting key-value pairs, ok is false. +func InterfaceMaps(a map[string]interface{}, b map[string]interface{}) (c map[string]interface{}, ok bool) { + + c = map[string]interface{}{} + for k := range a { + c[k] = a[k] + } + + for k := range b { + + add := b[k] + exist, ok := c[k] + if !ok { + c[k] = add + continue + } + + existObj, existOk := exist.(map[string]interface{}) + addObj, addOk := add.(map[string]interface{}) + if !existOk || !addOk { + return nil, false + } + + c[k], ok = InterfaceMaps(existObj, addObj) + if !ok { + return nil, false + } + } + + return c, true +} diff --git a/vendor/github.com/open-policy-agent/opa/internal/planner/planner.go b/vendor/github.com/open-policy-agent/opa/internal/planner/planner.go new file mode 100644 index 000000000..de9bfeec7 --- /dev/null +++ b/vendor/github.com/open-policy-agent/opa/internal/planner/planner.go @@ -0,0 +1,1758 @@ +// Copyright 2018 The OPA Authors. All rights reserved. +// Use of this source code is governed by an Apache2 +// license that can be found in the LICENSE file. + +// Package planner contains a query planner for Rego queries. +package planner + +import ( + "errors" + "fmt" + "sort" + + "github.com/open-policy-agent/opa/ast" + "github.com/open-policy-agent/opa/internal/ir" +) + +type planiter func() error +type binaryiter func(ir.Local, ir.Local) error + +// Planner implements a query planner for Rego queries. +type Planner struct { + policy *ir.Policy // result of planning + queries []ast.Body // input query to plan + modules []*ast.Module // input modules to support queries + rewritten map[ast.Var]ast.Var // rewritten query vars + strings map[string]int // global string constant indices + externs map[string]struct{} // built-in functions that are required in execution environment + decls map[string]*ast.Builtin // built-in functions that may be provided in execution environment + rules *ruletrie // rules that may be planned + funcs *funcstack // functions that have been planned + curr *ir.Block // in-progress query block + vars *varstack // in-scope variables + ltarget ir.Local // target variable of last planned statement + lnext ir.Local // next variable to use +} + +// New returns a new Planner object. +func New() *Planner { + return &Planner{ + policy: &ir.Policy{ + Static: &ir.Static{}, + Plan: &ir.Plan{}, + Funcs: &ir.Funcs{}, + }, + strings: map[string]int{}, + externs: map[string]struct{}{}, + lnext: ir.Unused, + vars: newVarstack(map[ast.Var]ir.Local{ + ast.InputRootDocument.Value.(ast.Var): ir.Input, + ast.DefaultRootDocument.Value.(ast.Var): ir.Data, + }), + rules: newRuletrie(), + funcs: newFuncstack(), + } +} + +// WithBuiltinDecls tells the planner what built-in function may be available +// inside the execution environment. +func (p *Planner) WithBuiltinDecls(decls map[string]*ast.Builtin) *Planner { + p.decls = decls + return p +} + +// WithQueries sets the query set to generate a plan for. +func (p *Planner) WithQueries(queries []ast.Body) *Planner { + p.queries = queries + return p +} + +// WithModules sets the module set that contains query dependencies. +func (p *Planner) WithModules(modules []*ast.Module) *Planner { + p.modules = modules + return p +} + +// WithRewrittenVars sets a mapping of rewritten query vars on the planner. The +// plan will use the rewritten variable name but the result set key will be the +// original variable name. +func (p *Planner) WithRewrittenVars(vs map[ast.Var]ast.Var) *Planner { + p.rewritten = vs + return p +} + +// Plan returns a IR plan for the policy query. +func (p *Planner) Plan() (*ir.Policy, error) { + + if err := p.buildFunctrie(); err != nil { + return nil, err + } + + if err := p.planQueries(); err != nil { + return nil, err + } + + if err := p.planExterns(); err != nil { + return nil, err + } + + return p.policy, nil +} + +func (p *Planner) buildFunctrie() error { + + for _, module := range p.modules { + + // Create functrie node for empty packages so that extent queries return + // empty objects. For example: + // + // package x.y + // + // Query: data.x + // + // Expected result: {"y": {}} + if len(module.Rules) == 0 { + _ = p.rules.LookupOrInsert(module.Package.Path) + continue + } + + for _, rule := range module.Rules { + val := p.rules.LookupOrInsert(rule.Path()) + val.rules = append(val.rules, rule) + } + } + + return nil +} + +func (p *Planner) planRules(rules []*ast.Rule) (string, error) { + + path := rules[0].Path().String() + + if funcName, ok := p.funcs.Get(path); ok { + return funcName, nil + } + + // Save current state of planner. + // + // TODO(tsandall): perhaps we would be better off using stacks here or + // splitting block planner into separate struct that could be instantiated + // for rule and comprehension bodies. + pvars := p.vars + pcurr := p.curr + pltarget := p.ltarget + plnext := p.lnext + + // Reset the variable counter for the function plan. + p.lnext = ir.Input + + // Create function definition for rules. + fn := &ir.Func{ + Name: fmt.Sprintf("g%d.%s", p.funcs.gen, path), + Params: []ir.Local{ + p.newLocal(), // input document + p.newLocal(), // data document + }, + Return: p.newLocal(), + } + + // Initialize parameters for functions. + for i := 0; i < len(rules[0].Head.Args); i++ { + fn.Params = append(fn.Params, p.newLocal()) + } + + params := fn.Params[2:] + + // Initialize return value for partial set/object rules. Complete docs do + // not require their return value to be initialized. + if rules[0].Head.DocKind() == ast.PartialObjectDoc { + fn.Blocks = append(fn.Blocks, &ir.Block{ + Stmts: []ir.Stmt{ + &ir.MakeObjectStmt{ + Target: fn.Return, + }, + }, + }) + } else if rules[0].Head.DocKind() == ast.PartialSetDoc { + fn.Blocks = append(fn.Blocks, &ir.Block{ + Stmts: []ir.Stmt{ + &ir.MakeSetStmt{ + Target: fn.Return, + }, + }, + }) + } + + // At this point the locals for the params and return value have been + // allocated. This will be the first local that can be used in each block. + lnext := p.lnext + + var defaultRule *ast.Rule + + // Generate function blocks for rules. + for i := range rules { + + // Save default rule for the end. + if rules[i].Default { + defaultRule = rules[i] + continue + } + + // Ordered rules are nested inside an additional block so that execution + // can short-circuit. For unordered rules blocks can be added directly + // to the function. + var blocks *[]*ir.Block + + if rules[i].Else == nil { + blocks = &fn.Blocks + } else { + stmt := &ir.BlockStmt{} + block := &ir.Block{Stmts: []ir.Stmt{stmt}} + fn.Blocks = append(fn.Blocks, block) + blocks = &stmt.Blocks + } + + // Unordered rules are treated as a special case of ordered rules. + for rule := rules[i]; rule != nil; rule = rule.Else { + + // Setup planner for block. + p.lnext = lnext + p.vars = newVarstack(map[ast.Var]ir.Local{ + ast.InputRootDocument.Value.(ast.Var): fn.Params[0], + ast.DefaultRootDocument.Value.(ast.Var): fn.Params[1], + }) + + curr := &ir.Block{} + *blocks = append(*blocks, curr) + p.curr = curr + + // Complete and partial rules are treated as special cases of + // functions. If there are args, the first step is a no-op. + err := p.planFuncParams(params, rule.Head.Args, 0, func() error { + + // Run planner on the rule body. + err := p.planQuery(rule.Body, 0, func() error { + + // Run planner on the result. + switch rule.Head.DocKind() { + case ast.CompleteDoc: + return p.planTerm(rule.Head.Value, func() error { + p.appendStmt(&ir.AssignVarOnceStmt{ + Target: fn.Return, + Source: p.ltarget, + }) + return nil + }) + case ast.PartialSetDoc: + return p.planTerm(rule.Head.Key, func() error { + p.appendStmt(&ir.SetAddStmt{ + Set: fn.Return, + Value: p.ltarget, + }) + return nil + }) + case ast.PartialObjectDoc: + return p.planTerm(rule.Head.Key, func() error { + key := p.ltarget + return p.planTerm(rule.Head.Value, func() error { + value := p.ltarget + p.appendStmt(&ir.ObjectInsertOnceStmt{ + Object: fn.Return, + Key: key, + Value: value, + }) + return nil + }) + }) + default: + return fmt.Errorf("illegal rule kind") + } + }) + + if err != nil { + return err + } + + // Ordered rules are handled by short circuiting execution. The + // plan will jump out to the extra block that was planned above. + if rule.Else != nil { + p.appendStmt(&ir.IsDefinedStmt{Source: fn.Return}) + p.appendStmt(&ir.BreakStmt{Index: 1}) + } + + return nil + }) + + if err != nil { + return "", err + } + } + } + + // Default rules execute if the return is undefined. + if defaultRule != nil { + + fn.Blocks = append(fn.Blocks, &ir.Block{ + Stmts: []ir.Stmt{ + &ir.IsUndefinedStmt{Source: fn.Return}, + }, + }) + + p.curr = fn.Blocks[len(fn.Blocks)-1] + + err := p.planQuery(defaultRule.Body, 0, func() error { + return p.planTerm(defaultRule.Head.Value, func() error { + p.appendStmt(&ir.AssignVarOnceStmt{ + Target: fn.Return, + Source: p.ltarget, + }) + return nil + }) + }) + + if err != nil { + return "", err + } + } + + // All rules return a value. + fn.Blocks = append(fn.Blocks, &ir.Block{ + Stmts: []ir.Stmt{ + &ir.ReturnLocalStmt{ + Source: fn.Return, + }, + }, + }) + + p.appendFunc(fn) + p.funcs.Add(path, fn.Name) + + // Restore the state of the planner. + p.lnext = plnext + p.ltarget = pltarget + p.vars = pvars + p.curr = pcurr + + return fn.Name, nil +} + +func (p *Planner) planFuncParams(params []ir.Local, args ast.Args, idx int, iter planiter) error { + if idx >= len(args) { + return iter() + } + return p.planUnifyLocal(params[idx], args[idx], func() error { + return p.planFuncParams(params, args, idx+1, iter) + }) +} + +func (p *Planner) planQueries() error { + + // Initialize the plan with a block that prepares the query result. + p.curr = &ir.Block{} + + // Build a set of variables appearing in the query and allocate strings for + // each one. The strings will be used in the result set objects. + qvs := ast.NewVarSet() + + for _, q := range p.queries { + vs := q.Vars(ast.VarVisitorParams{SkipRefCallHead: true, SkipClosures: true}).Diff(ast.ReservedVars) + qvs.Update(vs) + } + + lvarnames := make(map[ast.Var]ir.Local, len(qvs)) + + for _, qv := range qvs.Sorted() { + qv = p.rewrittenVar(qv) + if !qv.IsGenerated() && !qv.IsWildcard() { + stmt := &ir.MakeStringStmt{ + Index: p.getStringConst(string(qv)), + Target: p.newLocal(), + } + p.appendStmt(stmt) + lvarnames[qv] = stmt.Target + } + } + + if len(p.curr.Stmts) > 0 { + p.appendBlock(p.curr) + } + + lnext := p.lnext + + for _, q := range p.queries { + p.lnext = lnext + p.vars.Push(map[ast.Var]ir.Local{}) + p.curr = &ir.Block{} + defined := false + qvs := q.Vars(ast.VarVisitorParams{SkipRefCallHead: true, SkipClosures: true}).Diff(ast.ReservedVars).Sorted() + + if err := p.planQuery(q, 0, func() error { + + // Add an object containing variable bindings into the result set. + lr := p.newLocal() + + p.appendStmt(&ir.MakeObjectStmt{ + Target: lr, + }) + + for _, qv := range qvs { + rw := p.rewrittenVar(qv) + if !rw.IsGenerated() && !rw.IsWildcard() { + p.appendStmt(&ir.ObjectInsertStmt{ + Object: lr, + Key: lvarnames[rw], + Value: p.vars.GetOrEmpty(qv), + }) + } + } + + p.appendStmt(&ir.ResultSetAdd{ + Value: lr, + }) + + defined = true + return nil + }); err != nil { + return err + } + + p.vars.Pop() + + if defined { + p.appendBlock(p.curr) + } + } + + return nil +} + +func (p *Planner) planQuery(q ast.Body, index int, iter planiter) error { + + if index >= len(q) { + return iter() + } + + return p.planExpr(q[index], func() error { + return p.planQuery(q, index+1, iter) + }) +} + +// TODO(tsandall): improve errors to include location information. +func (p *Planner) planExpr(e *ast.Expr, iter planiter) error { + if e.Negated { + return p.planNot(e, iter) + } + + if len(e.With) > 0 { + return p.planWith(e, iter) + } + + if e.IsCall() { + return p.planExprCall(e, iter) + } + + return p.planExprTerm(e, iter) +} + +func (p *Planner) planNot(e *ast.Expr, iter planiter) error { + + not := &ir.NotStmt{ + Block: &ir.Block{}, + } + + prev := p.curr + p.curr = not.Block + + if err := p.planExpr(e.Complement(), func() error { + return nil + }); err != nil { + return err + } + + p.curr = prev + p.appendStmt(not) + + return iter() +} + +func (p *Planner) planWith(e *ast.Expr, iter planiter) error { + + // Plan the values that will be applied by the with modifiers. All values + // must be defined for the overall expression to evaluate. + values := make([]*ast.Term, len(e.With)) + + for i := range e.With { + values[i] = e.With[i].Value + } + + return p.planTermSlice(values, func(locals []ir.Local) error { + + paths := make([][]int, len(e.With)) + saveVars := ast.NewVarSet() + dataRefs := []ast.Ref{} + + for i := range e.With { + + target := e.With[i].Target.Value.(ast.Ref) + paths[i] = make([]int, len(target)-1) + + for j := 1; j < len(target); j++ { + if s, ok := target[j].Value.(ast.String); ok { + paths[i][j-1] = p.getStringConst(string(s)) + } else { + return errors.New("invalid with target") + } + } + + head := target[0].Value.(ast.Var) + saveVars.Add(head) + + if head.Equal(ast.DefaultRootDocument.Value) { + dataRefs = append(dataRefs, target) + } + } + + restore := make([][2]ir.Local, len(saveVars)) + + for i, v := range saveVars.Sorted() { + lorig := p.vars.GetOrEmpty(v) + lsave := p.newLocal() + p.appendStmt(&ir.AssignVarStmt{Source: lorig, Target: lsave}) + restore[i] = [2]ir.Local{lorig, lsave} + } + + // If any of the with statements targeted the data document we shadow + // the existing planned functions during expression planning. This + // causes the planner to re-plan any rules that may be required during + // planning of this expression (transitively). + if len(dataRefs) > 0 { + p.funcs.Push(map[string]string{}) + for _, ref := range dataRefs { + p.rules.Push(ref) + } + } + + return p.planWithRec(e, paths, locals, 0, func() error { + + if len(dataRefs) > 0 { + p.funcs.Pop() + for i := len(dataRefs) - 1; i >= 0; i-- { + p.rules.Pop(dataRefs[i]) + } + } + + return p.planWithUndoRec(restore, 0, iter) + }) + }) +} + +func (p *Planner) planWithRec(e *ast.Expr, targets [][]int, values []ir.Local, index int, iter planiter) error { + + if index >= len(e.With) { + return p.planExpr(e.NoWith(), iter) + } + + prev := p.curr + p.curr = &ir.Block{} + + err := p.planWithRec(e, targets, values, index+1, iter) + if err != nil { + return err + } + + block := p.curr + p.curr = prev + target := e.With[index].Target.Value.(ast.Ref) + head := target[0].Value.(ast.Var) + + stmt := &ir.WithStmt{ + Local: p.vars.GetOrEmpty(head), + Path: targets[index], + Value: values[index], + Block: block, + } + + p.appendStmt(stmt) + + return nil +} + +func (p *Planner) planWithUndoRec(restore [][2]ir.Local, index int, iter planiter) error { + + if index >= len(restore) { + return iter() + } + + prev := p.curr + p.curr = &ir.Block{} + + if err := p.planWithUndoRec(restore, index+1, iter); err != nil { + return err + } + + block := p.curr + p.curr = prev + lorig := restore[index][0] + lsave := restore[index][1] + + p.appendStmt(&ir.WithStmt{ + Local: lorig, + Value: lsave, + Block: block, + }) + + return nil +} + +func (p *Planner) planExprTerm(e *ast.Expr, iter planiter) error { + return p.planTerm(e.Terms.(*ast.Term), func() error { + falsy := p.newLocal() + p.appendStmt(&ir.MakeBooleanStmt{ + Value: false, + Target: falsy, + }) + p.appendStmt(&ir.NotEqualStmt{ + A: p.ltarget, + B: falsy, + }) + return iter() + }) +} + +func (p *Planner) planExprCall(e *ast.Expr, iter planiter) error { + operator := e.Operator().String() + switch operator { + case ast.Equality.Name: + return p.planUnify(e.Operand(0), e.Operand(1), iter) + case ast.Equal.Name: + return p.planBinaryExpr(e, func(a, b ir.Local) error { + p.appendStmt(&ir.EqualStmt{ + A: a, + B: b, + }) + return iter() + }) + case ast.LessThan.Name: + return p.planBinaryExpr(e, func(a, b ir.Local) error { + p.appendStmt(&ir.LessThanStmt{ + A: a, + B: b, + }) + return iter() + }) + case ast.LessThanEq.Name: + return p.planBinaryExpr(e, func(a, b ir.Local) error { + p.appendStmt(&ir.LessThanEqualStmt{ + A: a, + B: b, + }) + return iter() + }) + case ast.GreaterThan.Name: + return p.planBinaryExpr(e, func(a, b ir.Local) error { + p.appendStmt(&ir.GreaterThanStmt{ + A: a, + B: b, + }) + return iter() + }) + case ast.GreaterThanEq.Name: + return p.planBinaryExpr(e, func(a, b ir.Local) error { + p.appendStmt(&ir.GreaterThanEqualStmt{ + A: a, + B: b, + }) + return iter() + }) + case ast.NotEqual.Name: + return p.planBinaryExpr(e, func(a, b ir.Local) error { + p.appendStmt(&ir.NotEqualStmt{ + A: a, + B: b, + }) + return iter() + }) + default: + + var name string + var arity int + var args []ir.Local + + node := p.rules.Lookup(e.Operator()) + + if node != nil { + var err error + name, err = p.planRules(node.Rules()) + if err != nil { + return err + } + arity = node.Arity() + args = []ir.Local{ + p.vars.GetOrEmpty(ast.InputRootDocument.Value.(ast.Var)), + p.vars.GetOrEmpty(ast.DefaultRootDocument.Value.(ast.Var)), + } + } else if decl, ok := p.decls[operator]; ok { + arity = len(decl.Decl.Args()) + name = operator + p.externs[operator] = struct{}{} + } else { + return fmt.Errorf("illegal call: unknown operator %q", operator) + } + + operands := e.Operands() + + if len(operands) == arity { + // rule: f(x) = x { ... } + // call: f(x) # result not captured + return p.planCallArgs(operands, 0, args, func(args []ir.Local) error { + p.ltarget = p.newLocal() + p.appendStmt(&ir.CallStmt{ + Func: name, + Args: args, + Result: p.ltarget, + }) + + falsy := p.newLocal() + + p.appendStmt(&ir.MakeBooleanStmt{ + Value: false, + Target: falsy, + }) + + p.appendStmt(&ir.NotEqualStmt{ + A: p.ltarget, + B: falsy, + }) + + return iter() + }) + } else if len(operands) == arity+1 { + // rule: f(x) = x { ... } + // call: f(x, 1) # caller captures result + return p.planCallArgs(operands[:len(operands)-1], 0, args, func(args []ir.Local) error { + result := p.newLocal() + p.appendStmt(&ir.CallStmt{ + Func: name, + Args: args, + Result: result, + }) + return p.planUnifyLocal(result, operands[len(operands)-1], iter) + }) + } + + return fmt.Errorf("illegal call: wrong number of operands: got %v, want %v)", len(operands), arity) + } +} + +func (p *Planner) planCallArgs(terms []*ast.Term, idx int, args []ir.Local, iter func([]ir.Local) error) error { + if idx >= len(terms) { + return iter(args) + } + return p.planTerm(terms[idx], func() error { + args = append(args, p.ltarget) + return p.planCallArgs(terms, idx+1, args, iter) + }) +} + +func (p *Planner) planUnify(a, b *ast.Term, iter planiter) error { + + switch va := a.Value.(type) { + case ast.Null, ast.Boolean, ast.Number, ast.String, ast.Ref, ast.Set, *ast.SetComprehension, *ast.ArrayComprehension, *ast.ObjectComprehension: + return p.planTerm(a, func() error { + return p.planUnifyLocal(p.ltarget, b, iter) + }) + case ast.Var: + return p.planUnifyVar(va, b, iter) + case ast.Array: + switch vb := b.Value.(type) { + case ast.Var: + return p.planUnifyVar(vb, a, iter) + case ast.Ref: + return p.planTerm(b, func() error { + return p.planUnifyLocalArray(p.ltarget, va, iter) + }) + case ast.Array: + if len(va) == len(vb) { + return p.planUnifyArraysRec(va, vb, 0, iter) + } + return nil + } + case ast.Object: + switch vb := b.Value.(type) { + case ast.Var: + return p.planUnifyVar(vb, a, iter) + case ast.Ref: + return p.planTerm(b, func() error { + return p.planUnifyLocalObject(p.ltarget, va, iter) + }) + case ast.Object: + if va.Len() == vb.Len() { + return p.planUnifyObjectsRec(va, vb, va.Keys(), 0, iter) + } + return nil + } + } + + return fmt.Errorf("not implemented: unify(%v, %v)", a, b) +} + +func (p *Planner) planUnifyVar(a ast.Var, b *ast.Term, iter planiter) error { + + if la, ok := p.vars.Get(a); ok { + return p.planUnifyLocal(la, b, iter) + } + + return p.planTerm(b, func() error { + target := p.newLocal() + p.vars.Put(a, target) + p.appendStmt(&ir.AssignVarStmt{ + Source: p.ltarget, + Target: target, + }) + return iter() + }) +} + +func (p *Planner) planUnifyLocal(a ir.Local, b *ast.Term, iter planiter) error { + switch vb := b.Value.(type) { + case ast.Null, ast.Boolean, ast.Number, ast.String, ast.Ref, ast.Set, *ast.SetComprehension, *ast.ArrayComprehension, *ast.ObjectComprehension: + return p.planTerm(b, func() error { + p.appendStmt(&ir.EqualStmt{ + A: a, + B: p.ltarget, + }) + return iter() + }) + case ast.Var: + if lv, ok := p.vars.Get(vb); ok { + p.appendStmt(&ir.EqualStmt{ + A: a, + B: lv, + }) + return iter() + } + lv := p.newLocal() + p.vars.Put(vb, lv) + p.appendStmt(&ir.AssignVarStmt{ + Source: a, + Target: lv, + }) + return iter() + case ast.Array: + return p.planUnifyLocalArray(a, vb, iter) + case ast.Object: + return p.planUnifyLocalObject(a, vb, iter) + } + + return fmt.Errorf("not implemented: unifyLocal(%v, %v)", a, b) +} + +func (p *Planner) planUnifyLocalArray(a ir.Local, b ast.Array, iter planiter) error { + p.appendStmt(&ir.IsArrayStmt{ + Source: a, + }) + + blen := p.newLocal() + alen := p.newLocal() + + p.appendStmt(&ir.LenStmt{ + Source: a, + Target: alen, + }) + + p.appendStmt(&ir.MakeNumberIntStmt{ + Value: int64(len(b)), + Target: blen, + }) + + p.appendStmt(&ir.EqualStmt{ + A: alen, + B: blen, + }) + + lkey := p.newLocal() + + p.appendStmt(&ir.MakeNumberIntStmt{ + Target: lkey, + }) + + lval := p.newLocal() + + return p.planUnifyLocalArrayRec(a, 0, b, lkey, lval, iter) +} + +func (p *Planner) planUnifyLocalArrayRec(a ir.Local, index int, b ast.Array, lkey, lval ir.Local, iter planiter) error { + if len(b) == index { + return iter() + } + + p.appendStmt(&ir.AssignIntStmt{ + Value: int64(index), + Target: lkey, + }) + + p.appendStmt(&ir.DotStmt{ + Source: a, + Key: lkey, + Target: lval, + }) + + return p.planUnifyLocal(lval, b[index], func() error { + return p.planUnifyLocalArrayRec(a, index+1, b, lkey, lval, iter) + }) +} + +func (p *Planner) planUnifyLocalObject(a ir.Local, b ast.Object, iter planiter) error { + p.appendStmt(&ir.IsObjectStmt{ + Source: a, + }) + + blen := p.newLocal() + alen := p.newLocal() + + p.appendStmt(&ir.LenStmt{ + Source: a, + Target: alen, + }) + + p.appendStmt(&ir.MakeNumberIntStmt{ + Value: int64(b.Len()), + Target: blen, + }) + + p.appendStmt(&ir.EqualStmt{ + A: alen, + B: blen, + }) + + lkey := p.newLocal() + lval := p.newLocal() + bkeys := b.Keys() + + return p.planUnifyLocalObjectRec(a, 0, bkeys, b, lkey, lval, iter) +} + +func (p *Planner) planUnifyLocalObjectRec(a ir.Local, index int, keys []*ast.Term, b ast.Object, lkey, lval ir.Local, iter planiter) error { + + if index == len(keys) { + return iter() + } + + return p.planTerm(keys[index], func() error { + p.appendStmt(&ir.AssignVarStmt{ + Source: p.ltarget, + Target: lkey, + }) + p.appendStmt(&ir.DotStmt{ + Source: a, + Key: lkey, + Target: lval, + }) + return p.planUnifyLocal(lval, b.Get(keys[index]), func() error { + return p.planUnifyLocalObjectRec(a, index+1, keys, b, lkey, lval, iter) + }) + }) +} + +func (p *Planner) planUnifyArraysRec(a, b ast.Array, index int, iter planiter) error { + if index == len(a) { + return iter() + } + return p.planUnify(a[index], b[index], func() error { + return p.planUnifyArraysRec(a, b, index+1, iter) + }) +} + +func (p *Planner) planUnifyObjectsRec(a, b ast.Object, keys []*ast.Term, index int, iter planiter) error { + if index == len(keys) { + return iter() + } + + aval := a.Get(keys[index]) + bval := b.Get(keys[index]) + if aval == nil || bval == nil { + return nil + } + + return p.planUnify(aval, bval, func() error { + return p.planUnifyObjectsRec(a, b, keys, index+1, iter) + }) +} + +func (p *Planner) planBinaryExpr(e *ast.Expr, iter binaryiter) error { + return p.planTerm(e.Operand(0), func() error { + a := p.ltarget + return p.planTerm(e.Operand(1), func() error { + b := p.ltarget + return iter(a, b) + }) + }) +} + +func (p *Planner) planTerm(t *ast.Term, iter planiter) error { + + switch v := t.Value.(type) { + case ast.Null: + return p.planNull(v, iter) + case ast.Boolean: + return p.planBoolean(v, iter) + case ast.Number: + return p.planNumber(v, iter) + case ast.String: + return p.planString(v, iter) + case ast.Var: + return p.planVar(v, iter) + case ast.Ref: + return p.planRef(v, iter) + case ast.Array: + return p.planArray(v, iter) + case ast.Object: + return p.planObject(v, iter) + case ast.Set: + return p.planSet(v, iter) + case *ast.SetComprehension: + return p.planSetComprehension(v, iter) + case *ast.ArrayComprehension: + return p.planArrayComprehension(v, iter) + case *ast.ObjectComprehension: + return p.planObjectComprehension(v, iter) + default: + return fmt.Errorf("%v term not implemented", ast.TypeName(v)) + } +} + +func (p *Planner) planNull(null ast.Null, iter planiter) error { + + target := p.newLocal() + + p.appendStmt(&ir.MakeNullStmt{ + Target: target, + }) + + p.ltarget = target + + return iter() +} + +func (p *Planner) planBoolean(b ast.Boolean, iter planiter) error { + + target := p.newLocal() + + p.appendStmt(&ir.MakeBooleanStmt{ + Value: bool(b), + Target: target, + }) + + p.ltarget = target + + return iter() +} + +func (p *Planner) planNumber(num ast.Number, iter planiter) error { + + index := p.getStringConst(string(num)) + target := p.newLocal() + + p.appendStmt(&ir.MakeNumberRefStmt{ + Index: index, + Target: target, + }) + + p.ltarget = target + return iter() +} + +func (p *Planner) planNumberFloat(f float64, iter planiter) error { + + target := p.newLocal() + + p.appendStmt(&ir.MakeNumberFloatStmt{ + Value: f, + Target: target, + }) + + p.ltarget = target + + return iter() +} + +func (p *Planner) planNumberInt(i int64, iter planiter) error { + + target := p.newLocal() + + p.appendStmt(&ir.MakeNumberIntStmt{ + Value: i, + Target: target, + }) + + p.ltarget = target + + return iter() +} + +func (p *Planner) planString(str ast.String, iter planiter) error { + + index := p.getStringConst(string(str)) + target := p.newLocal() + + p.appendStmt(&ir.MakeStringStmt{ + Index: index, + Target: target, + }) + + p.ltarget = target + + return iter() +} + +func (p *Planner) planVar(v ast.Var, iter planiter) error { + p.ltarget = p.vars.GetOrElse(v, func() ir.Local { + return p.newLocal() + }) + return iter() +} + +func (p *Planner) planArray(arr ast.Array, iter planiter) error { + + larr := p.newLocal() + + p.appendStmt(&ir.MakeArrayStmt{ + Capacity: int32(len(arr)), + Target: larr, + }) + + return p.planArrayRec(arr, 0, larr, iter) +} + +func (p *Planner) planArrayRec(arr ast.Array, index int, larr ir.Local, iter planiter) error { + if index == len(arr) { + p.ltarget = larr + return iter() + } + + return p.planTerm(arr[index], func() error { + + p.appendStmt(&ir.ArrayAppendStmt{ + Value: p.ltarget, + Array: larr, + }) + + return p.planArrayRec(arr, index+1, larr, iter) + }) +} + +func (p *Planner) planObject(obj ast.Object, iter planiter) error { + + lobj := p.newLocal() + + p.appendStmt(&ir.MakeObjectStmt{ + Target: lobj, + }) + + return p.planObjectRec(obj, 0, obj.Keys(), lobj, iter) +} + +func (p *Planner) planObjectRec(obj ast.Object, index int, keys []*ast.Term, lobj ir.Local, iter planiter) error { + if index == len(keys) { + p.ltarget = lobj + return iter() + } + + return p.planTerm(keys[index], func() error { + lkey := p.ltarget + + return p.planTerm(obj.Get(keys[index]), func() error { + lval := p.ltarget + p.appendStmt(&ir.ObjectInsertStmt{ + Key: lkey, + Value: lval, + Object: lobj, + }) + + return p.planObjectRec(obj, index+1, keys, lobj, iter) + }) + }) +} + +func (p *Planner) planSet(set ast.Set, iter planiter) error { + lset := p.newLocal() + + p.appendStmt(&ir.MakeSetStmt{ + Target: lset, + }) + + return p.planSetRec(set, 0, set.Slice(), lset, iter) +} + +func (p *Planner) planSetRec(set ast.Set, index int, elems []*ast.Term, lset ir.Local, iter planiter) error { + if index == len(elems) { + p.ltarget = lset + return iter() + } + + return p.planTerm(elems[index], func() error { + p.appendStmt(&ir.SetAddStmt{ + Value: p.ltarget, + Set: lset, + }) + return p.planSetRec(set, index+1, elems, lset, iter) + }) +} + +func (p *Planner) planSetComprehension(sc *ast.SetComprehension, iter planiter) error { + + lset := p.newLocal() + + p.appendStmt(&ir.MakeSetStmt{ + Target: lset, + }) + + return p.planComprehension(sc.Body, func() error { + return p.planTerm(sc.Term, func() error { + p.appendStmt(&ir.SetAddStmt{ + Value: p.ltarget, + Set: lset, + }) + return nil + }) + }, lset, iter) +} + +func (p *Planner) planArrayComprehension(ac *ast.ArrayComprehension, iter planiter) error { + + larr := p.newLocal() + + p.appendStmt(&ir.MakeArrayStmt{ + Target: larr, + }) + + return p.planComprehension(ac.Body, func() error { + return p.planTerm(ac.Term, func() error { + p.appendStmt(&ir.ArrayAppendStmt{ + Value: p.ltarget, + Array: larr, + }) + return nil + }) + }, larr, iter) +} + +func (p *Planner) planObjectComprehension(oc *ast.ObjectComprehension, iter planiter) error { + + lobj := p.newLocal() + + p.appendStmt(&ir.MakeObjectStmt{ + Target: lobj, + }) + + return p.planComprehension(oc.Body, func() error { + return p.planTerm(oc.Key, func() error { + lkey := p.ltarget + return p.planTerm(oc.Value, func() error { + p.appendStmt(&ir.ObjectInsertOnceStmt{ + Key: lkey, + Value: p.ltarget, + Object: lobj, + }) + return nil + }) + }) + }, lobj, iter) +} + +func (p *Planner) planComprehension(body ast.Body, closureIter planiter, target ir.Local, iter planiter) error { + + prev := p.curr + p.curr = &ir.Block{} + + if err := p.planQuery(body, 0, func() error { + return closureIter() + }); err != nil { + return err + } + + block := p.curr + p.curr = prev + + p.appendStmt(&ir.BlockStmt{ + Blocks: []*ir.Block{ + block, + }, + }) + + p.ltarget = target + return iter() +} + +func (p *Planner) planRef(ref ast.Ref, iter planiter) error { + + head, ok := ref[0].Value.(ast.Var) + if !ok { + return fmt.Errorf("illegal ref: non-var head") + } + + if head.Compare(ast.DefaultRootDocument.Value) == 0 { + virtual := p.rules.Get(ref[0].Value) + base := &baseptr{local: p.vars.GetOrEmpty(ast.DefaultRootDocument.Value.(ast.Var))} + return p.planRefData(virtual, base, ref, 1, iter) + } + + p.ltarget, ok = p.vars.Get(head) + if !ok { + return fmt.Errorf("illegal ref: unsafe head") + } + + return p.planRefRec(ref, 1, iter) +} + +func (p *Planner) planRefRec(ref ast.Ref, index int, iter planiter) error { + + if len(ref) == index { + return iter() + } + + scan := false + + ast.WalkVars(ref[index], func(v ast.Var) bool { + if !scan { + _, exists := p.vars.Get(v) + if !exists { + scan = true + } + } + return scan + }) + + if !scan { + return p.planDot(ref[index], func() error { + return p.planRefRec(ref, index+1, iter) + }) + } + + return p.planScan(ref[index], func(lkey ir.Local) error { + return p.planRefRec(ref, index+1, iter) + }) +} + +type baseptr struct { + local ir.Local + path ast.Ref +} + +// planRefData implements the virtual document model by generating the value of +// the ref parameter and invoking the iterator with the planner target set to +// the virtual document and all variables in the reference assigned. +func (p *Planner) planRefData(virtual *ruletrie, base *baseptr, ref ast.Ref, index int, iter planiter) error { + + // Early-exit if the end of the reference has been reached. In this case the + // plan has to materialize the full extent of the referenced value. + if index >= len(ref) { + return p.planRefDataExtent(virtual, base, iter) + } + + // If the reference operand is ground then either continue to the next + // operand or invoke the function for the rule referred to by this operand. + if ref[index].IsGround() { + + var vchild *ruletrie + + if virtual != nil { + vchild = virtual.Get(ref[index].Value) + } + + rules := vchild.Rules() + + if len(rules) > 0 { + p.ltarget = p.newLocal() + + funcName, err := p.planRules(rules) + if err != nil { + return err + } + + p.appendStmt(&ir.CallStmt{ + Func: funcName, + Args: []ir.Local{ + p.vars.GetOrEmpty(ast.InputRootDocument.Value.(ast.Var)), + p.vars.GetOrEmpty(ast.DefaultRootDocument.Value.(ast.Var)), + }, + Result: p.ltarget, + }) + + return p.planRefRec(ref, index+1, iter) + } + + bchild := *base + bchild.path = append(bchild.path, ref[index]) + + return p.planRefData(vchild, &bchild, ref, index+1, iter) + } + + exclude := ast.NewSet() + + // The planner does not support dynamic dispatch so generate blocks to + // evaluate each of the rulesets on the child nodes. + if virtual != nil { + + stmt := &ir.BlockStmt{} + + for _, child := range virtual.Children() { + + block := &ir.Block{} + prev := p.curr + p.curr = block + key := ast.NewTerm(child) + exclude.Add(key) + + // Assignments in each block due to local unification must be undone + // so create a new frame that will be popped after this key is + // processed. + p.vars.Push(map[ast.Var]ir.Local{}) + + if err := p.planTerm(key, func() error { + return p.planUnifyLocal(p.ltarget, ref[index], func() error { + // Create a copy of the reference with this operand plugged. + // This will result in evaluation of the rulesets on the + // child node. + cpy := ref.Copy() + cpy[index] = key + return p.planRefData(virtual, base, cpy, index, iter) + }) + }); err != nil { + return err + } + + p.vars.Pop() + p.curr = prev + stmt.Blocks = append(stmt.Blocks, block) + } + + p.appendStmt(stmt) + } + + // If the virtual tree was enumerated then we do not want to enumerate base + // trees that are rooted at the same key as any of the virtual sub trees. To + // prevent this we build a set of keys that are to be excluded and check + // below during the base scan. + var lexclude *ir.Local + + if exclude.Len() > 0 { + if err := p.planSet(exclude, func() error { + v := p.ltarget + lexclude = &v + return nil + }); err != nil { + return err + } + } + + p.ltarget = base.local + + // Perform a scan of the base documents starting from the location referred + // to by the data pointer. Use the set we built above to avoid revisiting + // sub trees. + return p.planRefRec(base.path, 0, func() error { + return p.planScan(ref[index], func(lkey ir.Local) error { + if lexclude != nil { + lignore := p.newLocal() + p.appendStmt(&ir.NotStmt{ + Block: &ir.Block{ + Stmts: []ir.Stmt{ + &ir.DotStmt{ + Source: *lexclude, + Key: lkey, + Target: lignore, + }, + }, + }, + }) + } + + // Assume that virtual sub trees have been visited already so + // recurse without the virtual node. + return p.planRefData(nil, &baseptr{local: p.ltarget}, ref, index+1, iter) + }) + }) +} + +// planRefDataExtent generates the full extent (combined) of the base and +// virtual nodes and then invokes the iterator with the planner target set to +// the full extent. +func (p *Planner) planRefDataExtent(virtual *ruletrie, base *baseptr, iter planiter) error { + + vtarget := p.newLocal() + + // Generate the virtual document out of rules contained under the virtual + // node (recursively). This document will _ONLY_ contain values generated by + // rules. No base document values will be included. + if virtual != nil { + + p.appendStmt(&ir.MakeObjectStmt{ + Target: vtarget, + }) + + for _, key := range virtual.Children() { + child := virtual.Get(key) + + // Skip functions. + if child.Arity() > 0 { + continue + } + + lkey := p.newLocal() + idx := p.getStringConst(string(key.(ast.String))) + p.appendStmt(&ir.MakeStringStmt{ + Index: idx, + Target: lkey, + }) + + rules := child.Rules() + + // Build object hierarchy depth-first. + if len(rules) == 0 { + err := p.planRefDataExtent(child, nil, func() error { + p.appendStmt(&ir.ObjectInsertStmt{ + Object: vtarget, + Key: lkey, + Value: p.ltarget, + }) + return nil + }) + if err != nil { + return err + } + continue + } + + // Generate virtual document for leaf. + lvalue := p.newLocal() + + funcName, err := p.planRules(rules) + if err != nil { + return err + } + + // Add leaf to object if defined. + p.appendStmt(&ir.BlockStmt{ + Blocks: []*ir.Block{ + &ir.Block{ + Stmts: []ir.Stmt{ + &ir.CallStmt{ + Func: funcName, + Args: []ir.Local{ + p.vars.GetOrEmpty(ast.InputRootDocument.Value.(ast.Var)), + p.vars.GetOrEmpty(ast.DefaultRootDocument.Value.(ast.Var)), + }, + Result: lvalue, + }, + &ir.ObjectInsertStmt{ + Object: vtarget, + Key: lkey, + Value: lvalue, + }, + }, + }, + }, + }) + } + + // At this point vtarget refers to the full extent of the virtual + // document at ref. If the base pointer is unset, no further processing + // is required. + if base == nil { + p.ltarget = vtarget + return iter() + } + } + + // Obtain the base document value and merge (recursively) with the virtual + // document value above if needed. + prev := p.curr + p.curr = &ir.Block{} + p.ltarget = base.local + target := p.newLocal() + + err := p.planRefRec(base.path, 0, func() error { + + if virtual == nil { + target = p.ltarget + } else { + stmt := &ir.ObjectMergeStmt{ + A: p.ltarget, + B: vtarget, + Target: target, + } + p.appendStmt(stmt) + } + + p.appendStmt(&ir.BreakStmt{Index: 1}) + return nil + }) + + if err != nil { + return err + } + + inner := p.curr + + // Fallback to virtual document value if base document is undefined. + // Otherwise, this block is undefined. + p.curr = &ir.Block{} + p.appendStmt(&ir.BlockStmt{Blocks: []*ir.Block{inner}}) + + if virtual != nil { + p.appendStmt(&ir.AssignVarStmt{ + Source: vtarget, + Target: target, + }) + } else { + p.appendStmt(&ir.BreakStmt{Index: 1}) + } + + outer := p.curr + p.curr = prev + p.appendStmt(&ir.BlockStmt{Blocks: []*ir.Block{outer}}) + + // At this point, target refers to either the full extent of the base and + // virtual documents at ref or just the base document at ref. + p.ltarget = target + + return iter() +} + +func (p *Planner) planDot(key *ast.Term, iter planiter) error { + + source := p.ltarget + + return p.planTerm(key, func() error { + + target := p.newLocal() + + p.appendStmt(&ir.DotStmt{ + Source: source, + Key: p.ltarget, + Target: target, + }) + + p.ltarget = target + + return iter() + }) +} + +type scaniter func(ir.Local) error + +func (p *Planner) planScan(key *ast.Term, iter scaniter) error { + + scan := &ir.ScanStmt{ + Source: p.ltarget, + Key: p.newLocal(), + Value: p.newLocal(), + Block: &ir.Block{}, + } + + prev := p.curr + p.curr = scan.Block + + if err := p.planUnifyLocal(scan.Key, key, func() error { + p.ltarget = scan.Value + return iter(scan.Key) + }); err != nil { + return err + } + + p.curr = prev + p.appendStmt(scan) + + return nil + +} + +// planSaveLocals returns a slice of locals holding temporary variables that +// have been assigned from the supplied vars. +func (p *Planner) planSaveLocals(vars ...ir.Local) []ir.Local { + + lsaved := make([]ir.Local, len(vars)) + + for i := range vars { + + lsaved[i] = p.newLocal() + + p.appendStmt(&ir.AssignVarStmt{ + Source: vars[i], + Target: lsaved[i], + }) + } + + return lsaved +} + +type termsliceiter func([]ir.Local) error + +func (p *Planner) planTermSlice(terms []*ast.Term, iter termsliceiter) error { + return p.planTermSliceRec(terms, make([]ir.Local, len(terms)), 0, iter) +} + +func (p *Planner) planTermSliceRec(terms []*ast.Term, locals []ir.Local, index int, iter termsliceiter) error { + if index >= len(terms) { + return iter(locals) + } + + return p.planTerm(terms[index], func() error { + locals[index] = p.ltarget + return p.planTermSliceRec(terms, locals, index+1, iter) + }) +} + +func (p *Planner) planExterns() error { + + p.policy.Static.BuiltinFuncs = make([]*ir.BuiltinFunc, 0, len(p.externs)) + + for name := range p.externs { + p.policy.Static.BuiltinFuncs = append(p.policy.Static.BuiltinFuncs, &ir.BuiltinFunc{Name: name}) + } + + sort.Slice(p.policy.Static.BuiltinFuncs, func(i, j int) bool { + return p.policy.Static.BuiltinFuncs[i].Name < p.policy.Static.BuiltinFuncs[j].Name + }) + + return nil +} + +func (p *Planner) getStringConst(s string) int { + index, ok := p.strings[s] + if !ok { + index = len(p.policy.Static.Strings) + p.policy.Static.Strings = append(p.policy.Static.Strings, &ir.StringConst{ + Value: s, + }) + p.strings[s] = index + } + return index +} + +func (p *Planner) appendStmt(s ir.Stmt) { + p.curr.Stmts = append(p.curr.Stmts, s) +} + +func (p *Planner) appendFunc(f *ir.Func) { + p.policy.Funcs.Funcs = append(p.policy.Funcs.Funcs, f) +} + +func (p *Planner) appendBlock(b *ir.Block) { + p.policy.Plan.Blocks = append(p.policy.Plan.Blocks, b) +} + +func (p *Planner) newLocal() ir.Local { + x := p.lnext + p.lnext++ + return x +} + +func (p *Planner) rewrittenVar(k ast.Var) ast.Var { + rw, ok := p.rewritten[k] + if !ok { + return k + } + return rw +} diff --git a/vendor/github.com/open-policy-agent/opa/internal/planner/rules.go b/vendor/github.com/open-policy-agent/opa/internal/planner/rules.go new file mode 100644 index 000000000..e2d9d1561 --- /dev/null +++ b/vendor/github.com/open-policy-agent/opa/internal/planner/rules.go @@ -0,0 +1,156 @@ +package planner + +import ( + "sort" + + "github.com/open-policy-agent/opa/ast" +) + +// funcstack implements a simple map structure used to keep track of virtual +// document => planned function names. The structure supports Push and Pop +// operations so that the planner can shadow planned functions when 'with' +// statements are found. +type funcstack struct { + stack []map[string]string + gen int +} + +func newFuncstack() *funcstack { + return &funcstack{ + stack: []map[string]string{ + map[string]string{}, + }, + gen: 0, + } +} + +func (p funcstack) Add(key, value string) { + p.stack[len(p.stack)-1][key] = value +} + +func (p funcstack) Get(key string) (string, bool) { + value, ok := p.stack[len(p.stack)-1][key] + return value, ok +} + +func (p *funcstack) Push(funcs map[string]string) { + p.stack = append(p.stack, funcs) + p.gen++ +} + +func (p *funcstack) Pop() map[string]string { + last := p.stack[len(p.stack)-1] + p.stack = p.stack[:len(p.stack)-1] + p.gen++ + return last +} + +// ruletrie implements a simple trie structure for organizing rules that may be +// planned. The trie nodes are keyed by the rule path. The ruletrie supports +// Push and Pop operations that allow the planner to shadow subtrees when 'with' +// statements are found. +type ruletrie struct { + children map[ast.Value][]*ruletrie + rules []*ast.Rule +} + +func newRuletrie() *ruletrie { + return &ruletrie{ + children: map[ast.Value][]*ruletrie{}, + } +} + +func (t *ruletrie) Arity() int { + rules := t.Rules() + if len(rules) > 0 { + return len(rules[0].Head.Args) + } + return 0 +} + +func (t *ruletrie) Rules() []*ast.Rule { + if t != nil { + return t.rules + } + return nil +} + +func (t *ruletrie) Push(key ast.Ref) { + node := t + for i := 0; i < len(key)-1; i++ { + node = node.Get(key[i].Value) + if node == nil { + return + } + } + elem := key[len(key)-1] + node.children[elem.Value] = append(node.children[elem.Value], nil) +} + +func (t *ruletrie) Pop(key ast.Ref) { + node := t + for i := 0; i < len(key)-1; i++ { + node = node.Get(key[i].Value) + if node == nil { + return + } + } + elem := key[len(key)-1] + sl := node.children[elem.Value] + node.children[elem.Value] = sl[:len(sl)-1] +} + +func (t *ruletrie) Insert(key ast.Ref) *ruletrie { + node := t + for _, elem := range key { + child := node.Get(elem.Value) + if child == nil { + child = newRuletrie() + node.children[elem.Value] = append(node.children[elem.Value], child) + } + node = child + } + return node +} + +func (t *ruletrie) Lookup(key ast.Ref) *ruletrie { + node := t + for _, elem := range key { + node = node.Get(elem.Value) + if node == nil { + return nil + } + } + return node +} + +func (t *ruletrie) LookupOrInsert(key ast.Ref) *ruletrie { + if val := t.Lookup(key); val != nil { + return val + } + return t.Insert(key) +} + +func (t *ruletrie) Children() []ast.Value { + sorted := make([]ast.Value, 0, len(t.children)) + for key := range t.children { + if t.Get(key) != nil { + sorted = append(sorted, key) + } + } + sort.Slice(sorted, func(i, j int) bool { + return sorted[i].Compare(sorted[j]) < 0 + }) + return sorted +} + +func (t *ruletrie) Get(k ast.Value) *ruletrie { + if t == nil { + return nil + } + nodes := t.children[k] + if len(nodes) == 0 { + return nil + } + return nodes[len(nodes)-1] +} diff --git a/vendor/github.com/open-policy-agent/opa/internal/planner/varstack.go b/vendor/github.com/open-policy-agent/opa/internal/planner/varstack.go new file mode 100644 index 000000000..7dbf8f43c --- /dev/null +++ b/vendor/github.com/open-policy-agent/opa/internal/planner/varstack.go @@ -0,0 +1,58 @@ +// Copyright 2019 The OPA Authors. All rights reserved. +// Use of this source code is governed by an Apache2 +// license that can be found in the LICENSE file. + +package planner + +import ( + "github.com/open-policy-agent/opa/ast" + "github.com/open-policy-agent/opa/internal/ir" +) + +type varstack []map[ast.Var]ir.Local + +func newVarstack(frames ...map[ast.Var]ir.Local) *varstack { + vs := &varstack{} + for _, f := range frames { + vs.Push(f) + } + return vs +} + +func (vs varstack) GetOrElse(k ast.Var, orElse func() ir.Local) ir.Local { + l, ok := vs.Get(k) + if !ok { + l = orElse() + vs.Put(k, l) + } + return l +} + +func (vs varstack) GetOrEmpty(k ast.Var) ir.Local { + l, _ := vs.Get(k) + return l +} + +func (vs varstack) Get(k ast.Var) (ir.Local, bool) { + for i := len(vs) - 1; i >= 0; i-- { + if l, ok := vs[i][k]; ok { + return l, true + } + } + return 0, false +} + +func (vs varstack) Put(k ast.Var, v ir.Local) { + vs[len(vs)-1][k] = v +} + +func (vs *varstack) Push(frame map[ast.Var]ir.Local) { + *vs = append(*vs, frame) +} + +func (vs *varstack) Pop() map[ast.Var]ir.Local { + sl := *vs + last := sl[len(sl)-1] + *vs = sl[:len(sl)-1] + return last +} diff --git a/vendor/github.com/open-policy-agent/opa/internal/version/version.go b/vendor/github.com/open-policy-agent/opa/internal/version/version.go new file mode 100644 index 000000000..02f1c1b88 --- /dev/null +++ b/vendor/github.com/open-policy-agent/opa/internal/version/version.go @@ -0,0 +1,40 @@ +// Copyright 2019 The OPA Authors. All rights reserved. +// Use of this source code is governed by an Apache2 +// license that can be found in the LICENSE file. + +// Package version implements helper functions for the stored version. +package version + +import ( + "context" + "fmt" + "runtime" + + "github.com/open-policy-agent/opa/storage" + "github.com/open-policy-agent/opa/version" +) + +var versionPath = storage.MustParsePath("/system/version") + +// Write the build version information into storage. This makes the +// version information available to the REPL and the HTTP server. +func Write(ctx context.Context, store storage.Store, txn storage.Transaction) error { + + if err := storage.MakeDir(ctx, store, txn, versionPath); err != nil { + return err + } + + if err := store.Write(ctx, txn, storage.AddOp, versionPath, map[string]interface{}{ + "version": version.Version, + "build_commit": version.Vcs, + "build_timestamp": version.Timestamp, + "build_hostname": version.Hostname, + }); err != nil { + return err + } + + return nil +} + +// UserAgent defines the current OPA instances User-Agent default header value. +var UserAgent = fmt.Sprintf("Open Policy Agent/%s (%s, %s)", version.Version, runtime.GOOS, runtime.GOARCH) diff --git a/vendor/github.com/open-policy-agent/opa/internal/wasm/constant/constant.go b/vendor/github.com/open-policy-agent/opa/internal/wasm/constant/constant.go new file mode 100644 index 000000000..84e4d4746 --- /dev/null +++ b/vendor/github.com/open-policy-agent/opa/internal/wasm/constant/constant.go @@ -0,0 +1,67 @@ +// Copyright 2018 The OPA Authors. All rights reserved. +// Use of this source code is governed by an Apache2 +// license that can be found in the LICENSE file. + +// Package constant contains WASM constant definitions. +package constant + +// Magic bytes at the beginning of every WASM file ("\0asm"). +const Magic = uint32(0x6D736100) + +// Version defines the WASM version. +const Version = uint32(1) + +// WASM module section IDs. +const ( + CustomSectionID uint8 = iota + TypeSectionID + ImportSectionID + FunctionSectionID + TableSectionID + MemorySectionID + GlobalSectionID + ExportSectionID + StartSectionID + ElementSectionID + CodeSectionID + DataSectionID +) + +// FunctionTypeID indicates the start of a function type definition. +const FunctionTypeID = byte(0x60) + +// ValueType represents an intrinsic value type in WASM. +const ( + ValueTypeF64 byte = iota + 0x7C + ValueTypeF32 + ValueTypeI64 + ValueTypeI32 +) + +// WASM import descriptor types. +const ( + ImportDescType byte = iota + ImportDescTable + ImportDescMem + ImportDescGlobal +) + +// WASM export descriptor types. +const ( + ExportDescType byte = iota + ExportDescTable + ExportDescMem + ExportDescGlobal +) + +// ElementTypeAnyFunc indicates the type of a table import. +const ElementTypeAnyFunc byte = 0x70 + +// BlockTypeEmpty represents a block type. +const BlockTypeEmpty byte = 0x40 + +// WASM global varialbe mutability flag. +const ( + Const byte = iota + Mutable +) diff --git a/vendor/github.com/open-policy-agent/opa/internal/wasm/encoding/doc.go b/vendor/github.com/open-policy-agent/opa/internal/wasm/encoding/doc.go new file mode 100644 index 000000000..b25236968 --- /dev/null +++ b/vendor/github.com/open-policy-agent/opa/internal/wasm/encoding/doc.go @@ -0,0 +1,6 @@ +// Copyright 2018 The OPA Authors. All rights reserved. +// Use of this source code is governed by an Apache2 +// license that can be found in the LICENSE file. + +// Package encoding implements WASM module reading and writing. +package encoding diff --git a/vendor/github.com/open-policy-agent/opa/internal/wasm/encoding/reader.go b/vendor/github.com/open-policy-agent/opa/internal/wasm/encoding/reader.go new file mode 100644 index 000000000..9ecb8a43f --- /dev/null +++ b/vendor/github.com/open-policy-agent/opa/internal/wasm/encoding/reader.go @@ -0,0 +1,809 @@ +// Copyright 2018 The OPA Authors. All rights reserved. +// Use of this source code is governed by an Apache2 +// license that can be found in the LICENSE file. + +package encoding + +import ( + "bytes" + "encoding/binary" + "fmt" + "io" + + "github.com/pkg/errors" + + "github.com/open-policy-agent/opa/internal/leb128" + "github.com/open-policy-agent/opa/internal/wasm/constant" + "github.com/open-policy-agent/opa/internal/wasm/instruction" + "github.com/open-policy-agent/opa/internal/wasm/module" + "github.com/open-policy-agent/opa/internal/wasm/opcode" + "github.com/open-policy-agent/opa/internal/wasm/types" +) + +// ReadModule reads a binary-encoded WASM module from r. +func ReadModule(r io.Reader) (*module.Module, error) { + + wr := &reader{r: r, n: 0} + module, err := readModule(wr) + if err != nil { + return nil, errors.Wrapf(err, "offset 0x%x", wr.n) + } + + return module, nil +} + +// ReadCodeEntry reads a binary-encoded WASM code entry from r. +func ReadCodeEntry(r io.Reader) (*module.CodeEntry, error) { + + wr := &reader{r: r, n: 0} + entry, err := readCodeEntry(wr) + if err != nil { + return nil, errors.Wrapf(err, "offset 0x%x", wr.n) + } + + return entry, nil +} + +// CodeEntries returns the WASM code entries contained in r. +func CodeEntries(m *module.Module) ([]*module.CodeEntry, error) { + + entries := make([]*module.CodeEntry, len(m.Code.Segments)) + + for i, s := range m.Code.Segments { + buf := bytes.NewBuffer(s.Code) + entry, err := ReadCodeEntry(buf) + if err != nil { + return nil, err + } + entries[i] = entry + } + + return entries, nil +} + +type reader struct { + r io.Reader + n int +} + +func (r *reader) Read(bs []byte) (int, error) { + n, err := r.r.Read(bs) + r.n += n + return n, err +} + +func readModule(r io.Reader) (*module.Module, error) { + + if err := readMagic(r); err != nil { + return nil, err + } + + if err := readVersion(r); err != nil { + return nil, err + } + + var m module.Module + + if err := readSections(r, &m); err != nil && err != io.EOF { + return nil, err + } + + return &m, nil +} + +func readCodeEntry(r io.Reader) (*module.CodeEntry, error) { + + var entry module.CodeEntry + + if err := readLocals(r, &entry.Func.Locals); err != nil { + return nil, errors.Wrapf(err, "local declarations") + } + + return &entry, readExpr(r, &entry.Func.Expr) +} + +func readMagic(r io.Reader) error { + var v uint32 + if err := binary.Read(r, binary.LittleEndian, &v); err != nil { + return err + } else if v != constant.Magic { + return fmt.Errorf("illegal magic value") + } + return nil +} + +func readVersion(r io.Reader) error { + var v uint32 + if err := binary.Read(r, binary.LittleEndian, &v); err != nil { + return err + } else if v != constant.Version { + return fmt.Errorf("illegal wasm version") + } + return nil +} + +func readSections(r io.Reader, m *module.Module) error { + for { + id, err := readByte(r) + if err != nil { + return err + } + + size, err := leb128.ReadVarUint32(r) + if err != nil { + return err + } + + buf := make([]byte, size) + if _, err := io.ReadFull(r, buf); err != nil { + return err + } + + bufr := bytes.NewReader(buf) + + switch id { + case constant.CustomSectionID, constant.StartSectionID, constant.MemorySectionID: + continue + case constant.TypeSectionID: + if err := readTypeSection(bufr, &m.Type); err != nil { + return errors.Wrap(err, "type section") + } + case constant.ImportSectionID: + if err := readImportSection(bufr, &m.Import); err != nil { + return errors.Wrap(err, "import section") + } + case constant.GlobalSectionID: + if err := readGlobalSection(bufr, &m.Global); err != nil { + return errors.Wrap(err, "global section") + } + case constant.TableSectionID: + if err := readTableSection(bufr, &m.Table); err != nil { + return errors.Wrap(err, "table section") + } + case constant.FunctionSectionID: + if err := readFunctionSection(bufr, &m.Function); err != nil { + return errors.Wrap(err, "function section") + } + case constant.ExportSectionID: + if err := readExportSection(bufr, &m.Export); err != nil { + return errors.Wrap(err, "export section") + } + case constant.ElementSectionID: + if err := readElementSection(bufr, &m.Element); err != nil { + return errors.Wrap(err, "element section") + } + case constant.DataSectionID: + if err := readDataSection(bufr, &m.Data); err != nil { + return errors.Wrap(err, "data section") + } + case constant.CodeSectionID: + if err := readRawCodeSection(bufr, &m.Code); err != nil { + return errors.Wrap(err, "code section") + } + default: + return fmt.Errorf("illegal section id") + } + } +} + +func readTypeSection(r io.Reader, s *module.TypeSection) error { + + n, err := leb128.ReadVarUint32(r) + if err != nil { + return err + } + + for i := uint32(0); i < n; i++ { + + var ftype module.FunctionType + if err := readFunctionType(r, &ftype); err != nil { + return err + } + + s.Functions = append(s.Functions, ftype) + } + + return nil +} + +func readImportSection(r io.Reader, s *module.ImportSection) error { + + n, err := leb128.ReadVarUint32(r) + if err != nil { + return err + } + + for i := uint32(0); i < n; i++ { + + var imp module.Import + + if err := readImport(r, &imp); err != nil { + return err + } + + s.Imports = append(s.Imports, imp) + } + + return nil +} + +func readTableSection(r io.Reader, s *module.TableSection) error { + + n, err := leb128.ReadVarUint32(r) + if err != nil { + return err + } + + for i := uint32(0); i < n; i++ { + + var table module.Table + + if elem, err := readByte(r); err != nil { + return err + } else if elem != constant.ElementTypeAnyFunc { + return fmt.Errorf("illegal element type") + } else { + table.Type = types.Anyfunc + } + + if err := readLimits(r, &table.Lim); err != nil { + return err + } + + s.Tables = append(s.Tables, table) + } + + return nil +} + +func readGlobalSection(r io.Reader, s *module.GlobalSection) error { + + n, err := leb128.ReadVarUint32(r) + if err != nil { + return err + } + + for i := uint32(0); i < n; i++ { + + var global module.Global + + if err := readGlobal(r, &global); err != nil { + return err + } + + s.Globals = append(s.Globals, global) + } + + return nil +} + +func readFunctionSection(r io.Reader, s *module.FunctionSection) error { + return readVarUint32Vector(r, &s.TypeIndices) +} + +func readExportSection(r io.Reader, s *module.ExportSection) error { + + n, err := leb128.ReadVarUint32(r) + if err != nil { + return err + } + + for i := uint32(0); i < n; i++ { + + var exp module.Export + + if err := readExport(r, &exp); err != nil { + return err + } + + s.Exports = append(s.Exports, exp) + } + + return nil +} + +func readElementSection(r io.Reader, s *module.ElementSection) error { + + n, err := leb128.ReadVarUint32(r) + if err != nil { + return err + } + + for i := uint32(0); i < n; i++ { + + var seg module.ElementSegment + + if err := readElementSegment(r, &seg); err != nil { + return err + } + + s.Segments = append(s.Segments, seg) + } + + return nil +} + +func readDataSection(r io.Reader, s *module.DataSection) error { + + n, err := leb128.ReadVarUint32(r) + if err != nil { + return err + } + + for i := uint32(0); i < n; i++ { + + var seg module.DataSegment + + if err := readDataSegment(r, &seg); err != nil { + return err + } + + s.Segments = append(s.Segments, seg) + } + + return nil +} + +func readRawCodeSection(r io.Reader, s *module.RawCodeSection) error { + + n, err := leb128.ReadVarUint32(r) + if err != nil { + return err + } + + for i := uint32(0); i < n; i++ { + var seg module.RawCodeSegment + + if err := readRawCodeSegment(r, &seg); err != nil { + return err + } + + s.Segments = append(s.Segments, seg) + } + + return nil +} + +func readFunctionType(r io.Reader, ftype *module.FunctionType) error { + + if b, err := readByte(r); err != nil { + return err + } else if b != constant.FunctionTypeID { + return fmt.Errorf("illegal function type id 0x%x", b) + } + + if err := readValueTypeVector(r, &ftype.Params); err != nil { + return err + } + + return readValueTypeVector(r, &ftype.Results) +} + +func readGlobal(r io.Reader, global *module.Global) error { + + if err := readValueType(r, &global.Type); err != nil { + return err + } + + b, err := readByte(r) + if err != nil { + return err + } + + if b == 1 { + global.Mutable = true + } else if b != 0 { + return fmt.Errorf("illegal mutability flag") + } + + if err := readConstantExpr(r, &global.Init); err != nil { + return err + } + + return nil +} + +func readImport(r io.Reader, imp *module.Import) error { + + if err := readByteVectorString(r, &imp.Module); err != nil { + return err + } + + if err := readByteVectorString(r, &imp.Name); err != nil { + return err + } + + b, err := readByte(r) + if err != nil { + return err + + } + + if b == constant.ImportDescType { + index, err := leb128.ReadVarUint32(r) + if err != nil { + return err + } + imp.Descriptor = module.FunctionImport{ + Func: index, + } + return nil + } + + if b == constant.ImportDescTable { + if elem, err := readByte(r); err != nil { + return err + } else if elem != constant.ElementTypeAnyFunc { + return fmt.Errorf("illegal element type") + } + desc := module.TableImport{ + Type: types.Anyfunc, + } + if err := readLimits(r, &desc.Lim); err != nil { + return err + } + imp.Descriptor = desc + return nil + } + + if b == constant.ImportDescMem { + desc := module.MemoryImport{} + if err := readLimits(r, &desc.Mem.Lim); err != nil { + return err + } + imp.Descriptor = desc + return nil + } + + if b == constant.ImportDescGlobal { + desc := module.GlobalImport{} + if err := readValueType(r, &desc.Type); err != nil { + return err + } + b, err := readByte(r) + if err != nil { + return err + } + if b == 1 { + desc.Mutable = true + } else if b != 0 { + return fmt.Errorf("illegal mutability flag") + } + return nil + } + + return fmt.Errorf("illegal import descriptor type") +} + +func readExport(r io.Reader, exp *module.Export) error { + + if err := readByteVectorString(r, &exp.Name); err != nil { + return err + } + + b, err := readByte(r) + if err != nil { + return err + } + + switch b { + case constant.ExportDescType: + exp.Descriptor.Type = module.FunctionExportType + case constant.ExportDescTable: + exp.Descriptor.Type = module.TableExportType + case constant.ExportDescMem: + exp.Descriptor.Type = module.MemoryExportType + case constant.ExportDescGlobal: + exp.Descriptor.Type = module.GlobalExportType + default: + return fmt.Errorf("illegal export descriptor type") + } + + exp.Descriptor.Index, err = leb128.ReadVarUint32(r) + if err != nil { + return err + } + + return nil +} + +func readElementSegment(r io.Reader, seg *module.ElementSegment) error { + + if err := readVarUint32(r, &seg.Index); err != nil { + return err + } + + if err := readConstantExpr(r, &seg.Offset); err != nil { + return err + } + + if err := readVarUint32Vector(r, &seg.Indices); err != nil { + return err + } + + return nil +} + +func readDataSegment(r io.Reader, seg *module.DataSegment) error { + + if err := readVarUint32(r, &seg.Index); err != nil { + return err + } + + if err := readConstantExpr(r, &seg.Offset); err != nil { + return err + } + + if err := readByteVector(r, &seg.Init); err != nil { + return err + } + + return nil +} + +func readRawCodeSegment(r io.Reader, seg *module.RawCodeSegment) error { + return readByteVector(r, &seg.Code) +} + +func readConstantExpr(r io.Reader, expr *module.Expr) error { + + instrs := make([]instruction.Instruction, 0) + + for { + b, err := readByte(r) + if err != nil { + return err + } + + switch opcode.Opcode(b) { + case opcode.I32Const: + i32, err := leb128.ReadVarInt32(r) + if err != nil { + return err + } + instrs = append(instrs, instruction.I32Const{Value: i32}) + case opcode.I64Const: + i64, err := leb128.ReadVarInt64(r) + if err != nil { + return err + } + instrs = append(instrs, instruction.I64Const{Value: i64}) + case opcode.End: + expr.Instrs = instrs + return nil + default: + return fmt.Errorf("illegal constant expr opcode 0x%x", b) + } + } +} + +func readExpr(r io.Reader, expr *module.Expr) (err error) { + + defer func() { + if r := recover(); r != nil { + switch r := r.(type) { + case error: + err = r + default: + err = fmt.Errorf("unknown panic") + } + } + }() + + return readInstructions(r, &expr.Instrs) +} + +func readInstructions(r io.Reader, instrs *[]instruction.Instruction) error { + + ret := make([]instruction.Instruction, 0) + + for { + b, err := readByte(r) + if err != nil { + return err + } + + switch opcode.Opcode(b) { + case opcode.I32Const: + ret = append(ret, instruction.I32Const{Value: leb128.MustReadVarInt32(r)}) + case opcode.I64Const: + ret = append(ret, instruction.I64Const{Value: leb128.MustReadVarInt64(r)}) + case opcode.I32Eqz: + ret = append(ret, instruction.I32Eqz{}) + case opcode.GetLocal: + ret = append(ret, instruction.GetLocal{Index: leb128.MustReadVarUint32(r)}) + case opcode.SetLocal: + ret = append(ret, instruction.SetLocal{Index: leb128.MustReadVarUint32(r)}) + case opcode.Call: + ret = append(ret, instruction.Call{Index: leb128.MustReadVarUint32(r)}) + case opcode.BrIf: + ret = append(ret, instruction.BrIf{Index: leb128.MustReadVarUint32(r)}) + case opcode.Return: + ret = append(ret, instruction.Return{}) + case opcode.Block: + block := instruction.Block{} + if err := readBlockValueType(r, block.Type); err != nil { + return err + } + if err := readInstructions(r, &block.Instrs); err != nil { + return err + } + ret = append(ret, block) + case opcode.Loop: + loop := instruction.Loop{} + if err := readBlockValueType(r, loop.Type); err != nil { + return err + } + if err := readInstructions(r, &loop.Instrs); err != nil { + return err + } + ret = append(ret, loop) + case opcode.End: + *instrs = ret + return nil + default: + return fmt.Errorf("illegal opcode 0x%x", b) + } + } +} + +func readLimits(r io.Reader, l *module.Limit) error { + + b, err := readByte(r) + if err != nil { + return err + } + + min, err := leb128.ReadVarUint32(r) + if err != nil { + return err + } + + l.Min = min + + if b == 1 { + max, err := leb128.ReadVarUint32(r) + if err != nil { + return err + } + l.Max = &max + } else if b != 0 { + return fmt.Errorf("illegal limit flag") + } + + return nil +} + +func readLocals(r io.Reader, locals *[]module.LocalDeclaration) error { + + n, err := leb128.ReadVarUint32(r) + if err != nil { + return err + } + + ret := make([]module.LocalDeclaration, n) + + for i := uint32(0); i < n; i++ { + if err := readVarUint32(r, &ret[i].Count); err != nil { + return err + } + if err := readValueType(r, &ret[i].Type); err != nil { + return err + } + } + + *locals = ret + return nil +} + +func readByteVector(r io.Reader, v *[]byte) error { + + n, err := leb128.ReadVarUint32(r) + if err != nil { + return err + } + + buf := make([]byte, n) + if _, err := io.ReadFull(r, buf); err != nil { + return err + } + + *v = buf + return nil +} + +func readByteVectorString(r io.Reader, v *string) error { + + var buf []byte + + if err := readByteVector(r, &buf); err != nil { + return err + } + + *v = string(buf) + return nil +} + +func readVarUint32Vector(r io.Reader, v *[]uint32) error { + + n, err := leb128.ReadVarUint32(r) + if err != nil { + return err + } + + ret := make([]uint32, n) + + for i := uint32(0); i < n; i++ { + if err := readVarUint32(r, &ret[i]); err != nil { + return err + } + } + + *v = ret + return nil +} + +func readValueTypeVector(r io.Reader, v *[]types.ValueType) error { + + n, err := leb128.ReadVarUint32(r) + if err != nil { + return err + } + + ret := make([]types.ValueType, n) + + for i := uint32(0); i < n; i++ { + if err := readValueType(r, &ret[i]); err != nil { + return err + } + } + + *v = ret + return nil +} + +func readVarUint32(r io.Reader, v *uint32) error { + var err error + *v, err = leb128.ReadVarUint32(r) + return err +} + +func readValueType(r io.Reader, v *types.ValueType) error { + if b, err := readByte(r); err != nil { + return err + } else if b == constant.ValueTypeI32 { + *v = types.I32 + } else if b == constant.ValueTypeI64 { + *v = types.I64 + } else if b == constant.ValueTypeF32 { + *v = types.F32 + } else if b == constant.ValueTypeF64 { + *v = types.F64 + } else { + return fmt.Errorf("illegal value type: 0x%x", b) + } + return nil +} + +func readBlockValueType(r io.Reader, v *types.ValueType) error { + if b, err := readByte(r); err != nil { + return err + } else if b == constant.ValueTypeI32 { + *v = types.I32 + } else if b == constant.ValueTypeI64 { + *v = types.I64 + } else if b == constant.ValueTypeF32 { + *v = types.F32 + } else if b == constant.ValueTypeF64 { + *v = types.F64 + } else if b != constant.BlockTypeEmpty { + return fmt.Errorf("illegal value type: 0x%x", b) + } + return nil +} + +func readByte(r io.Reader) (byte, error) { + buf := make([]byte, 1) + _, err := r.Read(buf) + return buf[0], err +} diff --git a/vendor/github.com/open-policy-agent/opa/internal/wasm/encoding/writer.go b/vendor/github.com/open-policy-agent/opa/internal/wasm/encoding/writer.go new file mode 100644 index 000000000..c95045b05 --- /dev/null +++ b/vendor/github.com/open-policy-agent/opa/internal/wasm/encoding/writer.go @@ -0,0 +1,615 @@ +// Copyright 2018 The OPA Authors. All rights reserved. +// Use of this source code is governed by an Apache2 +// license that can be found in the LICENSE file. + +package encoding + +import ( + "bytes" + "encoding/binary" + "fmt" + "io" + "math" + + "github.com/open-policy-agent/opa/internal/leb128" + "github.com/open-policy-agent/opa/internal/wasm/constant" + "github.com/open-policy-agent/opa/internal/wasm/instruction" + "github.com/open-policy-agent/opa/internal/wasm/module" + "github.com/open-policy-agent/opa/internal/wasm/opcode" + "github.com/open-policy-agent/opa/internal/wasm/types" +) + +// WriteModule writes a binary-encoded representation of module to w. +func WriteModule(w io.Writer, module *module.Module) error { + + if err := writeMagic(w); err != nil { + return err + } + + if err := writeVersion(w); err != nil { + return err + } + + if module == nil { + return nil + } + + if err := writeTypeSection(w, module.Type); err != nil { + return err + } + + if err := writeImportSection(w, module.Import); err != nil { + return err + } + + if err := writeFunctionSection(w, module.Function); err != nil { + return err + } + + if err := writeTableSection(w, module.Table); err != nil { + return err + } + + if err := writeGlobalSection(w, module.Global); err != nil { + return err + } + + if err := writeExportSection(w, module.Export); err != nil { + return err + } + + if err := writeElementSection(w, module.Element); err != nil { + return err + } + + if err := writeRawCodeSection(w, module.Code); err != nil { + return err + } + + if err := writeDataSection(w, module.Data); err != nil { + return err + } + + return nil +} + +// WriteCodeEntry writes a binary encoded representation of entry to w. +func WriteCodeEntry(w io.Writer, entry *module.CodeEntry) error { + + if err := leb128.WriteVarUint32(w, uint32(len(entry.Func.Locals))); err != nil { + return err + } + + for _, local := range entry.Func.Locals { + + if err := leb128.WriteVarUint32(w, local.Count); err != nil { + return err + } + + if err := writeValueType(w, local.Type); err != nil { + return err + } + } + + return writeInstructions(w, entry.Func.Expr.Instrs) +} + +func writeMagic(w io.Writer) error { + return binary.Write(w, binary.LittleEndian, constant.Magic) +} + +func writeVersion(w io.Writer) error { + return binary.Write(w, binary.LittleEndian, constant.Version) +} + +func writeTypeSection(w io.Writer, s module.TypeSection) error { + + if len(s.Functions) == 0 { + return nil + } + + if err := writeByte(w, constant.TypeSectionID); err != nil { + return err + } + + var buf bytes.Buffer + + if err := leb128.WriteVarUint32(&buf, uint32(len(s.Functions))); err != nil { + return err + } + + for _, fsig := range s.Functions { + if err := writeFunctionType(&buf, fsig); err != nil { + return err + } + } + + return writeRawSection(w, &buf) +} + +func writeImportSection(w io.Writer, s module.ImportSection) error { + + if len(s.Imports) == 0 { + return nil + } + + if err := writeByte(w, constant.ImportSectionID); err != nil { + return err + } + + var buf bytes.Buffer + + if err := leb128.WriteVarUint32(&buf, uint32(len(s.Imports))); err != nil { + return err + } + + for _, imp := range s.Imports { + if err := writeImport(&buf, imp); err != nil { + return err + } + } + + return writeRawSection(w, &buf) +} + +func writeGlobalSection(w io.Writer, s module.GlobalSection) error { + + if len(s.Globals) == 0 { + return nil + } + + if err := writeByte(w, constant.GlobalSectionID); err != nil { + return err + } + + var buf bytes.Buffer + + if err := leb128.WriteVarUint32(&buf, uint32(len(s.Globals))); err != nil { + return err + } + + for _, global := range s.Globals { + if err := writeGlobal(&buf, global); err != nil { + return err + } + } + + return writeRawSection(w, &buf) +} + +func writeFunctionSection(w io.Writer, s module.FunctionSection) error { + + if len(s.TypeIndices) == 0 { + return nil + } + + if err := writeByte(w, constant.FunctionSectionID); err != nil { + return err + } + + var buf bytes.Buffer + + if err := leb128.WriteVarUint32(&buf, uint32(len(s.TypeIndices))); err != nil { + return err + } + + for _, idx := range s.TypeIndices { + if err := leb128.WriteVarUint32(&buf, uint32(idx)); err != nil { + return err + } + } + + return writeRawSection(w, &buf) +} + +func writeTableSection(w io.Writer, s module.TableSection) error { + + if len(s.Tables) == 0 { + return nil + } + + if err := writeByte(w, constant.TableSectionID); err != nil { + return err + } + + var buf bytes.Buffer + + if err := leb128.WriteVarUint32(&buf, uint32(len(s.Tables))); err != nil { + return err + } + + for _, table := range s.Tables { + switch table.Type { + case types.Anyfunc: + if err := writeByte(&buf, constant.ElementTypeAnyFunc); err != nil { + return err + } + default: + return fmt.Errorf("illegal table element type") + } + if err := writeLimits(&buf, table.Lim); err != nil { + return err + } + } + + return writeRawSection(w, &buf) + +} + +func writeExportSection(w io.Writer, s module.ExportSection) error { + + if len(s.Exports) == 0 { + return nil + } + + if err := writeByte(w, constant.ExportSectionID); err != nil { + return err + } + + var buf bytes.Buffer + + if err := leb128.WriteVarUint32(&buf, uint32(len(s.Exports))); err != nil { + return err + } + + for _, exp := range s.Exports { + if err := writeByteVector(&buf, []byte(exp.Name)); err != nil { + return err + } + var tpe byte + switch exp.Descriptor.Type { + case module.FunctionExportType: + tpe = constant.ExportDescType + case module.TableExportType: + tpe = constant.ExportDescTable + case module.MemoryExportType: + tpe = constant.ExportDescMem + case module.GlobalExportType: + tpe = constant.ExportDescGlobal + default: + return fmt.Errorf("illegal export descriptor type 0x%x", exp.Descriptor.Type) + } + if err := writeByte(&buf, tpe); err != nil { + return err + } + if err := leb128.WriteVarUint32(&buf, exp.Descriptor.Index); err != nil { + return err + } + } + + return writeRawSection(w, &buf) +} + +func writeElementSection(w io.Writer, s module.ElementSection) error { + + if len(s.Segments) == 0 { + return nil + } + + if err := writeByte(w, constant.ElementSectionID); err != nil { + return err + } + + var buf bytes.Buffer + + if err := leb128.WriteVarUint32(&buf, uint32(len(s.Segments))); err != nil { + return err + } + + for _, seg := range s.Segments { + if err := leb128.WriteVarUint32(&buf, seg.Index); err != nil { + return err + } + if err := writeInstructions(&buf, seg.Offset.Instrs); err != nil { + return err + } + if err := writeVarUint32Vector(&buf, seg.Indices); err != nil { + return err + } + } + + return writeRawSection(w, &buf) +} + +func writeRawCodeSection(w io.Writer, s module.RawCodeSection) error { + + if len(s.Segments) == 0 { + return nil + } + + if err := writeByte(w, constant.CodeSectionID); err != nil { + return err + } + + var buf bytes.Buffer + + if err := leb128.WriteVarUint32(&buf, uint32(len(s.Segments))); err != nil { + return err + } + + for _, seg := range s.Segments { + if err := leb128.WriteVarUint32(&buf, uint32(len(seg.Code))); err != nil { + return err + } + if _, err := buf.Write(seg.Code); err != nil { + return err + } + } + + return writeRawSection(w, &buf) +} + +func writeDataSection(w io.Writer, s module.DataSection) error { + + if len(s.Segments) == 0 { + return nil + } + + if err := writeByte(w, constant.DataSectionID); err != nil { + return err + } + + var buf bytes.Buffer + + if err := leb128.WriteVarUint32(&buf, uint32(len(s.Segments))); err != nil { + return err + } + + for _, seg := range s.Segments { + if err := leb128.WriteVarUint32(&buf, seg.Index); err != nil { + return err + } + if err := writeInstructions(&buf, seg.Offset.Instrs); err != nil { + return err + } + if err := writeByteVector(&buf, seg.Init); err != nil { + return err + } + } + + return writeRawSection(w, &buf) +} + +func writeFunctionType(w io.Writer, fsig module.FunctionType) error { + + if err := writeByte(w, constant.FunctionTypeID); err != nil { + return err + } + + if err := writeValueTypeVector(w, fsig.Params); err != nil { + return err + } + + return writeValueTypeVector(w, fsig.Results) +} + +func writeImport(w io.Writer, imp module.Import) error { + + if err := writeByteVector(w, []byte(imp.Module)); err != nil { + return err + } + + if err := writeByteVector(w, []byte(imp.Name)); err != nil { + return err + } + + switch desc := imp.Descriptor.(type) { + case module.FunctionImport: + if err := writeByte(w, constant.ImportDescType); err != nil { + return err + } + return leb128.WriteVarUint32(w, desc.Func) + case module.TableImport: + if err := writeByte(w, constant.ImportDescTable); err != nil { + return err + } + if err := writeByte(w, constant.ElementTypeAnyFunc); err != nil { + return err + } + return writeLimits(w, desc.Lim) + case module.MemoryImport: + if err := writeByte(w, constant.ImportDescMem); err != nil { + return err + } + return writeLimits(w, desc.Mem.Lim) + case module.GlobalImport: + if err := writeByte(w, constant.ImportDescGlobal); err != nil { + return err + } + if err := writeValueType(w, desc.Type); err != nil { + return err + } + if desc.Mutable { + return writeByte(w, constant.Mutable) + } + return writeByte(w, constant.Const) + default: + return fmt.Errorf("illegal import descriptor type") + } +} + +func writeGlobal(w io.Writer, global module.Global) error { + + if err := writeValueType(w, global.Type); err != nil { + return err + } + + var err error + + if global.Mutable { + err = writeByte(w, constant.Mutable) + } else { + err = writeByte(w, constant.Const) + } + + if err != nil { + return err + } + + if err := writeInstructions(w, global.Init.Instrs); err != nil { + return err + } + + return nil +} + +func writeInstructions(w io.Writer, instrs []instruction.Instruction) error { + + for i, instr := range instrs { + + _, err := w.Write([]byte{byte(instr.Op())}) + if err != nil { + return err + } + + for _, arg := range instr.ImmediateArgs() { + var err error + switch arg := arg.(type) { + case int32: + err = leb128.WriteVarInt32(w, arg) + case int64: + err = leb128.WriteVarInt64(w, arg) + case uint32: + err = leb128.WriteVarUint32(w, arg) + case uint64: + err = leb128.WriteVarUint64(w, arg) + case float32: + u32 := math.Float32bits(arg) + err = binary.Write(w, binary.LittleEndian, u32) + case float64: + u64 := math.Float64bits(arg) + err = binary.Write(w, binary.LittleEndian, u64) + default: + return fmt.Errorf("illegal immediate argument type on instruction %d", i) + } + if err != nil { + return err + } + } + + if si, ok := instr.(instruction.StructuredInstruction); ok { + if err := writeBlockValueType(w, si.BlockType()); err != nil { + return err + } + if err := writeInstructions(w, si.Instructions()); err != nil { + return err + } + } + + } + + _, err := w.Write([]byte{byte(opcode.End)}) + return err +} + +func writeLimits(w io.Writer, lim module.Limit) error { + if lim.Max == nil { + if err := writeByte(w, 0); err != nil { + return err + } + } else { + if err := writeByte(w, 1); err != nil { + return err + } + } + if err := leb128.WriteVarUint32(w, lim.Min); err != nil { + return err + } + if lim.Max != nil { + return leb128.WriteVarUint32(w, *lim.Max) + } + return nil +} + +func writeVarUint32Vector(w io.Writer, v []uint32) error { + + if err := leb128.WriteVarUint32(w, uint32(len(v))); err != nil { + return err + } + + for i := range v { + if err := leb128.WriteVarUint32(w, v[i]); err != nil { + return err + } + } + + return nil +} + +func writeValueTypeVector(w io.Writer, v []types.ValueType) error { + + if err := leb128.WriteVarUint32(w, uint32(len(v))); err != nil { + return err + } + + for i := range v { + if err := writeValueType(w, v[i]); err != nil { + return err + } + } + + return nil +} + +func writeBlockValueType(w io.Writer, v *types.ValueType) error { + var b byte + if v != nil { + switch *v { + case types.I32: + b = constant.ValueTypeI32 + case types.I64: + b = constant.ValueTypeI64 + case types.F32: + b = constant.ValueTypeF32 + case types.F64: + b = constant.ValueTypeF64 + } + } else { + b = constant.BlockTypeEmpty + } + return writeByte(w, b) +} + +func writeValueType(w io.Writer, v types.ValueType) error { + var b byte + switch v { + case types.I32: + b = constant.ValueTypeI32 + case types.I64: + b = constant.ValueTypeI64 + case types.F32: + b = constant.ValueTypeF32 + case types.F64: + b = constant.ValueTypeF64 + } + return writeByte(w, b) +} + +func writeRawSection(w io.Writer, buf *bytes.Buffer) error { + + size := buf.Len() + + if err := leb128.WriteVarUint32(w, uint32(size)); err != nil { + return err + } + + _, err := io.Copy(w, buf) + return err +} + +func writeByteVector(w io.Writer, bs []byte) error { + + if err := leb128.WriteVarUint32(w, uint32(len(bs))); err != nil { + return err + } + + _, err := w.Write(bs) + return err +} + +func writeByte(w io.Writer, b byte) error { + buf := make([]byte, 1) + buf[0] = b + _, err := w.Write(buf) + return err +} diff --git a/vendor/github.com/open-policy-agent/opa/internal/wasm/instruction/control.go b/vendor/github.com/open-policy-agent/opa/internal/wasm/instruction/control.go new file mode 100644 index 000000000..51567153d --- /dev/null +++ b/vendor/github.com/open-policy-agent/opa/internal/wasm/instruction/control.go @@ -0,0 +1,139 @@ +// Copyright 2018 The OPA Authors. All rights reserved. +// Use of this source code is governed by an Apache2 +// license that can be found in the LICENSE file. + +package instruction + +import ( + "github.com/open-policy-agent/opa/internal/wasm/opcode" + "github.com/open-policy-agent/opa/internal/wasm/types" +) + +// Unreachable reprsents an unreachable opcode. +type Unreachable struct { + NoImmediateArgs +} + +// Op returns the opcode of the instruction. +func (Unreachable) Op() opcode.Opcode { + return opcode.Unreachable +} + +// Nop represents a WASM no-op instruction. +type Nop struct { + NoImmediateArgs +} + +// Op returns the opcode of the instruction. +func (Nop) Op() opcode.Opcode { + return opcode.Nop +} + +// Block represents a WASM block instruction. +type Block struct { + NoImmediateArgs + Type *types.ValueType + Instrs []Instruction +} + +// Op returns the opcode of the instruction +func (Block) Op() opcode.Opcode { + return opcode.Block +} + +// BlockType returns the type of the block's return value. +func (i Block) BlockType() *types.ValueType { + return i.Type +} + +// Instructions returns the instructions contained in the block. +func (i Block) Instructions() []Instruction { + return i.Instrs +} + +// Loop represents a WASM loop instruction. +type Loop struct { + NoImmediateArgs + Type *types.ValueType + Instrs []Instruction +} + +// Op returns the opcode of the instruction. +func (Loop) Op() opcode.Opcode { + return opcode.Loop +} + +// BlockType returns the type of the loop's return value. +func (i Loop) BlockType() *types.ValueType { + return i.Type +} + +// Instructions represents the instructions contained in the loop. +func (i Loop) Instructions() []Instruction { + return i.Instrs +} + +// Br represents a WASM br instruction. +type Br struct { + Index uint32 +} + +// Op returns the opcode of the instruction. +func (Br) Op() opcode.Opcode { + return opcode.Br +} + +// ImmediateArgs returns the block index to break to. +func (i Br) ImmediateArgs() []interface{} { + return []interface{}{i.Index} +} + +// BrIf represents a WASM br_if instruction. +type BrIf struct { + Index uint32 +} + +// Op returns the opcode of the instruction. +func (BrIf) Op() opcode.Opcode { + return opcode.BrIf +} + +// ImmediateArgs returns the block index to break to. +func (i BrIf) ImmediateArgs() []interface{} { + return []interface{}{i.Index} +} + +// Call represents a WASM call instruction. +type Call struct { + Index uint32 +} + +// Op returns the opcode of the instruction. +func (Call) Op() opcode.Opcode { + return opcode.Call +} + +// ImmediateArgs returns the function index. +func (i Call) ImmediateArgs() []interface{} { + return []interface{}{i.Index} +} + +// Return represents a WASM return instruction. +type Return struct { + NoImmediateArgs +} + +// Op returns the opcode of the instruction. +func (Return) Op() opcode.Opcode { + return opcode.Return +} + +// End represents the special WASM end instruction. +type End struct { + NoImmediateArgs +} + +// Op returns the opcode of the instruction. +func (End) Op() opcode.Opcode { + return opcode.End +} diff --git a/vendor/github.com/open-policy-agent/opa/internal/wasm/instruction/instruction.go b/vendor/github.com/open-policy-agent/opa/internal/wasm/instruction/instruction.go new file mode 100644 index 000000000..066be77c4 --- /dev/null +++ b/vendor/github.com/open-policy-agent/opa/internal/wasm/instruction/instruction.go @@ -0,0 +1,33 @@ +// Copyright 2018 The OPA Authors. All rights reserved. +// Use of this source code is governed by an Apache2 +// license that can be found in the LICENSE file. + +// Package instruction defines WASM instruction types. +package instruction + +import ( + "github.com/open-policy-agent/opa/internal/wasm/opcode" + "github.com/open-policy-agent/opa/internal/wasm/types" +) + +// NoImmediateArgs indicates the instruction has no immediate arguments. +type NoImmediateArgs struct { +} + +// ImmediateArgs returns the immedate arguments of an instruction. +func (NoImmediateArgs) ImmediateArgs() []interface{} { + return nil +} + +// Instruction represents a single WASM instruction. +type Instruction interface { + Op() opcode.Opcode + ImmediateArgs() []interface{} +} + +// StructuredInstruction represents a structured control instruction like br_if. +type StructuredInstruction interface { + Instruction + BlockType() *types.ValueType + Instructions() []Instruction +} diff --git a/vendor/github.com/open-policy-agent/opa/internal/wasm/instruction/memory.go b/vendor/github.com/open-policy-agent/opa/internal/wasm/instruction/memory.go new file mode 100644 index 000000000..c449cb1b6 --- /dev/null +++ b/vendor/github.com/open-policy-agent/opa/internal/wasm/instruction/memory.go @@ -0,0 +1,39 @@ +// Copyright 2019 The OPA Authors. All rights reserved. +// Use of this source code is governed by an Apache2 +// license that can be found in the LICENSE file. + +package instruction + +import "github.com/open-policy-agent/opa/internal/wasm/opcode" + +// I32Load represents the WASM i32.load instruction. +type I32Load struct { + Offset int32 + Align int32 // expressed as a power of two +} + +// Op returns the opcode of the instruction. +func (I32Load) Op() opcode.Opcode { + return opcode.I32Load +} + +// ImmediateArgs returns the static offset and alignment operands. +func (i I32Load) ImmediateArgs() []interface{} { + return []interface{}{i.Align, i.Offset} +} + +// I32Store represents the WASM i32.store instruction. +type I32Store struct { + Offset int32 + Align int32 // expressed as a power of two +} + +// Op returns the opcode of the instruction. +func (I32Store) Op() opcode.Opcode { + return opcode.I32Store +} + +// ImmediateArgs returns the static offset and alignment operands. +func (i I32Store) ImmediateArgs() []interface{} { + return []interface{}{i.Align, i.Offset} +} diff --git a/vendor/github.com/open-policy-agent/opa/internal/wasm/instruction/numeric.go b/vendor/github.com/open-policy-agent/opa/internal/wasm/instruction/numeric.go new file mode 100644 index 000000000..f1acb31fc --- /dev/null +++ b/vendor/github.com/open-policy-agent/opa/internal/wasm/instruction/numeric.go @@ -0,0 +1,139 @@ +// Copyright 2018 The OPA Authors. All rights reserved. +// Use of this source code is governed by an Apache2 +// license that can be found in the LICENSE file. + +package instruction + +import ( + "github.com/open-policy-agent/opa/internal/wasm/opcode" +) + +// I32Const represents the WASM i32.const instruction. +type I32Const struct { + Value int32 +} + +// Op returns the opcode of the instruction. +func (I32Const) Op() opcode.Opcode { + return opcode.I32Const +} + +// ImmediateArgs returns the i32 value to push onto the stack. +func (i I32Const) ImmediateArgs() []interface{} { + return []interface{}{i.Value} +} + +// I64Const represents the WASM i64.const instruction. +type I64Const struct { + Value int64 +} + +// Op returns the opcode of the instruction. +func (I64Const) Op() opcode.Opcode { + return opcode.I64Const +} + +// ImmediateArgs returns the i64 value to push onto the stack. +func (i I64Const) ImmediateArgs() []interface{} { + return []interface{}{i.Value} +} + +// F32Const represents the WASM f32.const instruction. +type F32Const struct { + Value int32 +} + +// Op returns the opcode of the instruction. +func (F32Const) Op() opcode.Opcode { + return opcode.F32Const +} + +// ImmediateArgs returns the f32 value to push onto the stack. +func (i F32Const) ImmediateArgs() []interface{} { + return []interface{}{i.Value} +} + +// F64Const represents the WASM f64.const instruction. +type F64Const struct { + Value float64 +} + +// Op returns the opcode of the instruction. +func (F64Const) Op() opcode.Opcode { + return opcode.F64Const +} + +// ImmediateArgs returns the f64 value to push onto the stack. +func (i F64Const) ImmediateArgs() []interface{} { + return []interface{}{i.Value} +} + +// I32Eqz represents the WASM i32.eqz instruction. +type I32Eqz struct { + NoImmediateArgs +} + +// Op returns the opcode of the instruction. +func (I32Eqz) Op() opcode.Opcode { + return opcode.I32Eqz +} + +// I32Eq represents the WASM i32.eq instruction. +type I32Eq struct { + NoImmediateArgs +} + +// Op returns the opcode of the instruction. +func (I32Eq) Op() opcode.Opcode { + return opcode.I32Eq +} + +// I32Ne represents the WASM i32.ne instruction. +type I32Ne struct { + NoImmediateArgs +} + +// Op returns the opcode of the instruction. +func (I32Ne) Op() opcode.Opcode { + return opcode.I32Ne +} + +// I32GtS represents the WASM i32.gt_s instruction. +type I32GtS struct { + NoImmediateArgs +} + +// Op returns the opcode of the instruction. +func (I32GtS) Op() opcode.Opcode { + return opcode.I32GtS +} + +// I32GeS represents the WASM i32.ge_s instruction. +type I32GeS struct { + NoImmediateArgs +} + +// Op returns the opcode of the instruction. +func (I32GeS) Op() opcode.Opcode { + return opcode.I32GeS +} + +// I32LtS represents the WASM i32.lt_s instruction. +type I32LtS struct { + NoImmediateArgs +} + +// Op returns the opcode of the instruction. +func (I32LtS) Op() opcode.Opcode { + return opcode.I32LtS +} + +// I32LeS represents the WASM i32.le_s instruction. +type I32LeS struct { + NoImmediateArgs +} + +// Op returns the opcode of the instruction. +func (I32LeS) Op() opcode.Opcode { + return opcode.I32LeS +} diff --git a/vendor/github.com/open-policy-agent/opa/internal/wasm/instruction/variable.go b/vendor/github.com/open-policy-agent/opa/internal/wasm/instruction/variable.go new file mode 100644 index 000000000..ac57e5048 --- /dev/null +++ b/vendor/github.com/open-policy-agent/opa/internal/wasm/instruction/variable.go @@ -0,0 +1,38 @@ +// Copyright 2018 The OPA Authors. All rights reserved. +// Use of this source code is governed by an Apache2 +// license that can be found in the LICENSE file. + +package instruction + +import "github.com/open-policy-agent/opa/internal/wasm/opcode" + +// GetLocal represents the WASM get_local instruction. +type GetLocal struct { + Index uint32 +} + +// Op returns the opcode of the instruction. +func (GetLocal) Op() opcode.Opcode { + return opcode.GetLocal +} + +// ImmediateArgs returns the index of the local variable to push onto the stack. +func (i GetLocal) ImmediateArgs() []interface{} { + return []interface{}{i.Index} +} + +// SetLocal represents the WASM set_local instruction. +type SetLocal struct { + Index uint32 +} + +// Op returns the opcode of the instruction. +func (SetLocal) Op() opcode.Opcode { + return opcode.SetLocal +} + +// ImmediateArgs returns the index of the local variable to set with the top of +// the stack. +func (i SetLocal) ImmediateArgs() []interface{} { + return []interface{}{i.Index} +} diff --git a/vendor/github.com/open-policy-agent/opa/internal/wasm/module/module.go b/vendor/github.com/open-policy-agent/opa/internal/wasm/module/module.go new file mode 100644 index 000000000..b55094c8c --- /dev/null +++ b/vendor/github.com/open-policy-agent/opa/internal/wasm/module/module.go @@ -0,0 +1,340 @@ +// Copyright 2018 The OPA Authors. All rights reserved. +// Use of this source code is governed by an Apache2 +// license that can be found in the LICENSE file. + +package module + +import ( + "fmt" + "strings" + + "github.com/open-policy-agent/opa/internal/wasm/instruction" + "github.com/open-policy-agent/opa/internal/wasm/types" +) + +type ( + // Module represents a WASM module. + Module struct { + Version uint32 + Type TypeSection + Import ImportSection + Function FunctionSection + Table TableSection + Element ElementSection + Global GlobalSection + Export ExportSection + Code RawCodeSection + Data DataSection + } + + // TypeSection represents a WASM type section. + TypeSection struct { + Functions []FunctionType + } + + // ImportSection represents a WASM import section. + ImportSection struct { + Imports []Import + } + + // FunctionSection represents a WASM function section. + FunctionSection struct { + TypeIndices []uint32 + } + + // TableSection represents a WASM table section. + TableSection struct { + Tables []Table + } + + // ElementSection represents a WASM element section. + ElementSection struct { + Segments []ElementSegment + } + + // GlobalSection represents a WASM global section. + GlobalSection struct { + Globals []Global + } + + // ExportSection represents a WASM export section. + ExportSection struct { + Exports []Export + } + + // RawCodeSection represents a WASM code section. The code section is left as a + // raw byte sequence. See CodeSection for the decoded version. + RawCodeSection struct { + Segments []RawCodeSegment + } + + // DataSection represents a WASM data section. + DataSection struct { + Segments []DataSegment + } + + // FunctionType represents a WASM function type definition. + FunctionType struct { + Params []types.ValueType + Results []types.ValueType + } + + // Import represents a WASM import statement. + Import struct { + Module string + Name string + Descriptor ImportDescriptor + } + + // ImportDescriptor represents a WASM import descriptor. + ImportDescriptor interface { + fmt.Stringer + Kind() ImportDescriptorType + } + + // ImportDescriptorType defines allowed kinds of import descriptors. + ImportDescriptorType int + + // FunctionImport represents a WASM function import statement. + FunctionImport struct { + Func uint32 + } + + // MemoryImport represents a WASM memory import statement. + MemoryImport struct { + Mem MemType + } + + // MemType defines the attributes of a memory import. + MemType struct { + Lim Limit + } + + // TableImport represents a WASM table import statement. + TableImport struct { + Type types.ElementType + Lim Limit + } + + // ElementSegment represents a WASM element segment. + ElementSegment struct { + Index uint32 + Offset Expr + Indices []uint32 + } + + // GlobalImport represents a WASM global variable import statement. + GlobalImport struct { + Type types.ValueType + Mutable bool + } + + // Limit represents a WASM limit. + Limit struct { + Min uint32 + Max *uint32 + } + + // Table represents a WASM table statement. + Table struct { + Type types.ElementType + Lim Limit + } + + // Global represents a WASM global statement. + Global struct { + Type types.ValueType + Mutable bool + Init Expr + } + + // Export represents a WASM export statement. + Export struct { + Name string + Descriptor ExportDescriptor + } + + // ExportDescriptor represents a WASM export descriptor. + ExportDescriptor struct { + Type ExportDescriptorType + Index uint32 + } + + // ExportDescriptorType defines the allowed kinds of export descriptors. + ExportDescriptorType int + + // RawCodeSegment represents a binary-encoded WASM code segment. + RawCodeSegment struct { + Code []byte + } + + // DataSegment represents a WASM data segment. + DataSegment struct { + Index uint32 + Offset Expr + Init []byte + } + + // Expr represents a WASM expression. + Expr struct { + Instrs []instruction.Instruction + } + + // CodeEntry represents a code segment entry. + CodeEntry struct { + Func Function + } + + // Function represents a function in a code segment. + Function struct { + Locals []LocalDeclaration + Expr Expr + } + + // LocalDeclaration represents a local variable declaration. + LocalDeclaration struct { + Count uint32 + Type types.ValueType + } +) + +// Defines the allowed kinds of imports. +const ( + FunctionImportType ImportDescriptorType = iota + TableImportType + MemoryImportType + GlobalImportType +) + +func (x ImportDescriptorType) String() string { + switch x { + case FunctionImportType: + return "func" + case TableImportType: + return "table" + case MemoryImportType: + return "memory" + case GlobalImportType: + return "global" + } + panic("illegal value") +} + +// Defines the allowed kinds of exports. +const ( + FunctionExportType ExportDescriptorType = iota + TableExportType + MemoryExportType + GlobalExportType +) + +func (x ExportDescriptorType) String() string { + switch x { + case FunctionExportType: + return "func" + case TableExportType: + return "table" + case MemoryExportType: + return "memory" + case GlobalExportType: + return "global" + } + panic("illegal value") +} + +// Kind returns the function import type kind. +func (i FunctionImport) Kind() ImportDescriptorType { + return FunctionImportType +} + +func (i FunctionImport) String() string { + return fmt.Sprintf("%v[type=%v]", i.Kind(), i.Func) +} + +// Kind returns the memory import type kind. +func (i MemoryImport) Kind() ImportDescriptorType { + return MemoryImportType +} + +func (i MemoryImport) String() string { + return fmt.Sprintf("%v[%v]", i.Kind(), i.Mem.Lim) +} + +// Kind returns the table import type kind. +func (i TableImport) Kind() ImportDescriptorType { + return TableImportType +} + +func (i TableImport) String() string { + return fmt.Sprintf("%v[%v, %v]", i.Kind(), i.Type, i.Lim) +} + +// Kind returns the global import type kind. +func (i GlobalImport) Kind() ImportDescriptorType { + return GlobalImportType +} + +func (i GlobalImport) String() string { + return fmt.Sprintf("%v[%v, mut=%v]", i.Kind(), i.Type, i.Mutable) +} + +func (tpe FunctionType) String() string { + params := make([]string, len(tpe.Params)) + results := make([]string, len(tpe.Results)) + for i := range tpe.Params { + params[i] = tpe.Params[i].String() + } + for i := range tpe.Results { + results[i] = tpe.Results[i].String() + } + return "(" + strings.Join(params, ", ") + ") -> (" + strings.Join(results, ", ") + ")" +} + +// Equal returns true if tpe equals other. +func (tpe FunctionType) Equal(other FunctionType) bool { + + if len(tpe.Params) != len(other.Params) || len(tpe.Results) != len(other.Results) { + return false + } + + for i := range tpe.Params { + if tpe.Params[i] != other.Params[i] { + return false + } + } + + for i := range tpe.Results { + if tpe.Results[i] != other.Results[i] { + return false + } + } + + return true +} + +func (imp Import) String() string { + return fmt.Sprintf("%v %v.%v", imp.Descriptor.String(), imp.Module, imp.Name) +} + +func (exp Export) String() string { + return fmt.Sprintf("%v[%v] %v", exp.Descriptor.Type, exp.Descriptor.Index, exp.Name) +} + +func (seg RawCodeSegment) String() string { + return fmt.Sprintf("", len(seg.Code)) +} + +func (seg DataSegment) String() string { + return fmt.Sprintf("", seg.Index, seg.Offset, len(seg.Init)) +} + +func (e Expr) String() string { + return fmt.Sprintf("%d instr(s)", len(e.Instrs)) +} + +func (lim Limit) String() string { + if lim.Max == nil { + return fmt.Sprintf("min=%v", lim.Min) + } + return fmt.Sprintf("min=%v max=%v", lim.Min, lim.Max) +} diff --git a/vendor/github.com/open-policy-agent/opa/internal/wasm/module/pretty.go b/vendor/github.com/open-policy-agent/opa/internal/wasm/module/pretty.go new file mode 100644 index 000000000..2b28ad85b --- /dev/null +++ b/vendor/github.com/open-policy-agent/opa/internal/wasm/module/pretty.go @@ -0,0 +1,84 @@ +// Copyright 2018 The OPA Authors. All rights reserved. +// Use of this source code is governed by an Apache2 +// license that can be found in the LICENSE file. + +package module + +import ( + "encoding/hex" + "fmt" + "io" +) + +// PrettyOption defines options for controlling pretty printing. +type PrettyOption struct { + Contents bool // show raw byte content of data+code sections. +} + +// Pretty writes a human-readable representation of m to w. +func Pretty(w io.Writer, m *Module, opts ...PrettyOption) { + fmt.Fprintln(w, "version:", m.Version) + fmt.Fprintln(w, "types:") + for _, fn := range m.Type.Functions { + fmt.Fprintln(w, " -", fn) + } + fmt.Fprintln(w, "imports:") + for i, imp := range m.Import.Imports { + if imp.Descriptor.Kind() == FunctionImportType { + fmt.Printf(" - [%d] %v\n", i, imp) + } else { + fmt.Fprintln(w, " -", imp) + } + } + fmt.Fprintln(w, "functions:") + for _, fn := range m.Function.TypeIndices { + if fn >= uint32(len(m.Type.Functions)) { + fmt.Fprintln(w, " -", "???") + } else { + fmt.Fprintln(w, " -", m.Type.Functions[fn]) + } + } + fmt.Fprintln(w, "exports:") + for _, exp := range m.Export.Exports { + fmt.Fprintln(w, " -", exp) + } + fmt.Fprintln(w, "code:") + for _, seg := range m.Code.Segments { + fmt.Fprintln(w, " -", seg) + } + fmt.Fprintln(w, "data:") + for _, seg := range m.Data.Segments { + fmt.Fprintln(w, " -", seg) + } + if len(opts) == 0 { + return + } + fmt.Fprintln(w) + for _, opt := range opts { + if opt.Contents { + newline := false + if len(m.Data.Segments) > 0 { + fmt.Fprintln(w, "data section:") + for _, seg := range m.Data.Segments { + if newline { + fmt.Fprintln(w) + } + fmt.Fprintln(w, hex.Dump(seg.Init)) + newline = true + } + newline = false + } + if len(m.Code.Segments) > 0 { + fmt.Fprintln(w, "code section:") + for _, seg := range m.Code.Segments { + if newline { + fmt.Fprintln(w) + } + fmt.Fprintln(w, hex.Dump(seg.Code)) + newline = true + } + newline = false + } + } + } +} diff --git a/vendor/github.com/open-policy-agent/opa/internal/wasm/opcode/opcode.go b/vendor/github.com/open-policy-agent/opa/internal/wasm/opcode/opcode.go new file mode 100644 index 000000000..7d35a3012 --- /dev/null +++ b/vendor/github.com/open-policy-agent/opa/internal/wasm/opcode/opcode.go @@ -0,0 +1,218 @@ +// Copyright 2018 The OPA Authors. All rights reserved. +// Use of this source code is governed by an Apache2 +// license that can be found in the LICENSE file. + +// Package opcode contains constants and utilities for working with WASM opcodes. +package opcode + +// Opcode represents a WASM instruction opcode. +type Opcode byte + +// Control instructions. +const ( + Unreachable Opcode = iota + Nop + Block + Loop + If + Else +) + +const ( + // End defines the special end WASM opcode. + End Opcode = 0x0B +) + +// Extended control instructions. +const ( + Br Opcode = iota + 0x0C + BrIf + BrTable + Return + Call + CallIndirect +) + +// Parameter instructions. +const ( + Drop Opcode = iota + 0x1A + Select +) + +// Variable instructions. +const ( + GetLocal Opcode = iota + 0x20 + SetLocal + TeeLocal + GetGlobal + SetGlobal +) + +// Memory instructions. +const ( + I32Load Opcode = iota + 0x28 + I64Load + F32Load + F64Load + I32Load8S + I32Load8U + I32Load16S + I32Load16U + I64Load8S + I64Load8U + I64Load16S + I64Load16U + I64Load32S + I64Load32U + I32Store + I64Store + F32Store + F64Store + I32Store8 + I32Store16 + I64Store8 + I64Store16 + I64Store32 + MemorySize + MemoryGrow +) + +// Numeric instructions. +const ( + I32Const Opcode = iota + 0x41 + I64Const + F32Const + F64Const + + I32Eqz + I32Eq + I32Ne + I32LtS + I32LtU + I32GtS + I32GtU + I32LeS + I32LeU + I32GeS + I32GeU + + I64Eqz + I64Eq + I64Ne + I64LtS + I64LtU + I64GtS + I64GtU + I64LeS + I64LeU + I64GeS + I64GeU + + F32Eq + F32Ne + F32Lt + F32Gt + F32Le + F32Ge + + F64Eq + F64Ne + F64Lt + F64Gt + F64Le + F64Ge + + I32Clz + I32Ctz + I32Popcnt + I32Add + I32Sub + I32Mul + I32DivS + I32DivU + I32RemS + I32RemU + I32And + I32Or + I32Xor + I32Shl + I32ShrS + I32ShrU + I32Rotl + I32Rotr + + I64Clz + I64Ctz + I64Popcnt + I64Add + I64Sub + I64Mul + I64DivS + I64DivU + I64RemS + I64RemU + I64And + I64Or + I64Xor + I64Shl + I64ShrS + I64ShrU + I64Rotl + I64Rotr + + F32Abs + F32Neg + F32Ceil + F32Floor + F32Trunc + F32Nearest + F32Sqrt + F32Add + F32Sub + F32Mul + F32Div + F32Min + F32Max + F32Copysign + + F64Abs + F64Neg + F64Ceil + F64Floor + F64Trunc + F64Nearest + F64Sqrt + F64Add + F64Sub + F64Mul + F64Div + F64Min + F64Max + F64Copysign + + I32WrapI64 + I32TruncSF32 + I32TruncUF32 + I32TruncSF64 + I32TruncUF64 + I64ExtendSI32 + I64ExtendUI32 + I64TruncSF32 + I64TruncUF32 + I64TruncSF64 + I64TruncUF64 + F32ConvertSI32 + F32ConvertUI32 + F32ConvertSI64 + F32ConvertUI64 + F32DemoteF64 + F64ConvertSI32 + F64ConvertUI32 + F64ConvertSI64 + F64ConvertUI64 + F64PromoteF32 + I32ReinterpretF32 + I64ReinterpretF64 + F32ReinterpretI32 + F64ReinterpretI64 +) diff --git a/vendor/github.com/open-policy-agent/opa/internal/wasm/types/types.go b/vendor/github.com/open-policy-agent/opa/internal/wasm/types/types.go new file mode 100644 index 000000000..4e2b77622 --- /dev/null +++ b/vendor/github.com/open-policy-agent/opa/internal/wasm/types/types.go @@ -0,0 +1,36 @@ +// Copyright 2018 The OPA Authors. All rights reserved. +// Use of this source code is governed by an Apache2 +// license that can be found in the LICENSE file. + +// Package types defines the WASM value type constants. +package types + +// ValueType represents an intrinsic value in WASM. +type ValueType int + +// Defines the intrinsic value types. +const ( + I32 ValueType = iota + I64 + F32 + F64 +) + +func (tpe ValueType) String() string { + if tpe == I32 { + return "i32" + } else if tpe == I64 { + return "i64" + } else if tpe == F32 { + return "f32" + } + return "f64" +} + +// ElementType defines the type of table elements. +type ElementType int + +const ( + // Anyfunc is the union of all table types. + Anyfunc ElementType = iota +) diff --git a/vendor/github.com/open-policy-agent/opa/loader/errors.go b/vendor/github.com/open-policy-agent/opa/loader/errors.go new file mode 100644 index 000000000..b2f29c648 --- /dev/null +++ b/vendor/github.com/open-policy-agent/opa/loader/errors.go @@ -0,0 +1,68 @@ +// Copyright 2017 The OPA Authors. All rights reserved. +// Use of this source code is governed by an Apache2 +// license that can be found in the LICENSE file. + +package loader + +import ( + "fmt" + "strings" + + "github.com/open-policy-agent/opa/ast" +) + +// Errors is a wrapper for multiple loader errors. +type Errors []error + +func (e Errors) Error() string { + if len(e) == 0 { + return "no error(s)" + } + if len(e) == 1 { + return "1 error occurred during loading: " + e[0].Error() + } + buf := make([]string, len(e)) + for i := range buf { + buf[i] = e[i].Error() + } + return fmt.Sprintf("%v errors occurred during loading:\n", len(e)) + strings.Join(buf, "\n") +} + +func (e *Errors) add(err error) { + if errs, ok := err.(ast.Errors); ok { + for i := range errs { + *e = append(*e, errs[i]) + } + } else { + *e = append(*e, err) + } +} + +type unsupportedDocumentType string + +func (path unsupportedDocumentType) Error() string { + return string(path) + ": bad document type" +} + +type unrecognizedFile string + +func (path unrecognizedFile) Error() string { + return string(path) + ": can't recognize file type" +} + +func isUnrecognizedFile(err error) bool { + _, ok := err.(unrecognizedFile) + return ok +} + +type mergeError string + +func (e mergeError) Error() string { + return string(e) + ": merge error" +} + +type emptyModuleError string + +func (e emptyModuleError) Error() string { + return string(e) + ": empty policy" +} diff --git a/vendor/github.com/open-policy-agent/opa/loader/loader.go b/vendor/github.com/open-policy-agent/opa/loader/loader.go new file mode 100644 index 000000000..83f6a6592 --- /dev/null +++ b/vendor/github.com/open-policy-agent/opa/loader/loader.go @@ -0,0 +1,476 @@ +// Copyright 2017 The OPA Authors. All rights reserved. +// Use of this source code is governed by an Apache2 +// license that can be found in the LICENSE file. + +// Package loader contains utilities for loading files into OPA. +package loader + +import ( + "bytes" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "strings" + + "github.com/ghodss/yaml" + "github.com/pkg/errors" + + "github.com/open-policy-agent/opa/metrics" + + "github.com/open-policy-agent/opa/ast" + "github.com/open-policy-agent/opa/bundle" + fileurl "github.com/open-policy-agent/opa/internal/file/url" + "github.com/open-policy-agent/opa/internal/merge" + "github.com/open-policy-agent/opa/storage" + "github.com/open-policy-agent/opa/storage/inmem" + "github.com/open-policy-agent/opa/util" +) + +// Result represents the result of successfully loading zero or more files. +type Result struct { + Documents map[string]interface{} + Modules map[string]*RegoFile + path []string +} + +// ParsedModules returns the parsed modules stored on the result. +func (l *Result) ParsedModules() map[string]*ast.Module { + modules := make(map[string]*ast.Module) + for _, module := range l.Modules { + modules[module.Name] = module.Parsed + } + return modules +} + +// Compiler returns a Compiler object with the compiled modules from this loader +// result. +func (l *Result) Compiler() (*ast.Compiler, error) { + compiler := ast.NewCompiler() + compiler.Compile(l.ParsedModules()) + if compiler.Failed() { + return nil, compiler.Errors + } + return compiler, nil +} + +// Store returns a Store object with the documents from this loader result. +func (l *Result) Store() (storage.Store, error) { + return inmem.NewFromObject(l.Documents), nil +} + +// RegoFile represents the result of loading a single Rego source file. +type RegoFile struct { + Name string + Parsed *ast.Module + Raw []byte +} + +// Filter defines the interface for filtering files during loading. If the +// filter returns true, the file should be excluded from the result. +type Filter func(abspath string, info os.FileInfo, depth int) bool + +// GlobExcludeName excludes files and directories whose names do not match the +// shell style pattern at minDepth or greater. +func GlobExcludeName(pattern string, minDepth int) Filter { + return func(abspath string, info os.FileInfo, depth int) bool { + match, _ := filepath.Match(pattern, info.Name()) + return match && depth >= minDepth + } +} + +// FileLoader defines an interface for loading OPA data files +// and Rego policies. +type FileLoader interface { + All(paths []string) (*Result, error) + Filtered(paths []string, filter Filter) (*Result, error) + AsBundle(path string) (*bundle.Bundle, error) + + WithMetrics(m metrics.Metrics) FileLoader +} + +// NewFileLoader returns a new FileLoader instance. +func NewFileLoader() FileLoader { + return &fileLoader{ + metrics: metrics.New(), + } +} + +type fileLoader struct { + metrics metrics.Metrics +} + +// WithMetrics provides the metrics instance to use while loading +func (fl *fileLoader) WithMetrics(m metrics.Metrics) FileLoader { + fl.metrics = m + return fl +} + +// All returns a Result object loaded (recursively) from the specified paths. +func (fl fileLoader) All(paths []string) (*Result, error) { + return fl.Filtered(paths, nil) +} + +// Filtered returns a Result object loaded (recursively) from the specified +// paths while applying the given filters. If any filter returns true, the +// file/directory is excluded. +func (fl fileLoader) Filtered(paths []string, filter Filter) (*Result, error) { + return all(paths, filter, func(curr *Result, path string, depth int) error { + + bs, err := ioutil.ReadFile(path) + if err != nil { + return err + } + + result, err := loadKnownTypes(path, bs, fl.metrics) + if err != nil { + if !isUnrecognizedFile(err) { + return err + } + if depth > 0 { + return nil + } + result, err = loadFileForAnyType(path, bs, fl.metrics) + if err != nil { + return err + } + } + + return curr.merge(path, result) + }) +} + +// AsBundle loads a path as a bundle. If it is a single file +// it will be treated as a normal tarball bundle. If a directory +// is supplied it will be loaded as an unzipped bundle tree. +func (fl fileLoader) AsBundle(path string) (*bundle.Bundle, error) { + path, err := fileurl.Clean(path) + if err != nil { + return nil, err + } + + fi, err := os.Stat(path) + if err != nil { + return nil, fmt.Errorf("error reading %q: %s", path, err) + } + + var bundleLoader bundle.DirectoryLoader + + if fi.IsDir() { + bundleLoader = bundle.NewDirectoryLoader(path) + } else { + fh, err := os.Open(path) + if err != nil { + return nil, err + } + bundleLoader = bundle.NewTarballLoader(fh) + } + + br := bundle.NewCustomReader(bundleLoader).WithMetrics(fl.metrics) + + // For bundle directories add the full path in front of module file names + // to simplify debugging. + if fi.IsDir() { + br.WithBaseDir(path) + } + + b, err := br.Read() + if err != nil { + err = errors.Wrap(err, fmt.Sprintf("bundle %s", path)) + } + + return &b, err +} + +// All returns a Result object loaded (recursively) from the specified paths. +// Deprecated: Use FileLoader.Filtered() instead. +func All(paths []string) (*Result, error) { + return NewFileLoader().Filtered(paths, nil) +} + +// Filtered returns a Result object loaded (recursively) from the specified +// paths while applying the given filters. If any filter returns true, the +// file/directory is excluded. +// Deprecated: Use FileLoader.Filtered() instead. +func Filtered(paths []string, filter Filter) (*Result, error) { + return NewFileLoader().Filtered(paths, filter) +} + +// AsBundle loads a path as a bundle. If it is a single file +// it will be treated as a normal tarball bundle. If a directory +// is supplied it will be loaded as an unzipped bundle tree. +// Deprecated: Use FileLoader.AsBundle() instead. +func AsBundle(path string) (*bundle.Bundle, error) { + return NewFileLoader().AsBundle(path) +} + +// AllRegos returns a Result object loaded (recursively) with all Rego source +// files from the specified paths. +func AllRegos(paths []string) (*Result, error) { + return NewFileLoader().Filtered(paths, func(_ string, info os.FileInfo, depth int) bool { + return !info.IsDir() && !strings.HasSuffix(info.Name(), bundle.RegoExt) + }) +} + +// Rego returns a RegoFile object loaded from the given path. +func Rego(path string) (*RegoFile, error) { + path, err := fileurl.Clean(path) + if err != nil { + return nil, err + } + bs, err := ioutil.ReadFile(path) + if err != nil { + return nil, err + } + return loadRego(path, bs, metrics.New()) +} + +// CleanPath returns the normalized version of a path that can be used as an identifier. +func CleanPath(path string) string { + return strings.Trim(path, "/") +} + +// Paths returns a sorted list of files contained at path. If recurse is true +// and path is a directory, then Paths will walk the directory structure +// recursively and list files at each level. +func Paths(path string, recurse bool) (paths []string, err error) { + path, err = fileurl.Clean(path) + if err != nil { + return nil, err + } + err = filepath.Walk(path, func(f string, info os.FileInfo, err error) error { + if !recurse { + if path != f && path != filepath.Dir(f) { + return filepath.SkipDir + } + } + paths = append(paths, f) + return nil + }) + return paths, err +} + +// SplitPrefix returns a tuple specifying the document prefix and the file +// path. +func SplitPrefix(path string) ([]string, string) { + // Non-prefixed URLs can be returned without modification and their contents + // can be rooted directly under data. + if strings.Index(path, "://") == strings.Index(path, ":") { + return nil, path + } + parts := strings.SplitN(path, ":", 2) + if len(parts) == 2 && len(parts[0]) > 0 { + return strings.Split(parts[0], "."), parts[1] + } + return nil, path +} + +func (l *Result) merge(path string, result interface{}) error { + switch result := result.(type) { + case bundle.Bundle: + for _, module := range result.Modules { + l.Modules[module.Path] = &RegoFile{ + Name: module.Path, + Parsed: module.Parsed, + Raw: module.Raw, + } + } + return l.mergeDocument(path, result.Data) + case *RegoFile: + l.Modules[CleanPath(path)] = result + return nil + default: + return l.mergeDocument(path, result) + } +} + +func (l *Result) mergeDocument(path string, doc interface{}) error { + obj, ok := makeDir(l.path, doc) + if !ok { + return unsupportedDocumentType(path) + } + merged, ok := merge.InterfaceMaps(l.Documents, obj) + if !ok { + return mergeError(path) + } + for k := range merged { + l.Documents[k] = merged[k] + } + return nil +} + +func (l *Result) withParent(p string) *Result { + path := append(l.path, p) + return &Result{ + Documents: l.Documents, + Modules: l.Modules, + path: path, + } +} + +func newResult() *Result { + return &Result{ + Documents: map[string]interface{}{}, + Modules: map[string]*RegoFile{}, + } +} + +func all(paths []string, filter Filter, f func(*Result, string, int) error) (*Result, error) { + errors := Errors{} + root := newResult() + + for _, path := range paths { + + // Paths can be prefixed with a string that specifies where content should be + // loaded under data. E.g., foo.bar:/path/to/some.json will load the content + // of some.json under {"foo": {"bar": ...}}. + loaded := root + prefix, path := SplitPrefix(path) + if len(prefix) > 0 { + for _, part := range prefix { + loaded = loaded.withParent(part) + } + } + + allRec(path, filter, &errors, loaded, 0, f) + } + + if len(errors) > 0 { + return nil, errors + } + + return root, nil +} + +func allRec(path string, filter Filter, errors *Errors, loaded *Result, depth int, f func(*Result, string, int) error) { + + path, err := fileurl.Clean(path) + if err != nil { + errors.add(err) + return + } + + info, err := os.Stat(path) + if err != nil { + errors.add(err) + return + } + + if filter != nil && filter(path, info, depth) { + return + } + + if !info.IsDir() { + if err := f(loaded, path, depth); err != nil { + errors.add(err) + } + return + } + + // If we are recursing on directories then content must be loaded under path + // specified by directory hierarchy. + if depth > 0 { + loaded = loaded.withParent(info.Name()) + } + + files, err := ioutil.ReadDir(path) + if err != nil { + errors.add(err) + return + } + + for _, file := range files { + allRec(filepath.Join(path, file.Name()), filter, errors, loaded, depth+1, f) + } +} + +func loadKnownTypes(path string, bs []byte, m metrics.Metrics) (interface{}, error) { + switch filepath.Ext(path) { + case ".json": + return loadJSON(path, bs, m) + case ".rego": + return loadRego(path, bs, m) + case ".yaml", ".yml": + return loadYAML(path, bs, m) + default: + if strings.HasSuffix(path, ".tar.gz") { + r, err := loadBundleFile(bs, m) + if err != nil { + err = errors.Wrap(err, fmt.Sprintf("bundle %s", path)) + } + return r, err + } + } + return nil, unrecognizedFile(path) +} + +func loadFileForAnyType(path string, bs []byte, m metrics.Metrics) (interface{}, error) { + module, err := loadRego(path, bs, m) + if err == nil { + return module, nil + } + doc, err := loadJSON(path, bs, m) + if err == nil { + return doc, nil + } + doc, err = loadYAML(path, bs, m) + if err == nil { + return doc, nil + } + return nil, unrecognizedFile(path) +} + +func loadBundleFile(bs []byte, m metrics.Metrics) (bundle.Bundle, error) { + tl := bundle.NewTarballLoader(bytes.NewBuffer(bs)) + br := bundle.NewCustomReader(tl).WithMetrics(m).IncludeManifestInData(true) + return br.Read() +} + +func loadRego(path string, bs []byte, m metrics.Metrics) (*RegoFile, error) { + m.Timer(metrics.RegoModuleParse).Start() + module, err := ast.ParseModule(path, string(bs)) + m.Timer(metrics.RegoModuleParse).Stop() + if err != nil { + return nil, err + } + result := &RegoFile{ + Name: path, + Parsed: module, + Raw: bs, + } + return result, nil +} + +func loadJSON(path string, bs []byte, m metrics.Metrics) (interface{}, error) { + m.Timer(metrics.RegoDataParse).Start() + buf := bytes.NewBuffer(bs) + decoder := util.NewJSONDecoder(buf) + var x interface{} + err := decoder.Decode(&x) + m.Timer(metrics.RegoDataParse).Stop() + if err != nil { + return nil, errors.Wrap(err, path) + } + return x, nil +} + +func loadYAML(path string, bs []byte, m metrics.Metrics) (interface{}, error) { + m.Timer(metrics.RegoDataParse).Start() + bs, err := yaml.YAMLToJSON(bs) + m.Timer(metrics.RegoDataParse).Stop() + if err != nil { + return nil, fmt.Errorf("%v: error converting YAML to JSON: %v", path, err) + } + return loadJSON(path, bs, m) +} + +func makeDir(path []string, x interface{}) (map[string]interface{}, bool) { + if len(path) == 0 { + obj, ok := x.(map[string]interface{}) + if !ok { + return nil, false + } + return obj, true + } + return makeDir(path[:len(path)-1], map[string]interface{}{path[len(path)-1]: x}) +} diff --git a/vendor/github.com/open-policy-agent/opa/metrics/metrics.go b/vendor/github.com/open-policy-agent/opa/metrics/metrics.go new file mode 100644 index 000000000..70677e123 --- /dev/null +++ b/vendor/github.com/open-policy-agent/opa/metrics/metrics.go @@ -0,0 +1,282 @@ +// Copyright 2017 The OPA Authors. All rights reserved. +// Use of this source code is governed by an Apache2 +// license that can be found in the LICENSE file. + +// Package metrics contains helpers for performance metric management inside the policy engine. +package metrics + +import ( + "encoding/json" + "fmt" + "sort" + "strings" + "sync" + "sync/atomic" + "time" + + go_metrics "github.com/rcrowley/go-metrics" +) + +// Well-known metric names. +const ( + ServerHandler = "server_handler" + ServerQueryCacheHit = "server_query_cache_hit" + RegoQueryCompile = "rego_query_compile" + RegoQueryEval = "rego_query_eval" + RegoQueryParse = "rego_query_parse" + RegoModuleParse = "rego_module_parse" + RegoDataParse = "rego_data_parse" + RegoModuleCompile = "rego_module_compile" + RegoPartialEval = "rego_partial_eval" + RegoInputParse = "rego_input_parse" + RegoLoadFiles = "rego_load_files" + RegoLoadBundles = "rego_load_bundles" +) + +// Info contains attributes describing the underlying metrics provider. +type Info struct { + Name string `json:"name"` // name is a unique human-readable identifier for the provider. +} + +// Metrics defines the interface for a collection of performance metrics in the +// policy engine. +type Metrics interface { + Info() Info + Timer(name string) Timer + Histogram(name string) Histogram + Counter(name string) Counter + All() map[string]interface{} + Clear() + json.Marshaler +} + +type metrics struct { + mtx sync.Mutex + timers map[string]Timer + histograms map[string]Histogram + counters map[string]Counter +} + +// New returns a new Metrics object. +func New() Metrics { + m := &metrics{} + m.Clear() + return m +} + +type metric struct { + Key string + Value interface{} +} + +func (m *metrics) Info() Info { + return Info{ + Name: "", + } +} + +func (m *metrics) String() string { + + all := m.All() + sorted := make([]metric, 0, len(all)) + + for key, value := range all { + sorted = append(sorted, metric{ + Key: key, + Value: value, + }) + } + + sort.Slice(sorted, func(i, j int) bool { + return sorted[i].Key < sorted[j].Key + }) + + buf := make([]string, len(sorted)) + for i := range sorted { + buf[i] = fmt.Sprintf("%v:%v", sorted[i].Key, sorted[i].Value) + } + + return strings.Join(buf, " ") +} + +func (m *metrics) MarshalJSON() ([]byte, error) { + return json.Marshal(m.All()) +} + +func (m *metrics) Timer(name string) Timer { + m.mtx.Lock() + defer m.mtx.Unlock() + t, ok := m.timers[name] + if !ok { + t = &timer{} + m.timers[name] = t + } + return t +} + +func (m *metrics) Histogram(name string) Histogram { + m.mtx.Lock() + defer m.mtx.Unlock() + h, ok := m.histograms[name] + if !ok { + h = newHistogram() + m.histograms[name] = h + } + return h +} + +func (m *metrics) Counter(name string) Counter { + m.mtx.Lock() + defer m.mtx.Unlock() + c, ok := m.counters[name] + if !ok { + zero := counter{} + c = &zero + m.counters[name] = c + } + return c +} + +func (m *metrics) All() map[string]interface{} { + m.mtx.Lock() + defer m.mtx.Unlock() + result := map[string]interface{}{} + for name, timer := range m.timers { + result[m.formatKey(name, timer)] = timer.Value() + } + for name, hist := range m.histograms { + result[m.formatKey(name, hist)] = hist.Value() + } + for name, cntr := range m.counters { + result[m.formatKey(name, cntr)] = cntr.Value() + } + return result +} + +func (m *metrics) Clear() { + m.mtx.Lock() + defer m.mtx.Unlock() + m.timers = map[string]Timer{} + m.histograms = map[string]Histogram{} + m.counters = map[string]Counter{} +} + +func (m *metrics) formatKey(name string, metrics interface{}) string { + switch metrics.(type) { + case Timer: + return "timer_" + name + "_ns" + case Histogram: + return "histogram_" + name + case Counter: + return "counter_" + name + default: + return name + } +} + +// Timer defines the interface for a restartable timer that accumulates elapsed +// time. +type Timer interface { + Value() interface{} + Int64() int64 + Start() + Stop() int64 +} + +type timer struct { + mtx sync.Mutex + start time.Time + value int64 +} + +func (t *timer) Start() { + t.mtx.Lock() + defer t.mtx.Unlock() + t.start = time.Now() +} + +func (t *timer) Stop() int64 { + t.mtx.Lock() + defer t.mtx.Unlock() + delta := time.Now().Sub(t.start).Nanoseconds() + t.value += delta + return delta +} + +func (t *timer) Value() interface{} { + return t.Int64() +} + +func (t *timer) Int64() int64 { + t.mtx.Lock() + defer t.mtx.Unlock() + return t.value +} + +// Histogram defines the interface for a histogram with hardcoded percentiles. +type Histogram interface { + Value() interface{} + Update(int64) +} + +type histogram struct { + hist go_metrics.Histogram // is thread-safe because of the underlying ExpDecaySample +} + +func newHistogram() Histogram { + // NOTE(tsandall): the reservoir size and alpha factor are taken from + // https://github.com/rcrowley/go-metrics. They may need to be tweaked in + // the future. + sample := go_metrics.NewExpDecaySample(1028, 0.015) + hist := go_metrics.NewHistogram(sample) + return &histogram{hist} +} + +func (h *histogram) Update(v int64) { + h.hist.Update(v) +} + +func (h *histogram) Value() interface{} { + values := map[string]interface{}{} + snap := h.hist.Snapshot() + percentiles := snap.Percentiles([]float64{ + 0.5, + 0.75, + 0.9, + 0.95, + 0.99, + 0.999, + 0.9999, + }) + values["count"] = snap.Count() + values["min"] = snap.Min() + values["max"] = snap.Max() + values["mean"] = snap.Mean() + values["stddev"] = snap.StdDev() + values["median"] = percentiles[0] + values["75%"] = percentiles[1] + values["90%"] = percentiles[2] + values["95%"] = percentiles[3] + values["99%"] = percentiles[4] + values["99.9%"] = percentiles[5] + values["99.99%"] = percentiles[6] + return values +} + +// Counter defines the interface for a monotonic increasing counter. +type Counter interface { + Value() interface{} + Incr() +} + +type counter struct { + c uint64 +} + +func (c *counter) Incr() { + atomic.AddUint64(&c.c, 1) +} + +func (c *counter) Value() interface{} { + return atomic.LoadUint64(&c.c) +} diff --git a/vendor/github.com/open-policy-agent/opa/rego/rego.go b/vendor/github.com/open-policy-agent/opa/rego/rego.go new file mode 100644 index 000000000..703a1524e --- /dev/null +++ b/vendor/github.com/open-policy-agent/opa/rego/rego.go @@ -0,0 +1,1997 @@ +// Copyright 2017 The OPA Authors. All rights reserved. +// Use of this source code is governed by an Apache2 +// license that can be found in the LICENSE file. + +// Package rego exposes high level APIs for evaluating Rego policies. +package rego + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "strings" + + "github.com/open-policy-agent/opa/loader" + "github.com/open-policy-agent/opa/types" + + "github.com/open-policy-agent/opa/bundle" + + "github.com/open-policy-agent/opa/ast" + "github.com/open-policy-agent/opa/internal/compiler/wasm" + "github.com/open-policy-agent/opa/internal/ir" + "github.com/open-policy-agent/opa/internal/planner" + "github.com/open-policy-agent/opa/internal/wasm/encoding" + "github.com/open-policy-agent/opa/metrics" + "github.com/open-policy-agent/opa/storage" + "github.com/open-policy-agent/opa/storage/inmem" + "github.com/open-policy-agent/opa/topdown" + "github.com/open-policy-agent/opa/util" +) + +const defaultPartialNamespace = "partial" + +// CompileResult represents the result of compiling a Rego query, zero or more +// Rego modules, and arbitrary contextual data into an executable. +type CompileResult struct { + Bytes []byte `json:"bytes"` +} + +// PartialQueries contains the queries and support modules produced by partial +// evaluation. +type PartialQueries struct { + Queries []ast.Body `json:"queries,omitempty"` + Support []*ast.Module `json:"modules,omitempty"` +} + +// PartialResult represents the result of partial evaluation. The result can be +// used to generate a new query that can be run when inputs are known. +type PartialResult struct { + compiler *ast.Compiler + store storage.Store + body ast.Body + builtinDecls map[string]*ast.Builtin + builtinFuncs map[string]*topdown.Builtin +} + +// Rego returns an object that can be evaluated to produce a query result. +func (pr PartialResult) Rego(options ...func(*Rego)) *Rego { + options = append(options, Compiler(pr.compiler), Store(pr.store), ParsedQuery(pr.body)) + r := New(options...) + + // Propagate any custom builtins. + for k, v := range pr.builtinDecls { + r.builtinDecls[k] = v + } + for k, v := range pr.builtinFuncs { + r.builtinFuncs[k] = v + } + return r +} + +// preparedQuery is a wrapper around a Rego object which has pre-processed +// state stored on it. Once prepared there are a more limited number of actions +// that can be taken with it. It will, however, be able to evaluate faster since +// it will not have to re-parse or compile as much. +type preparedQuery struct { + r *Rego + cfg *PrepareConfig +} + +// EvalContext defines the set of options allowed to be set at evaluation +// time. Any other options will need to be set on a new Rego object. +type EvalContext struct { + hasInput bool + rawInput *interface{} + parsedInput ast.Value + metrics metrics.Metrics + txn storage.Transaction + instrument bool + instrumentation *topdown.Instrumentation + partialNamespace string + tracers []topdown.Tracer + compiledQuery compiledQuery + unknowns []string + disableInlining []ast.Ref + parsedUnknowns []*ast.Term + indexing bool +} + +// EvalOption defines a function to set an option on an EvalConfig +type EvalOption func(*EvalContext) + +// EvalInput configures the input for a Prepared Query's evaluation +func EvalInput(input interface{}) EvalOption { + return func(e *EvalContext) { + e.rawInput = &input + e.hasInput = true + } +} + +// EvalParsedInput configures the input for a Prepared Query's evaluation +func EvalParsedInput(input ast.Value) EvalOption { + return func(e *EvalContext) { + e.parsedInput = input + e.hasInput = true + } +} + +// EvalMetrics configures the metrics for a Prepared Query's evaluation +func EvalMetrics(metric metrics.Metrics) EvalOption { + return func(e *EvalContext) { + e.metrics = metric + } +} + +// EvalTransaction configures the Transaction for a Prepared Query's evaluation +func EvalTransaction(txn storage.Transaction) EvalOption { + return func(e *EvalContext) { + e.txn = txn + } +} + +// EvalInstrument enables or disables instrumenting for a Prepared Query's evaluation +func EvalInstrument(instrument bool) EvalOption { + return func(e *EvalContext) { + e.instrument = instrument + } +} + +// EvalTracer configures a tracer for a Prepared Query's evaluation +func EvalTracer(tracer topdown.Tracer) EvalOption { + return func(e *EvalContext) { + if tracer != nil { + e.tracers = append(e.tracers, tracer) + } + } +} + +// EvalPartialNamespace returns an argument that sets the namespace to use for +// partial evaluation results. The namespace must be a valid package path +// component. +func EvalPartialNamespace(ns string) EvalOption { + return func(e *EvalContext) { + e.partialNamespace = ns + } +} + +// EvalUnknowns returns an argument that sets the values to treat as +// unknown during partial evaluation. +func EvalUnknowns(unknowns []string) EvalOption { + return func(e *EvalContext) { + e.unknowns = unknowns + } +} + +// EvalDisableInlining returns an argument that adds a set of paths to exclude from +// partial evaluation inlining. +func EvalDisableInlining(paths []ast.Ref) EvalOption { + return func(e *EvalContext) { + e.disableInlining = paths + } +} + +// EvalParsedUnknowns returns an argument that sets the values to treat +// as unknown during partial evaluation. +func EvalParsedUnknowns(unknowns []*ast.Term) EvalOption { + return func(e *EvalContext) { + e.parsedUnknowns = unknowns + } +} + +// EvalRuleIndexing will disable indexing optimizations for the +// evaluation. This should only be used when tracing in debug mode. +func EvalRuleIndexing(enabled bool) EvalOption { + return func(e *EvalContext) { + e.indexing = enabled + } +} + +func (pq preparedQuery) Modules() map[string]*ast.Module { + mods := make(map[string]*ast.Module) + + for name, mod := range pq.r.parsedModules { + mods[name] = mod + } + + for path, b := range pq.r.bundles { + for name, mod := range b.ParsedModules(path) { + mods[name] = mod + } + } + + return mods +} + +// newEvalContext creates a new EvalContext overlaying any EvalOptions over top +// the Rego object on the preparedQuery. The returned function should be called +// once the evaluation is complete to close any transactions that might have +// been opened. +func (pq preparedQuery) newEvalContext(ctx context.Context, options []EvalOption) (*EvalContext, func(context.Context), error) { + ectx := &EvalContext{ + hasInput: false, + rawInput: nil, + parsedInput: nil, + metrics: nil, + txn: nil, + instrument: false, + instrumentation: nil, + partialNamespace: pq.r.partialNamespace, + tracers: nil, + unknowns: pq.r.unknowns, + parsedUnknowns: pq.r.parsedUnknowns, + compiledQuery: compiledQuery{}, + indexing: true, + } + + for _, o := range options { + o(ectx) + } + + if ectx.metrics == nil { + ectx.metrics = metrics.New() + } + + if ectx.instrument { + ectx.instrumentation = topdown.NewInstrumentation(ectx.metrics) + } + + // Default to an empty "finish" function + finishFunc := func(context.Context) {} + + var err error + ectx.disableInlining, err = parseStringsToRefs(pq.r.disableInlining) + if err != nil { + return nil, finishFunc, err + } + + if ectx.txn == nil { + ectx.txn, err = pq.r.store.NewTransaction(ctx) + if err != nil { + return nil, finishFunc, err + } + finishFunc = func(ctx context.Context) { + pq.r.store.Abort(ctx, ectx.txn) + } + } + + // If we didn't get an input specified in the Eval options + // then fall back to the Rego object's input fields. + if !ectx.hasInput { + ectx.rawInput = pq.r.rawInput + ectx.parsedInput = pq.r.parsedInput + } + + if ectx.parsedInput == nil { + if ectx.rawInput == nil { + // Fall back to the original Rego objects input if none was specified + // Note that it could still be nil + ectx.rawInput = pq.r.rawInput + } + ectx.parsedInput, err = pq.r.parseRawInput(ectx.rawInput, ectx.metrics) + if err != nil { + return nil, finishFunc, err + } + } + + return ectx, finishFunc, nil +} + +// PreparedEvalQuery holds the prepared Rego state that has been pre-processed +// for subsequent evaluations. +type PreparedEvalQuery struct { + preparedQuery +} + +// Eval evaluates this PartialResult's Rego object with additional eval options +// and returns a ResultSet. +// If options are provided they will override the original Rego options respective value. +// The original Rego object transaction will *not* be re-used. A new transaction will be opened +// if one is not provided with an EvalOption. +func (pq PreparedEvalQuery) Eval(ctx context.Context, options ...EvalOption) (ResultSet, error) { + ectx, finish, err := pq.newEvalContext(ctx, options) + if err != nil { + return nil, err + } + defer finish(ctx) + + ectx.compiledQuery = pq.r.compiledQueries[evalQueryType] + + return pq.r.eval(ctx, ectx) +} + +// PreparedPartialQuery holds the prepared Rego state that has been pre-processed +// for partial evaluations. +type PreparedPartialQuery struct { + preparedQuery +} + +// Partial runs partial evaluation on the prepared query and returns the result. +// The original Rego object transaction will *not* be re-used. A new transaction will be opened +// if one is not provided with an EvalOption. +func (pq PreparedPartialQuery) Partial(ctx context.Context, options ...EvalOption) (*PartialQueries, error) { + ectx, finish, err := pq.newEvalContext(ctx, options) + if err != nil { + return nil, err + } + defer finish(ctx) + + ectx.compiledQuery = pq.r.compiledQueries[partialQueryType] + + return pq.r.partial(ctx, ectx) +} + +// Result defines the output of Rego evaluation. +type Result struct { + Expressions []*ExpressionValue `json:"expressions"` + Bindings Vars `json:"bindings,omitempty"` +} + +func newResult() Result { + return Result{ + Bindings: Vars{}, + } +} + +// Location defines a position in a Rego query or module. +type Location struct { + Row int `json:"row"` + Col int `json:"col"` +} + +// ExpressionValue defines the value of an expression in a Rego query. +type ExpressionValue struct { + Value interface{} `json:"value"` + Text string `json:"text"` + Location *Location `json:"location"` +} + +func newExpressionValue(expr *ast.Expr, value interface{}) *ExpressionValue { + result := &ExpressionValue{ + Value: value, + } + if expr.Location != nil { + result.Text = string(expr.Location.Text) + result.Location = &Location{ + Row: expr.Location.Row, + Col: expr.Location.Col, + } + } + return result +} + +func (ev *ExpressionValue) String() string { + return fmt.Sprint(ev.Value) +} + +// ResultSet represents a collection of output from Rego evaluation. An empty +// result set represents an undefined query. +type ResultSet []Result + +// Vars represents a collection of variable bindings. The keys are the variable +// names and the values are the binding values. +type Vars map[string]interface{} + +// WithoutWildcards returns a copy of v with wildcard variables removed. +func (v Vars) WithoutWildcards() Vars { + n := Vars{} + for k, v := range v { + if ast.Var(k).IsWildcard() || ast.Var(k).IsGenerated() { + continue + } + n[k] = v + } + return n +} + +// Errors represents a collection of errors returned when evaluating Rego. +type Errors []error + +func (errs Errors) Error() string { + if len(errs) == 0 { + return "no error" + } + if len(errs) == 1 { + return fmt.Sprintf("1 error occurred: %v", errs[0].Error()) + } + buf := []string{fmt.Sprintf("%v errors occurred", len(errs))} + for _, err := range errs { + buf = append(buf, err.Error()) + } + return strings.Join(buf, "\n") +} + +type compiledQuery struct { + query ast.Body + compiler ast.QueryCompiler +} + +type queryType int + +// Define a query type for each of the top level Rego +// API's that compile queries differently. +const ( + evalQueryType queryType = iota + partialResultQueryType queryType = iota + partialQueryType queryType = iota + compileQueryType queryType = iota +) + +type loadPaths struct { + paths []string + filter loader.Filter +} + +// Rego constructs a query and can be evaluated to obtain results. +type Rego struct { + query string + parsedQuery ast.Body + compiledQueries map[queryType]compiledQuery + pkg string + parsedPackage *ast.Package + imports []string + parsedImports []*ast.Import + rawInput *interface{} + parsedInput ast.Value + unknowns []string + parsedUnknowns []*ast.Term + disableInlining []string + partialNamespace string + modules []rawModule + parsedModules map[string]*ast.Module + compiler *ast.Compiler + store storage.Store + ownStore bool + txn storage.Transaction + metrics metrics.Metrics + tracers []topdown.Tracer + tracebuf *topdown.BufferTracer + trace bool + instrumentation *topdown.Instrumentation + instrument bool + capture map[*ast.Expr]ast.Var // map exprs to generated capture vars + termVarID int + dump io.Writer + runtime *ast.Term + builtinDecls map[string]*ast.Builtin + builtinFuncs map[string]*topdown.Builtin + unsafeBuiltins map[string]struct{} + loadPaths loadPaths + bundlePaths []string + bundles map[string]*bundle.Bundle +} + +// Function represents a built-in function that is callable in Rego. +type Function struct { + Name string + Decl *types.Function + Memoize bool +} + +// BuiltinContext contains additional attributes from the evaluator that +// built-in functions can use, e.g., the request context.Context, caches, etc. +type BuiltinContext = topdown.BuiltinContext + +type ( + // Builtin1 defines a built-in function that accepts 1 argument. + Builtin1 func(bctx BuiltinContext, op1 *ast.Term) (*ast.Term, error) + + // Builtin2 defines a built-in function that accepts 2 arguments. + Builtin2 func(bctx BuiltinContext, op1, op2 *ast.Term) (*ast.Term, error) + + // Builtin3 defines a built-in function that accepts 3 argument. + Builtin3 func(bctx BuiltinContext, op1, op2, op3 *ast.Term) (*ast.Term, error) + + // Builtin4 defines a built-in function that accepts 4 argument. + Builtin4 func(bctx BuiltinContext, op1, op2, op3, op4 *ast.Term) (*ast.Term, error) + + // BuiltinDyn defines a built-in function that accepts a list of arguments. + BuiltinDyn func(bctx BuiltinContext, terms []*ast.Term) (*ast.Term, error) +) + +// Function1 returns an option that adds a built-in function to the Rego object. +func Function1(decl *Function, f Builtin1) func(*Rego) { + return newFunction(decl, func(bctx BuiltinContext, terms []*ast.Term, iter func(*ast.Term) error) error { + result, err := memoize(decl, bctx, terms, func() (*ast.Term, error) { return f(bctx, terms[0]) }) + return finishFunction(decl.Name, bctx, result, err, iter) + }) +} + +// Function2 returns an option that adds a built-in function to the Rego object. +func Function2(decl *Function, f Builtin2) func(*Rego) { + return newFunction(decl, func(bctx BuiltinContext, terms []*ast.Term, iter func(*ast.Term) error) error { + result, err := memoize(decl, bctx, terms, func() (*ast.Term, error) { return f(bctx, terms[0], terms[1]) }) + return finishFunction(decl.Name, bctx, result, err, iter) + }) +} + +// Function3 returns an option that adds a built-in function to the Rego object. +func Function3(decl *Function, f Builtin3) func(*Rego) { + return newFunction(decl, func(bctx BuiltinContext, terms []*ast.Term, iter func(*ast.Term) error) error { + result, err := memoize(decl, bctx, terms, func() (*ast.Term, error) { return f(bctx, terms[0], terms[1], terms[2]) }) + return finishFunction(decl.Name, bctx, result, err, iter) + }) +} + +// Function4 returns an option that adds a built-in function to the Rego object. +func Function4(decl *Function, f Builtin4) func(*Rego) { + return newFunction(decl, func(bctx BuiltinContext, terms []*ast.Term, iter func(*ast.Term) error) error { + result, err := memoize(decl, bctx, terms, func() (*ast.Term, error) { return f(bctx, terms[0], terms[1], terms[2], terms[3]) }) + return finishFunction(decl.Name, bctx, result, err, iter) + }) +} + +// FunctionDyn returns an option that adds a built-in function to the Rego object. +func FunctionDyn(decl *Function, f BuiltinDyn) func(*Rego) { + return newFunction(decl, func(bctx BuiltinContext, terms []*ast.Term, iter func(*ast.Term) error) error { + result, err := memoize(decl, bctx, terms, func() (*ast.Term, error) { return f(bctx, terms) }) + return finishFunction(decl.Name, bctx, result, err, iter) + }) +} + +// FunctionDecl returns an option that adds a custom-built-in function +// __declaration__. NO implementation is provided. This is used for +// non-interpreter execution envs (e.g., Wasm). +func FunctionDecl(decl *Function) func(*Rego) { + return newDecl(decl) +} + +func newDecl(decl *Function) func(*Rego) { + return func(r *Rego) { + r.builtinDecls[decl.Name] = &ast.Builtin{ + Name: decl.Name, + Decl: decl.Decl, + } + } +} + +type memo struct { + term *ast.Term + err error +} + +type memokey string + +func memoize(decl *Function, bctx BuiltinContext, terms []*ast.Term, ifEmpty func() (*ast.Term, error)) (*ast.Term, error) { + + if !decl.Memoize { + return ifEmpty() + } + + // NOTE(tsandall): we assume memoization is applied to infrequent built-in + // calls that do things like fetch data from remote locations. As such, + // converting the terms to strings is acceptable for now. + var b strings.Builder + if _, err := b.WriteString(decl.Name); err != nil { + return nil, err + } + + // The term slice _may_ include an output term depending on how the caller + // referred to the built-in function. Only use the arguments as the cache + // key. Unification ensures we don't get false positive matches. + for i := 0; i < len(decl.Decl.Args()); i++ { + if _, err := b.WriteString(terms[i].String()); err != nil { + return nil, err + } + } + + key := memokey(b.String()) + hit, ok := bctx.Cache.Get(key) + var m memo + if ok { + m = hit.(memo) + } else { + m.term, m.err = ifEmpty() + bctx.Cache.Put(key, m) + } + + return m.term, m.err +} + +// Dump returns an argument that sets the writer to dump debugging information to. +func Dump(w io.Writer) func(r *Rego) { + return func(r *Rego) { + r.dump = w + } +} + +// Query returns an argument that sets the Rego query. +func Query(q string) func(r *Rego) { + return func(r *Rego) { + r.query = q + } +} + +// ParsedQuery returns an argument that sets the Rego query. +func ParsedQuery(q ast.Body) func(r *Rego) { + return func(r *Rego) { + r.parsedQuery = q + } +} + +// Package returns an argument that sets the Rego package on the query's +// context. +func Package(p string) func(r *Rego) { + return func(r *Rego) { + r.pkg = p + } +} + +// ParsedPackage returns an argument that sets the Rego package on the query's +// context. +func ParsedPackage(pkg *ast.Package) func(r *Rego) { + return func(r *Rego) { + r.parsedPackage = pkg + } +} + +// Imports returns an argument that adds a Rego import to the query's context. +func Imports(p []string) func(r *Rego) { + return func(r *Rego) { + r.imports = append(r.imports, p...) + } +} + +// ParsedImports returns an argument that adds Rego imports to the query's +// context. +func ParsedImports(imp []*ast.Import) func(r *Rego) { + return func(r *Rego) { + r.parsedImports = append(r.parsedImports, imp...) + } +} + +// Input returns an argument that sets the Rego input document. Input should be +// a native Go value representing the input document. +func Input(x interface{}) func(r *Rego) { + return func(r *Rego) { + r.rawInput = &x + } +} + +// ParsedInput returns an argument that sets the Rego input document. +func ParsedInput(x ast.Value) func(r *Rego) { + return func(r *Rego) { + r.parsedInput = x + } +} + +// Unknowns returns an argument that sets the values to treat as unknown during +// partial evaluation. +func Unknowns(unknowns []string) func(r *Rego) { + return func(r *Rego) { + r.unknowns = unknowns + } +} + +// ParsedUnknowns returns an argument that sets the values to treat as unknown +// during partial evaluation. +func ParsedUnknowns(unknowns []*ast.Term) func(r *Rego) { + return func(r *Rego) { + r.parsedUnknowns = unknowns + } +} + +// DisableInlining adds a set of paths to exclude from partial evaluation inlining. +func DisableInlining(paths []string) func(r *Rego) { + return func(r *Rego) { + r.disableInlining = paths + } +} + +// PartialNamespace returns an argument that sets the namespace to use for +// partial evaluation results. The namespace must be a valid package path +// component. +func PartialNamespace(ns string) func(r *Rego) { + return func(r *Rego) { + r.partialNamespace = ns + } +} + +// Module returns an argument that adds a Rego module. +func Module(filename, input string) func(r *Rego) { + return func(r *Rego) { + r.modules = append(r.modules, rawModule{ + filename: filename, + module: input, + }) + } +} + +// ParsedModule returns an argument that adds a parsed Rego module. If a string +// module with the same filename name is added, it will override the parsed +// module. +func ParsedModule(module *ast.Module) func(*Rego) { + return func(r *Rego) { + var filename string + if module.Package.Location != nil { + filename = module.Package.Location.File + } else { + filename = fmt.Sprintf("module_%p.rego", module) + } + r.parsedModules[filename] = module + } +} + +// Load returns an argument that adds a filesystem path to load data +// and Rego modules from. Any file with a *.rego, *.yaml, or *.json +// extension will be loaded. The path can be either a directory or file, +// directories are loaded recursively. The optional ignore string patterns +// can be used to filter which files are used. +// The Load option can only be used once. +// Note: Loading files will require a write transaction on the store. +func Load(paths []string, filter loader.Filter) func(r *Rego) { + return func(r *Rego) { + r.loadPaths = loadPaths{paths, filter} + } +} + +// LoadBundle returns an argument that adds a filesystem path to load +// a bundle from. The path can be a compressed bundle file or a directory +// to be loaded as a bundle. +// Note: Loading bundles will require a write transaction on the store. +func LoadBundle(path string) func(r *Rego) { + return func(r *Rego) { + r.bundlePaths = append(r.bundlePaths, path) + } +} + +// ParsedBundle returns an argument that adds a bundle to be loaded. +func ParsedBundle(name string, b *bundle.Bundle) func(r *Rego) { + return func(r *Rego) { + r.bundles[name] = b + } +} + +// Compiler returns an argument that sets the Rego compiler. +func Compiler(c *ast.Compiler) func(r *Rego) { + return func(r *Rego) { + r.compiler = c + } +} + +// Store returns an argument that sets the policy engine's data storage layer. +// +// If using the Load, LoadBundle, or ParsedBundle options then a transaction +// must also be provided via the Transaction() option. After loading files +// or bundles the transaction should be aborted or committed. +func Store(s storage.Store) func(r *Rego) { + return func(r *Rego) { + r.store = s + } +} + +// Transaction returns an argument that sets the transaction to use for storage +// layer operations. +// +// Requires the store associated with the transaction to be provided via the +// Store() option. If using Load(), LoadBundle(), or ParsedBundle() options +// the transaction will likely require write params. +func Transaction(txn storage.Transaction) func(r *Rego) { + return func(r *Rego) { + r.txn = txn + } +} + +// Metrics returns an argument that sets the metrics collection. +func Metrics(m metrics.Metrics) func(r *Rego) { + return func(r *Rego) { + r.metrics = m + } +} + +// Instrument returns an argument that enables instrumentation for diagnosing +// performance issues. +func Instrument(yes bool) func(r *Rego) { + return func(r *Rego) { + r.instrument = yes + } +} + +// Trace returns an argument that enables tracing on r. +func Trace(yes bool) func(r *Rego) { + return func(r *Rego) { + r.trace = yes + } +} + +// Tracer returns an argument that adds a query tracer to r. +func Tracer(t topdown.Tracer) func(r *Rego) { + return func(r *Rego) { + if t != nil { + r.tracers = append(r.tracers, t) + } + } +} + +// Runtime returns an argument that sets the runtime data to provide to the +// evaluation engine. +func Runtime(term *ast.Term) func(r *Rego) { + return func(r *Rego) { + r.runtime = term + } +} + +// PrintTrace is a helper function to write a human-readable version of the +// trace to the writer w. +func PrintTrace(w io.Writer, r *Rego) { + if r == nil || r.tracebuf == nil { + return + } + topdown.PrettyTrace(w, *r.tracebuf) +} + +// UnsafeBuiltins sets the built-in functions to treat as unsafe and not allow. +// This option is ignored for module compilation if the caller supplies the +// compiler. This option is always honored for query compilation. Provide an +// empty (non-nil) map to disable checks on queries. +func UnsafeBuiltins(unsafeBuiltins map[string]struct{}) func(r *Rego) { + return func(r *Rego) { + r.unsafeBuiltins = unsafeBuiltins + } +} + +// New returns a new Rego object. +func New(options ...func(r *Rego)) *Rego { + + r := &Rego{ + parsedModules: map[string]*ast.Module{}, + capture: map[*ast.Expr]ast.Var{}, + compiledQueries: map[queryType]compiledQuery{}, + builtinDecls: map[string]*ast.Builtin{}, + builtinFuncs: map[string]*topdown.Builtin{}, + bundles: map[string]*bundle.Bundle{}, + } + + for _, option := range options { + option(r) + } + + if r.compiler == nil { + r.compiler = ast.NewCompiler(). + WithUnsafeBuiltins(r.unsafeBuiltins). + WithBuiltins(r.builtinDecls) + } + + if r.store == nil { + r.store = inmem.New() + r.ownStore = true + } else { + r.ownStore = false + } + + if r.metrics == nil { + r.metrics = metrics.New() + } + + if r.instrument { + r.instrumentation = topdown.NewInstrumentation(r.metrics) + r.compiler.WithMetrics(r.metrics) + } + + if r.trace { + r.tracebuf = topdown.NewBufferTracer() + r.tracers = append(r.tracers, r.tracebuf) + } + + if r.partialNamespace == "" { + r.partialNamespace = defaultPartialNamespace + } + + return r +} + +// Eval evaluates this Rego object and returns a ResultSet. +func (r *Rego) Eval(ctx context.Context) (ResultSet, error) { + var err error + var txnClose transactionCloser + r.txn, txnClose, err = r.getTxn(ctx) + if err != nil { + return nil, err + } + + pq, err := r.PrepareForEval(ctx) + if err != nil { + txnClose(ctx, err) // Ignore error + return nil, err + } + + evalArgs := []EvalOption{ + EvalTransaction(r.txn), + EvalMetrics(r.metrics), + EvalInstrument(r.instrument), + } + + for _, t := range r.tracers { + evalArgs = append(evalArgs, EvalTracer(t)) + } + + rs, err := pq.Eval(ctx, evalArgs...) + txnErr := txnClose(ctx, err) // Always call closer + if err == nil { + err = txnErr + } + return rs, err +} + +// PartialEval has been deprecated and renamed to PartialResult. +func (r *Rego) PartialEval(ctx context.Context) (PartialResult, error) { + return r.PartialResult(ctx) +} + +// PartialResult partially evaluates this Rego object and returns a PartialResult. +func (r *Rego) PartialResult(ctx context.Context) (PartialResult, error) { + var err error + var txnClose transactionCloser + r.txn, txnClose, err = r.getTxn(ctx) + if err != nil { + return PartialResult{}, err + } + + pq, err := r.PrepareForEval(ctx, WithPartialEval()) + txnErr := txnClose(ctx, err) // Always call closer + if err != nil { + return PartialResult{}, err + } + if txnErr != nil { + return PartialResult{}, txnErr + } + + pr := PartialResult{ + compiler: pq.r.compiler, + store: pq.r.store, + body: pq.r.parsedQuery, + builtinDecls: pq.r.builtinDecls, + builtinFuncs: pq.r.builtinFuncs, + } + + return pr, nil +} + +// Partial runs partial evaluation on r and returns the result. +func (r *Rego) Partial(ctx context.Context) (*PartialQueries, error) { + var err error + var txnClose transactionCloser + r.txn, txnClose, err = r.getTxn(ctx) + if err != nil { + return nil, err + } + + pq, err := r.PrepareForPartial(ctx) + if err != nil { + txnClose(ctx, err) // Ignore error + return nil, err + } + + evalArgs := []EvalOption{ + EvalTransaction(r.txn), + EvalMetrics(r.metrics), + EvalInstrument(r.instrument), + } + + for _, t := range r.tracers { + evalArgs = append(evalArgs, EvalTracer(t)) + } + + pqs, err := pq.Partial(ctx, evalArgs...) + txnErr := txnClose(ctx, err) // Always call closer + if err == nil { + err = txnErr + } + return pqs, err +} + +// CompileOption defines a function to set options on Compile calls. +type CompileOption func(*CompileContext) + +// CompileContext contains options for Compile calls. +type CompileContext struct { + partial bool +} + +// CompilePartial defines an option to control whether partial evaluation is run +// before the query is planned and compiled. +func CompilePartial(yes bool) CompileOption { + return func(cfg *CompileContext) { + cfg.partial = yes + } +} + +// Compile returns a compiled policy query. +func (r *Rego) Compile(ctx context.Context, opts ...CompileOption) (*CompileResult, error) { + + var cfg CompileContext + + for _, opt := range opts { + opt(&cfg) + } + + var queries []ast.Body + var modules []*ast.Module + + if cfg.partial { + + pq, err := r.Partial(ctx) + if err != nil { + return nil, err + } + if r.dump != nil { + if len(pq.Queries) != 0 { + msg := fmt.Sprintf("QUERIES (%d total):", len(pq.Queries)) + fmt.Fprintln(r.dump, msg) + fmt.Fprintln(r.dump, strings.Repeat("-", len(msg))) + for i := range pq.Queries { + fmt.Println(pq.Queries[i]) + } + fmt.Fprintln(r.dump) + } + if len(pq.Support) != 0 { + msg := fmt.Sprintf("SUPPORT (%d total):", len(pq.Support)) + fmt.Fprintln(r.dump, msg) + fmt.Fprintln(r.dump, strings.Repeat("-", len(msg))) + for i := range pq.Support { + fmt.Println(pq.Support[i]) + } + fmt.Fprintln(r.dump) + } + } + + queries = pq.Queries + modules = pq.Support + + for _, module := range r.compiler.Modules { + modules = append(modules, module) + } + } else { + var err error + // If creating a new transacation it should be closed before calling the + // planner to avoid holding open the transaction longer than needed. + // + // TODO(tsandall): in future, planner could make use of store, in which + // case this will need to change. + var txnClose transactionCloser + r.txn, txnClose, err = r.getTxn(ctx) + if err != nil { + return nil, err + } + + err = r.prepare(ctx, compileQueryType, nil) + txnErr := txnClose(ctx, err) // Always call closer + if err != nil { + return nil, err + } + if txnErr != nil { + return nil, err + } + + for _, module := range r.compiler.Modules { + modules = append(modules, module) + } + + queries = []ast.Body{r.compiledQueries[compileQueryType].query} + } + + decls := make(map[string]*ast.Builtin, len(r.builtinDecls)+len(ast.BuiltinMap)) + + for k, v := range ast.BuiltinMap { + decls[k] = v + } + + for k, v := range r.builtinDecls { + decls[k] = v + } + + policy, err := planner.New(). + WithQueries(queries). + WithModules(modules). + WithRewrittenVars(r.compiledQueries[compileQueryType].compiler.RewrittenVars()). + WithBuiltinDecls(decls). + Plan() + if err != nil { + return nil, err + } + + if r.dump != nil { + fmt.Fprintln(r.dump, "PLAN:") + fmt.Fprintln(r.dump, "-----") + ir.Pretty(r.dump, policy) + fmt.Fprintln(r.dump) + } + + m, err := wasm.New().WithPolicy(policy).Compile() + if err != nil { + return nil, err + } + + var out bytes.Buffer + + if err := encoding.WriteModule(&out, m); err != nil { + return nil, err + } + + result := &CompileResult{ + Bytes: out.Bytes(), + } + + return result, nil +} + +// PrepareOption defines a function to set an option to control +// the behavior of the Prepare call. +type PrepareOption func(*PrepareConfig) + +// PrepareConfig holds settings to control the behavior of the +// Prepare call. +type PrepareConfig struct { + doPartialEval bool + disableInlining *[]string +} + +// WithPartialEval configures an option for PrepareForEval +// which will have it perform partial evaluation while preparing +// the query (similar to rego.Rego#PartialResult) +func WithPartialEval() PrepareOption { + return func(p *PrepareConfig) { + p.doPartialEval = true + } +} + +// WithNoInline adds a set of paths to exclude from partial evaluation inlining. +func WithNoInline(paths []string) PrepareOption { + return func(p *PrepareConfig) { + p.disableInlining = &paths + } +} + +// PrepareForEval will parse inputs, modules, and query arguments in preparation +// of evaluating them. +func (r *Rego) PrepareForEval(ctx context.Context, opts ...PrepareOption) (PreparedEvalQuery, error) { + if !r.hasQuery() { + return PreparedEvalQuery{}, fmt.Errorf("cannot evaluate empty query") + } + + pCfg := &PrepareConfig{} + for _, o := range opts { + o(pCfg) + } + + var err error + var txnClose transactionCloser + r.txn, txnClose, err = r.getTxn(ctx) + if err != nil { + return PreparedEvalQuery{}, err + } + + // If the caller wanted to do partial evaluation as part of preparation + // do it now and use the new Rego object. + if pCfg.doPartialEval { + + pr, err := r.partialResult(ctx, pCfg) + if err != nil { + txnClose(ctx, err) // Ignore error + return PreparedEvalQuery{}, err + } + + // Prepare the new query using the result of partial evaluation + pq, err := pr.Rego(Transaction(r.txn)).PrepareForEval(ctx) + txnErr := txnClose(ctx, err) + if err != nil { + return pq, err + } + return pq, txnErr + } + + err = r.prepare(ctx, evalQueryType, []extraStage{ + { + after: "ResolveRefs", + stage: ast.QueryCompilerStageDefinition{ + Name: "RewriteToCaptureValue", + MetricName: "query_compile_stage_rewrite_to_capture_value", + Stage: r.rewriteQueryToCaptureValue, + }, + }, + }) + txnErr := txnClose(ctx, err) // Always call closer + if err != nil { + return PreparedEvalQuery{}, err + } + if txnErr != nil { + return PreparedEvalQuery{}, txnErr + } + + return PreparedEvalQuery{preparedQuery{r, pCfg}}, err +} + +// PrepareForPartial will parse inputs, modules, and query arguments in preparation +// of partially evaluating them. +func (r *Rego) PrepareForPartial(ctx context.Context, opts ...PrepareOption) (PreparedPartialQuery, error) { + if !r.hasQuery() { + return PreparedPartialQuery{}, fmt.Errorf("cannot evaluate empty query") + } + + pCfg := &PrepareConfig{} + for _, o := range opts { + o(pCfg) + } + + var err error + var txnClose transactionCloser + r.txn, txnClose, err = r.getTxn(ctx) + if err != nil { + return PreparedPartialQuery{}, err + } + + err = r.prepare(ctx, partialQueryType, []extraStage{ + { + after: "CheckSafety", + stage: ast.QueryCompilerStageDefinition{ + Name: "RewriteEquals", + MetricName: "query_compile_stage_rewrite_equals", + Stage: r.rewriteEqualsForPartialQueryCompile, + }, + }, + }) + txnErr := txnClose(ctx, err) // Always call closer + if err != nil { + return PreparedPartialQuery{}, err + } + if txnErr != nil { + return PreparedPartialQuery{}, txnErr + } + return PreparedPartialQuery{preparedQuery{r, pCfg}}, err +} + +func (r *Rego) prepare(ctx context.Context, qType queryType, extras []extraStage) error { + var err error + + r.parsedInput, err = r.parseInput() + if err != nil { + return err + } + + err = r.loadFiles(ctx, r.txn, r.metrics) + if err != nil { + return err + } + + err = r.loadBundles(ctx, r.txn, r.metrics) + if err != nil { + return err + } + + err = r.parseModules(ctx, r.txn, r.metrics) + if err != nil { + return err + } + + // Compile the modules *before* the query, else functions + // defined in the module won't be found... + err = r.compileModules(ctx, r.txn, r.metrics) + if err != nil { + return err + } + + r.parsedQuery, err = r.parseQuery(r.metrics) + if err != nil { + return err + } + + err = r.compileAndCacheQuery(qType, r.parsedQuery, r.metrics, extras) + if err != nil { + return err + } + + return nil +} + +func (r *Rego) parseModules(ctx context.Context, txn storage.Transaction, m metrics.Metrics) error { + if len(r.modules) == 0 { + return nil + } + + m.Timer(metrics.RegoModuleParse).Start() + defer m.Timer(metrics.RegoModuleParse).Stop() + var errs Errors + + // Parse any modules in the are saved to the store, but only if + // another compile step is going to occur (ie. we have parsed modules + // that need to be compiled). + ids, err := r.store.ListPolicies(ctx, txn) + if err != nil { + return err + } + + for _, id := range ids { + // if it is already on the compiler we're using + // then don't bother to re-parse it from source + if _, haveMod := r.compiler.Modules[id]; haveMod { + continue + } + + bs, err := r.store.GetPolicy(ctx, txn, id) + if err != nil { + return err + } + + parsed, err := ast.ParseModule(id, string(bs)) + if err != nil { + errs = append(errs, err) + } + + r.parsedModules[id] = parsed + } + + // Parse any passed in as arguments to the Rego object + for _, module := range r.modules { + p, err := module.Parse() + if err != nil { + errs = append(errs, err) + } + r.parsedModules[module.filename] = p + } + + if len(errs) > 0 { + return errs + } + + return nil +} + +func (r *Rego) loadFiles(ctx context.Context, txn storage.Transaction, m metrics.Metrics) error { + if len(r.loadPaths.paths) == 0 { + return nil + } + + m.Timer(metrics.RegoLoadFiles).Start() + defer m.Timer(metrics.RegoLoadFiles).Stop() + + result, err := loader.NewFileLoader().WithMetrics(m).Filtered(r.loadPaths.paths, r.loadPaths.filter) + if err != nil { + return err + } + for name, mod := range result.Modules { + r.parsedModules[name] = mod.Parsed + } + + if len(result.Documents) > 0 { + err = r.store.Write(ctx, txn, storage.AddOp, storage.Path{}, result.Documents) + if err != nil { + return err + } + } + return nil +} + +func (r *Rego) loadBundles(ctx context.Context, txn storage.Transaction, m metrics.Metrics) error { + if len(r.bundlePaths) == 0 { + return nil + } + + m.Timer(metrics.RegoLoadBundles).Start() + defer m.Timer(metrics.RegoLoadBundles).Stop() + + for _, path := range r.bundlePaths { + bndl, err := loader.NewFileLoader().WithMetrics(m).AsBundle(path) + if err != nil { + return fmt.Errorf("loading error: %s", err) + } + r.bundles[path] = bndl + } + return nil +} + +func (r *Rego) parseInput() (ast.Value, error) { + if r.parsedInput != nil { + return r.parsedInput, nil + } + return r.parseRawInput(r.rawInput, r.metrics) +} + +func (r *Rego) parseRawInput(rawInput *interface{}, m metrics.Metrics) (ast.Value, error) { + var input ast.Value + + if rawInput == nil { + return input, nil + } + + m.Timer(metrics.RegoInputParse).Start() + defer m.Timer(metrics.RegoInputParse).Stop() + + rawPtr := util.Reference(rawInput) + + // roundtrip through json: this turns slices (e.g. []string, []bool) into + // []interface{}, the only array type ast.InterfaceToValue can work with + if err := util.RoundTrip(rawPtr); err != nil { + return nil, err + } + + return ast.InterfaceToValue(*rawPtr) +} + +func (r *Rego) parseQuery(m metrics.Metrics) (ast.Body, error) { + if r.parsedQuery != nil { + return r.parsedQuery, nil + } + + m.Timer(metrics.RegoQueryParse).Start() + defer m.Timer(metrics.RegoQueryParse).Stop() + + return ast.ParseBody(r.query) +} + +func (r *Rego) compileModules(ctx context.Context, txn storage.Transaction, m metrics.Metrics) error { + + // Only compile again if there are new modules. + if len(r.bundles) > 0 || len(r.parsedModules) > 0 { + + // The bundle.Activate call will activate any bundles passed in + // (ie compile + handle data store changes), and include any of + // the additional modules passed in. If no bundles are provided + // it will only compile the passed in modules. + // Use this as the single-point of compiling everything only a + // single time. + opts := &bundle.ActivateOpts{ + Ctx: ctx, + Store: r.store, + Txn: txn, + Compiler: r.compiler.WithPathConflictsCheck(storage.NonEmpty(ctx, r.store, txn)), + Metrics: m, + Bundles: r.bundles, + ExtraModules: r.parsedModules, + } + err := bundle.Activate(opts) + if err != nil { + return err + } + } + return nil +} + +func (r *Rego) compileAndCacheQuery(qType queryType, query ast.Body, m metrics.Metrics, extras []extraStage) error { + m.Timer(metrics.RegoQueryCompile).Start() + defer m.Timer(metrics.RegoQueryCompile).Stop() + + cachedQuery, ok := r.compiledQueries[qType] + if ok && cachedQuery.query != nil && cachedQuery.compiler != nil { + return nil + } + + qc, compiled, err := r.compileQuery(query, m, extras) + if err != nil { + return err + } + + // cache the query for future use + r.compiledQueries[qType] = compiledQuery{ + query: compiled, + compiler: qc, + } + return nil +} + +func (r *Rego) compileQuery(query ast.Body, m metrics.Metrics, extras []extraStage) (ast.QueryCompiler, ast.Body, error) { + var pkg *ast.Package + + if r.pkg != "" { + var err error + pkg, err = ast.ParsePackage(fmt.Sprintf("package %v", r.pkg)) + if err != nil { + return nil, nil, err + } + } else { + pkg = r.parsedPackage + } + + imports := r.parsedImports + + if len(r.imports) > 0 { + s := make([]string, len(r.imports)) + for i := range r.imports { + s[i] = fmt.Sprintf("import %v", r.imports[i]) + } + parsed, err := ast.ParseImports(strings.Join(s, "\n")) + if err != nil { + return nil, nil, err + } + imports = append(imports, parsed...) + } + + qctx := ast.NewQueryContext(). + WithPackage(pkg). + WithImports(imports) + + qc := r.compiler.QueryCompiler(). + WithContext(qctx). + WithUnsafeBuiltins(r.unsafeBuiltins) + + for _, extra := range extras { + qc = qc.WithStageAfter(extra.after, extra.stage) + } + + compiled, err := qc.Compile(query) + + return qc, compiled, err + +} + +func (r *Rego) eval(ctx context.Context, ectx *EvalContext) (ResultSet, error) { + + q := topdown.NewQuery(ectx.compiledQuery.query). + WithQueryCompiler(ectx.compiledQuery.compiler). + WithCompiler(r.compiler). + WithStore(r.store). + WithTransaction(ectx.txn). + WithBuiltins(r.builtinFuncs). + WithMetrics(ectx.metrics). + WithInstrumentation(ectx.instrumentation). + WithRuntime(r.runtime). + WithIndexing(ectx.indexing) + + for i := range ectx.tracers { + q = q.WithTracer(ectx.tracers[i]) + } + + if ectx.parsedInput != nil { + q = q.WithInput(ast.NewTerm(ectx.parsedInput)) + } + + // Cancel query if context is cancelled or deadline is reached. + c := topdown.NewCancel() + q = q.WithCancel(c) + exit := make(chan struct{}) + defer close(exit) + go waitForDone(ctx, exit, func() { + c.Cancel() + }) + + rewritten := ectx.compiledQuery.compiler.RewrittenVars() + var rs ResultSet + err := q.Iter(ctx, func(qr topdown.QueryResult) error { + result := newResult() + for k := range qr { + v, err := ast.JSON(qr[k].Value) + if err != nil { + return err + } + if rw, ok := rewritten[k]; ok { + k = rw + } + if isTermVar(k) || k.IsGenerated() || k.IsWildcard() { + continue + } + result.Bindings[string(k)] = v + } + for _, expr := range ectx.compiledQuery.query { + if expr.Generated { + continue + } + if k, ok := r.capture[expr]; ok { + v, err := ast.JSON(qr[k].Value) + if err != nil { + return err + } + result.Expressions = append(result.Expressions, newExpressionValue(expr, v)) + } else { + result.Expressions = append(result.Expressions, newExpressionValue(expr, true)) + } + } + rs = append(rs, result) + return nil + }) + + if err != nil { + return nil, err + } + + if len(rs) == 0 { + return nil, nil + } + + return rs, nil +} + +func (r *Rego) partialResult(ctx context.Context, pCfg *PrepareConfig) (PartialResult, error) { + + err := r.prepare(ctx, partialResultQueryType, []extraStage{ + { + after: "ResolveRefs", + stage: ast.QueryCompilerStageDefinition{ + Name: "RewriteForPartialEval", + MetricName: "query_compile_stage_rewrite_for_partial_eval", + Stage: r.rewriteQueryForPartialEval, + }, + }, + }) + if err != nil { + return PartialResult{}, err + } + + ectx := &EvalContext{ + parsedInput: r.parsedInput, + metrics: r.metrics, + txn: r.txn, + partialNamespace: r.partialNamespace, + tracers: r.tracers, + compiledQuery: r.compiledQueries[partialResultQueryType], + instrumentation: r.instrumentation, + indexing: true, + } + + disableInlining := r.disableInlining + + if pCfg.disableInlining != nil { + disableInlining = *pCfg.disableInlining + } + + ectx.disableInlining, err = parseStringsToRefs(disableInlining) + if err != nil { + return PartialResult{}, err + } + + pq, err := r.partial(ctx, ectx) + if err != nil { + return PartialResult{}, err + } + + // Construct module for queries. + module := ast.MustParseModule("package " + ectx.partialNamespace) + module.Rules = make([]*ast.Rule, len(pq.Queries)) + for i, body := range pq.Queries { + module.Rules[i] = &ast.Rule{ + Head: ast.NewHead(ast.Var("__result__"), nil, ast.Wildcard), + Body: body, + Module: module, + } + } + + // Update compiler with partial evaluation output. + r.compiler.Modules["__partialresult__"] = module + for i, module := range pq.Support { + r.compiler.Modules[fmt.Sprintf("__partialsupport%d__", i)] = module + } + + r.metrics.Timer(metrics.RegoModuleCompile).Start() + r.compiler.Compile(r.compiler.Modules) + r.metrics.Timer(metrics.RegoModuleCompile).Stop() + + if r.compiler.Failed() { + return PartialResult{}, r.compiler.Errors + } + + result := PartialResult{ + compiler: r.compiler, + store: r.store, + body: ast.MustParseBody(fmt.Sprintf("data.%v.__result__", ectx.partialNamespace)), + builtinDecls: r.builtinDecls, + builtinFuncs: r.builtinFuncs, + } + + return result, nil +} + +func (r *Rego) partial(ctx context.Context, ectx *EvalContext) (*PartialQueries, error) { + + var unknowns []*ast.Term + + if ectx.parsedUnknowns != nil { + unknowns = ectx.parsedUnknowns + } else if ectx.unknowns != nil { + unknowns = make([]*ast.Term, len(ectx.unknowns)) + for i := range ectx.unknowns { + var err error + unknowns[i], err = ast.ParseTerm(ectx.unknowns[i]) + if err != nil { + return nil, err + } + } + } else { + // Use input document as unknown if caller has not specified any. + unknowns = []*ast.Term{ast.NewTerm(ast.InputRootRef)} + } + + // Check partial namespace to ensure it's valid. + if term, err := ast.ParseTerm(ectx.partialNamespace); err != nil { + return nil, err + } else if _, ok := term.Value.(ast.Var); !ok { + return nil, fmt.Errorf("bad partial namespace") + } + + q := topdown.NewQuery(ectx.compiledQuery.query). + WithQueryCompiler(ectx.compiledQuery.compiler). + WithCompiler(r.compiler). + WithStore(r.store). + WithTransaction(ectx.txn). + WithBuiltins(r.builtinFuncs). + WithMetrics(ectx.metrics). + WithInstrumentation(ectx.instrumentation). + WithUnknowns(unknowns). + WithDisableInlining(ectx.disableInlining). + WithRuntime(r.runtime). + WithIndexing(ectx.indexing) + + for i := range ectx.tracers { + q = q.WithTracer(ectx.tracers[i]) + } + + if ectx.parsedInput != nil { + q = q.WithInput(ast.NewTerm(ectx.parsedInput)) + } + + // Cancel query if context is cancelled or deadline is reached. + c := topdown.NewCancel() + q = q.WithCancel(c) + exit := make(chan struct{}) + defer close(exit) + go waitForDone(ctx, exit, func() { + c.Cancel() + }) + + queries, support, err := q.PartialRun(ctx) + if err != nil { + return nil, err + } + + pq := &PartialQueries{ + Queries: queries, + Support: support, + } + + return pq, nil +} + +func (r *Rego) rewriteQueryToCaptureValue(qc ast.QueryCompiler, query ast.Body) (ast.Body, error) { + + checkCapture := iteration(query) || len(query) > 1 + + for _, expr := range query { + + if expr.Negated { + continue + } + + if expr.IsAssignment() || expr.IsEquality() { + continue + } + + var capture *ast.Term + + // If the expression can be evaluated as a function, rewrite it to + // capture the return value. E.g., neq(1,2) becomes neq(1,2,x) but + // plus(1,2,x) does not get rewritten. + switch terms := expr.Terms.(type) { + case *ast.Term: + capture = r.generateTermVar() + expr.Terms = ast.Equality.Expr(terms, capture).Terms + r.capture[expr] = capture.Value.(ast.Var) + case []*ast.Term: + if r.compiler.GetArity(expr.Operator()) == len(terms)-1 { + capture = r.generateTermVar() + expr.Terms = append(terms, capture) + r.capture[expr] = capture.Value.(ast.Var) + } + } + + if capture != nil && checkCapture { + cpy := expr.Copy() + cpy.Terms = capture + cpy.Generated = true + cpy.With = nil + query.Append(cpy) + } + } + + return query, nil +} + +func (r *Rego) rewriteQueryForPartialEval(_ ast.QueryCompiler, query ast.Body) (ast.Body, error) { + if len(query) != 1 { + return nil, fmt.Errorf("partial evaluation requires single ref (not multiple expressions)") + } + + term, ok := query[0].Terms.(*ast.Term) + if !ok { + return nil, fmt.Errorf("partial evaluation requires ref (not expression)") + } + + ref, ok := term.Value.(ast.Ref) + if !ok { + return nil, fmt.Errorf("partial evaluation requires ref (not %v)", ast.TypeName(term.Value)) + } + + if !ref.IsGround() { + return nil, fmt.Errorf("partial evaluation requires ground ref") + } + + return ast.NewBody(ast.Equality.Expr(ast.Wildcard, term)), nil +} + +// rewriteEqualsForPartialQueryCompile will rewrite == to = in queries. Normally +// this wouldn't be done, except for handling queries with the `Partial` API +// where rewriting them can substantially simplify the result, and it is unlikely +// that the caller would need expression values. +func (r *Rego) rewriteEqualsForPartialQueryCompile(_ ast.QueryCompiler, query ast.Body) (ast.Body, error) { + doubleEq := ast.Equal.Ref() + unifyOp := ast.Equality.Ref() + ast.WalkExprs(query, func(x *ast.Expr) bool { + if x.IsCall() { + operator := x.Operator() + if operator.Equal(doubleEq) && len(x.Operands()) == 2 { + x.SetOperator(ast.NewTerm(unifyOp)) + } + } + return false + }) + return query, nil +} + +func (r *Rego) generateTermVar() *ast.Term { + r.termVarID++ + return ast.VarTerm(ast.WildcardPrefix + fmt.Sprintf("term%v", r.termVarID)) +} + +func (r Rego) hasQuery() bool { + return len(r.query) != 0 || len(r.parsedQuery) != 0 +} + +type transactionCloser func(ctx context.Context, err error) error + +// getTxn will conditionally create a read or write transaction suitable for +// the configured Rego object. The returned function should be used to close the txn +// regardless of status. +func (r *Rego) getTxn(ctx context.Context) (storage.Transaction, transactionCloser, error) { + + noopCloser := func(ctx context.Context, err error) error { + return nil // no-op default + } + + if r.txn != nil { + // Externally provided txn + return r.txn, noopCloser, nil + } + + // Create a new transaction.. + params := storage.TransactionParams{} + + // Bundles and data paths may require writing data files or manifests to storage + if len(r.bundles) > 0 || len(r.bundlePaths) > 0 || len(r.loadPaths.paths) > 0 { + + // If we were given a store we will *not* write to it, only do that on one + // which was created automatically on behalf of the user. + if !r.ownStore { + return nil, noopCloser, errors.New("unable to start write transaction when store was provided") + } + + params.Write = true + } + + txn, err := r.store.NewTransaction(ctx, params) + if err != nil { + return nil, noopCloser, err + } + + // Setup a closer function that will abort or commit as needed. + closer := func(ctx context.Context, txnErr error) error { + var err error + + if txnErr == nil && params.Write { + err = r.store.Commit(ctx, txn) + } else { + r.store.Abort(ctx, txn) + } + + // Clear the auto created transaction now that it is closed. + r.txn = nil + + return err + } + + return txn, closer, nil +} + +func isTermVar(v ast.Var) bool { + return strings.HasPrefix(string(v), ast.WildcardPrefix+"term") +} + +func waitForDone(ctx context.Context, exit chan struct{}, f func()) { + select { + case <-exit: + return + case <-ctx.Done(): + f() + return + } +} + +type rawModule struct { + filename string + module string +} + +func (m rawModule) Parse() (*ast.Module, error) { + return ast.ParseModule(m.filename, m.module) +} + +type extraStage struct { + after string + stage ast.QueryCompilerStageDefinition +} + +func iteration(x interface{}) bool { + + var stopped bool + + vis := ast.NewGenericVisitor(func(x interface{}) bool { + switch x := x.(type) { + case *ast.Term: + if ast.IsComprehension(x.Value) { + return true + } + case ast.Ref: + if !stopped { + if bi := ast.BuiltinMap[x.String()]; bi != nil { + if bi.Relation { + stopped = true + return stopped + } + } + for i := 1; i < len(x); i++ { + if _, ok := x[i].Value.(ast.Var); ok { + stopped = true + return stopped + } + } + } + return stopped + } + return stopped + }) + + vis.Walk(x) + + return stopped +} + +func parseStringsToRefs(s []string) ([]ast.Ref, error) { + + refs := make([]ast.Ref, len(s)) + for i := range refs { + var err error + refs[i], err = ast.ParseRef(s[i]) + if err != nil { + return nil, err + } + } + + return refs, nil +} + +// helper function to finish a built-in function call. If an error occured, +// wrap the error and return it. Otherwise, invoke the iterator if the result +// was defined. +func finishFunction(name string, bctx topdown.BuiltinContext, result *ast.Term, err error, iter func(*ast.Term) error) error { + if err != nil { + return &topdown.Error{ + Code: topdown.BuiltinErr, + Message: fmt.Sprintf("%v: %v", name, err.Error()), + Location: bctx.Location, + } + } + if result == nil { + return nil + } + return iter(result) +} + +// helper function to return an option that sets a custom built-in function. +func newFunction(decl *Function, f topdown.BuiltinFunc) func(*Rego) { + return func(r *Rego) { + r.builtinDecls[decl.Name] = &ast.Builtin{ + Name: decl.Name, + Decl: decl.Decl, + } + r.builtinFuncs[decl.Name] = &topdown.Builtin{ + Decl: r.builtinDecls[decl.Name], + Func: f, + } + } +} diff --git a/vendor/github.com/open-policy-agent/opa/storage/doc.go b/vendor/github.com/open-policy-agent/opa/storage/doc.go new file mode 100644 index 000000000..6fa2f86d9 --- /dev/null +++ b/vendor/github.com/open-policy-agent/opa/storage/doc.go @@ -0,0 +1,6 @@ +// Copyright 2016 The OPA Authors. All rights reserved. +// Use of this source code is governed by an Apache2 +// license that can be found in the LICENSE file. + +// Package storage exposes the policy engine's storage layer. +package storage diff --git a/vendor/github.com/open-policy-agent/opa/storage/errors.go b/vendor/github.com/open-policy-agent/opa/storage/errors.go new file mode 100644 index 000000000..6c8779586 --- /dev/null +++ b/vendor/github.com/open-policy-agent/opa/storage/errors.go @@ -0,0 +1,135 @@ +// Copyright 2016 The OPA Authors. All rights reserved. +// Use of this source code is governed by an Apache2 +// license that can be found in the LICENSE file. + +package storage + +import ( + "fmt" +) + +const ( + // InternalErr indicates an unknown, internal error has occurred. + InternalErr = "storage_internal_error" + + // NotFoundErr indicates the path used in the storage operation does not + // locate a document. + NotFoundErr = "storage_not_found_error" + + // WriteConflictErr indicates a write on the path enocuntered a conflicting + // value inside the transaction. + WriteConflictErr = "storage_write_conflict_error" + + // InvalidPatchErr indicates an invalid patch/write was issued. The patch + // was rejected. + InvalidPatchErr = "storage_invalid_patch_error" + + // InvalidTransactionErr indicates an invalid operation was performed + // inside of the transaction. + InvalidTransactionErr = "storage_invalid_txn_error" + + // TriggersNotSupportedErr indicates the caller attempted to register a + // trigger against a store that does not support them. + TriggersNotSupportedErr = "storage_triggers_not_supported_error" + + // WritesNotSupportedErr indicate the caller attempted to perform a write + // against a store that does not support them. + WritesNotSupportedErr = "storage_writes_not_supported_error" + + // PolicyNotSupportedErr indicate the caller attempted to perform a policy + // management operation against a store that does not support them. + PolicyNotSupportedErr = "storage_policy_not_supported_error" + + // IndexingNotSupportedErr indicate the caller attempted to perform an + // indexing operation against a store that does not support them. + IndexingNotSupportedErr = "storage_indexing_not_supported_error" +) + +// Error is the error type returned by the storage layer. +type Error struct { + Code string `json:"code"` + Message string `json:"message"` +} + +func (err *Error) Error() string { + if err.Message != "" { + return fmt.Sprintf("%v: %v", err.Code, err.Message) + } + return string(err.Code) +} + +// IsNotFound returns true if this error is a NotFoundErr. +func IsNotFound(err error) bool { + switch err := err.(type) { + case *Error: + return err.Code == NotFoundErr + } + return false +} + +// IsWriteConflictError returns true if this error a WriteConflictErr. +func IsWriteConflictError(err error) bool { + switch err := err.(type) { + case *Error: + return err.Code == WriteConflictErr + } + return false +} + +// IsInvalidPatch returns true if this error is a InvalidPatchErr. +func IsInvalidPatch(err error) bool { + switch err := err.(type) { + case *Error: + return err.Code == InvalidPatchErr + } + return false +} + +// IsInvalidTransaction returns true if this error is a InvalidTransactionErr. +func IsInvalidTransaction(err error) bool { + switch err := err.(type) { + case *Error: + return err.Code == InvalidTransactionErr + } + return false +} + +// IsIndexingNotSupported returns true if this error is a IndexingNotSupportedErr. +func IsIndexingNotSupported(err error) bool { + switch err := err.(type) { + case *Error: + return err.Code == IndexingNotSupportedErr + } + return false +} + +func writeConflictError(path Path) *Error { + return &Error{ + Code: WriteConflictErr, + Message: fmt.Sprint(path), + } +} + +func triggersNotSupportedError() *Error { + return &Error{ + Code: TriggersNotSupportedErr, + } +} + +func writesNotSupportedError() *Error { + return &Error{ + Code: WritesNotSupportedErr, + } +} + +func policyNotSupportedError() *Error { + return &Error{ + Code: PolicyNotSupportedErr, + } +} + +func indexingNotSupportedError() *Error { + return &Error{ + Code: IndexingNotSupportedErr, + } +} diff --git a/vendor/github.com/open-policy-agent/opa/storage/inmem/index.go b/vendor/github.com/open-policy-agent/opa/storage/inmem/index.go new file mode 100644 index 000000000..1a6f4aaf6 --- /dev/null +++ b/vendor/github.com/open-policy-agent/opa/storage/inmem/index.go @@ -0,0 +1,350 @@ +// Copyright 2016 The OPA Authors. All rights reserved. +// Use of this source code is governed by an Apache2 +// license that can be found in the LICENSE file. + +package inmem + +import ( + "context" + "encoding/json" + "fmt" + "hash/fnv" + "strings" + "sync" + + "github.com/open-policy-agent/opa/ast" + "github.com/open-policy-agent/opa/storage" + "github.com/open-policy-agent/opa/util" +) + +// indices contains a mapping of non-ground references to values to sets of bindings. +// +// +------+------------------------------------+ +// | ref1 | val1 | bindings-1, bindings-2, ... | +// | +------+-----------------------------+ +// | | val2 | bindings-m, bindings-m, ... | +// | +------+-----------------------------+ +// | | .... | ... | +// +------+------+-----------------------------+ +// | ref2 | .... | ... | +// +------+------+-----------------------------+ +// | ... | +// +-------------------------------------------+ +// +// The "value" is the data value stored at the location referred to by the ground +// reference obtained by plugging bindings into the non-ground reference that is the +// index key. +// +type indices struct { + mu sync.Mutex + table map[int]*indicesNode +} + +type indicesNode struct { + key ast.Ref + val *bindingIndex + next *indicesNode +} + +func newIndices() *indices { + return &indices{ + table: map[int]*indicesNode{}, + } +} + +func (ind *indices) Build(ctx context.Context, store storage.Store, txn storage.Transaction, ref ast.Ref) (*bindingIndex, error) { + + ind.mu.Lock() + defer ind.mu.Unlock() + + if exist := ind.get(ref); exist != nil { + return exist, nil + } + + index := newBindingIndex() + + if err := iterStorage(ctx, store, txn, ref, ast.EmptyRef(), ast.NewValueMap(), index.Add); err != nil { + return nil, err + } + + hashCode := ref.Hash() + head := ind.table[hashCode] + entry := &indicesNode{ + key: ref, + val: index, + next: head, + } + + ind.table[hashCode] = entry + + return index, nil +} + +func (ind *indices) get(ref ast.Ref) *bindingIndex { + node := ind.getNode(ref) + if node != nil { + return node.val + } + return nil +} + +func (ind *indices) iter(iter func(ast.Ref, *bindingIndex) error) error { + for _, head := range ind.table { + for entry := head; entry != nil; entry = entry.next { + if err := iter(entry.key, entry.val); err != nil { + return err + } + } + } + return nil +} + +func (ind *indices) getNode(ref ast.Ref) *indicesNode { + hashCode := ref.Hash() + for entry := ind.table[hashCode]; entry != nil; entry = entry.next { + if entry.key.Equal(ref) { + return entry + } + } + return nil +} + +func (ind *indices) String() string { + buf := []string{} + for _, head := range ind.table { + for entry := head; entry != nil; entry = entry.next { + str := fmt.Sprintf("%v: %v", entry.key, entry.val) + buf = append(buf, str) + } + } + return "{" + strings.Join(buf, ", ") + "}" +} + +// bindingIndex contains a mapping of values to bindings. +type bindingIndex struct { + table map[int]*indexNode +} + +type indexNode struct { + key interface{} + val *bindingSet + next *indexNode +} + +func newBindingIndex() *bindingIndex { + return &bindingIndex{ + table: map[int]*indexNode{}, + } +} + +func (ind *bindingIndex) Add(val interface{}, bindings *ast.ValueMap) { + + node := ind.getNode(val) + if node != nil { + node.val.Add(bindings) + return + } + + hashCode := hash(val) + bindingsSet := newBindingSet() + bindingsSet.Add(bindings) + + entry := &indexNode{ + key: val, + val: bindingsSet, + next: ind.table[hashCode], + } + + ind.table[hashCode] = entry +} + +func (ind *bindingIndex) Lookup(_ context.Context, _ storage.Transaction, val interface{}, iter storage.IndexIterator) error { + node := ind.getNode(val) + if node == nil { + return nil + } + return node.val.Iter(iter) +} + +func (ind *bindingIndex) getNode(val interface{}) *indexNode { + hashCode := hash(val) + head := ind.table[hashCode] + for entry := head; entry != nil; entry = entry.next { + if util.Compare(entry.key, val) == 0 { + return entry + } + } + return nil +} + +func (ind *bindingIndex) String() string { + + buf := []string{} + + for _, head := range ind.table { + for entry := head; entry != nil; entry = entry.next { + str := fmt.Sprintf("%v: %v", entry.key, entry.val) + buf = append(buf, str) + } + } + + return "{" + strings.Join(buf, ", ") + "}" +} + +type bindingSetNode struct { + val *ast.ValueMap + next *bindingSetNode +} + +type bindingSet struct { + table map[int]*bindingSetNode +} + +func newBindingSet() *bindingSet { + return &bindingSet{ + table: map[int]*bindingSetNode{}, + } +} + +func (set *bindingSet) Add(val *ast.ValueMap) { + node := set.getNode(val) + if node != nil { + return + } + hashCode := val.Hash() + head := set.table[hashCode] + set.table[hashCode] = &bindingSetNode{val, head} +} + +func (set *bindingSet) Iter(iter func(*ast.ValueMap) error) error { + for _, head := range set.table { + for entry := head; entry != nil; entry = entry.next { + if err := iter(entry.val); err != nil { + return err + } + } + } + return nil +} + +func (set *bindingSet) String() string { + buf := []string{} + set.Iter(func(bindings *ast.ValueMap) error { + buf = append(buf, bindings.String()) + return nil + }) + return "{" + strings.Join(buf, ", ") + "}" +} + +func (set *bindingSet) getNode(val *ast.ValueMap) *bindingSetNode { + hashCode := val.Hash() + for entry := set.table[hashCode]; entry != nil; entry = entry.next { + if entry.val.Equal(val) { + return entry + } + } + return nil +} + +func hash(v interface{}) int { + switch v := v.(type) { + case []interface{}: + var h int + for _, e := range v { + h += hash(e) + } + return h + case map[string]interface{}: + var h int + for k, v := range v { + h += hash(k) + hash(v) + } + return h + case string: + h := fnv.New64a() + h.Write([]byte(v)) + return int(h.Sum64()) + case bool: + if v { + return 1 + } + return 0 + case nil: + return 0 + case json.Number: + h := fnv.New64a() + h.Write([]byte(v)) + return int(h.Sum64()) + } + panic(fmt.Sprintf("illegal argument: %v (%T)", v, v)) +} + +func iterStorage(ctx context.Context, store storage.Store, txn storage.Transaction, nonGround, ground ast.Ref, bindings *ast.ValueMap, iter func(interface{}, *ast.ValueMap)) error { + + if len(nonGround) == 0 { + path, err := storage.NewPathForRef(ground) + if err != nil { + return err + } + node, err := store.Read(ctx, txn, path) + if err != nil { + if storage.IsNotFound(err) { + return nil + } + return err + } + iter(node, bindings) + return nil + } + + head := nonGround[0] + tail := nonGround[1:] + + headVar, isVar := head.Value.(ast.Var) + + if !isVar || len(ground) == 0 { + ground = append(ground, head) + return iterStorage(ctx, store, txn, tail, ground, bindings, iter) + } + + path, err := storage.NewPathForRef(ground) + if err != nil { + return err + } + + node, err := store.Read(ctx, txn, path) + if err != nil { + if storage.IsNotFound(err) { + return nil + } + return err + } + + switch node := node.(type) { + case map[string]interface{}: + for key := range node { + ground = append(ground, ast.StringTerm(key)) + cpy := bindings.Copy() + cpy.Put(headVar, ast.String(key)) + err := iterStorage(ctx, store, txn, tail, ground, cpy, iter) + if err != nil { + return err + } + ground = ground[:len(ground)-1] + } + case []interface{}: + for i := range node { + idx := ast.IntNumberTerm(i) + ground = append(ground, idx) + cpy := bindings.Copy() + cpy.Put(headVar, idx.Value) + err := iterStorage(ctx, store, txn, tail, ground, cpy, iter) + if err != nil { + return err + } + ground = ground[:len(ground)-1] + } + } + + return nil +} diff --git a/vendor/github.com/open-policy-agent/opa/storage/inmem/inmem.go b/vendor/github.com/open-policy-agent/opa/storage/inmem/inmem.go new file mode 100644 index 000000000..0b2ebafe0 --- /dev/null +++ b/vendor/github.com/open-policy-agent/opa/storage/inmem/inmem.go @@ -0,0 +1,287 @@ +// Copyright 2016 The OPA Authors. All rights reserved. +// Use of this source code is governed by an Apache2 +// license that can be found in the LICENSE file. + +// Package inmem implements an in-memory version of the policy engine's storage +// layer. +// +// The in-memory store is used as the default storage layer implementation. The +// in-memory store supports multi-reader/single-writer concurrency with +// rollback. +// +// Callers should assume the in-memory store does not make copies of written +// data. Once data is written to the in-memory store, it should not be modified +// (outside of calling Store.Write). Furthermore, data read from the in-memory +// store should be treated as read-only. +package inmem + +import ( + "context" + "fmt" + "io" + "sync" + "sync/atomic" + + "github.com/open-policy-agent/opa/ast" + "github.com/open-policy-agent/opa/storage" + "github.com/open-policy-agent/opa/util" +) + +// New returns an empty in-memory store. +func New() storage.Store { + return &store{ + data: map[string]interface{}{}, + triggers: map[*handle]storage.TriggerConfig{}, + policies: map[string][]byte{}, + indices: newIndices(), + } +} + +// NewFromObject returns a new in-memory store from the supplied data object. +func NewFromObject(data map[string]interface{}) storage.Store { + db := New() + ctx := context.Background() + txn, err := db.NewTransaction(ctx, storage.WriteParams) + if err != nil { + panic(err) + } + if err := db.Write(ctx, txn, storage.AddOp, storage.Path{}, data); err != nil { + panic(err) + } + if err := db.Commit(ctx, txn); err != nil { + panic(err) + } + return db +} + +// NewFromReader returns a new in-memory store from a reader that produces a +// JSON serialized object. This function is for test purposes. +func NewFromReader(r io.Reader) storage.Store { + d := util.NewJSONDecoder(r) + var data map[string]interface{} + if err := d.Decode(&data); err != nil { + panic(err) + } + return NewFromObject(data) +} + +type store struct { + rmu sync.RWMutex // reader-writer lock + wmu sync.Mutex // writer lock + xid uint64 // last generated transaction id + data map[string]interface{} // raw data + policies map[string][]byte // raw policies + triggers map[*handle]storage.TriggerConfig // registered triggers + indices *indices // data ref indices +} + +type handle struct { + db *store +} + +func (db *store) NewTransaction(ctx context.Context, params ...storage.TransactionParams) (storage.Transaction, error) { + var write bool + var context *storage.Context + if len(params) > 0 { + write = params[0].Write + context = params[0].Context + } + xid := atomic.AddUint64(&db.xid, uint64(1)) + if write { + db.wmu.Lock() + } else { + db.rmu.RLock() + } + return newTransaction(xid, write, context, db), nil +} + +func (db *store) Commit(ctx context.Context, txn storage.Transaction) error { + underlying, err := db.underlying(txn) + if err != nil { + return err + } + if underlying.write { + db.rmu.Lock() + event := underlying.Commit() + db.indices = newIndices() + db.runOnCommitTriggers(ctx, txn, event) + // Mark the transaction stale after executing triggers so they can + // perform store operations if needed. + underlying.stale = true + db.rmu.Unlock() + db.wmu.Unlock() + } else { + db.rmu.RUnlock() + } + return nil +} + +func (db *store) Abort(ctx context.Context, txn storage.Transaction) { + underlying, err := db.underlying(txn) + if err != nil { + panic(err) + } + underlying.stale = true + if underlying.write { + db.wmu.Unlock() + } else { + db.rmu.RUnlock() + } +} + +func (db *store) ListPolicies(_ context.Context, txn storage.Transaction) ([]string, error) { + underlying, err := db.underlying(txn) + if err != nil { + return nil, err + } + return underlying.ListPolicies(), nil +} + +func (db *store) GetPolicy(_ context.Context, txn storage.Transaction, id string) ([]byte, error) { + underlying, err := db.underlying(txn) + if err != nil { + return nil, err + } + return underlying.GetPolicy(id) +} + +func (db *store) UpsertPolicy(_ context.Context, txn storage.Transaction, id string, bs []byte) error { + underlying, err := db.underlying(txn) + if err != nil { + return err + } + return underlying.UpsertPolicy(id, bs) +} + +func (db *store) DeletePolicy(_ context.Context, txn storage.Transaction, id string) error { + underlying, err := db.underlying(txn) + if err != nil { + return err + } + if _, err := underlying.GetPolicy(id); err != nil { + return err + } + return underlying.DeletePolicy(id) +} + +func (db *store) Register(ctx context.Context, txn storage.Transaction, config storage.TriggerConfig) (storage.TriggerHandle, error) { + underlying, err := db.underlying(txn) + if err != nil { + return nil, err + } + if !underlying.write { + return nil, &storage.Error{ + Code: storage.InvalidTransactionErr, + Message: "triggers must be registered with a write transaction", + } + } + h := &handle{db} + db.triggers[h] = config + return h, nil +} + +func (db *store) Read(ctx context.Context, txn storage.Transaction, path storage.Path) (interface{}, error) { + underlying, err := db.underlying(txn) + if err != nil { + return nil, err + } + return underlying.Read(path) +} + +func (db *store) Write(ctx context.Context, txn storage.Transaction, op storage.PatchOp, path storage.Path, value interface{}) error { + underlying, err := db.underlying(txn) + if err != nil { + return err + } + val := util.Reference(value) + if err := util.RoundTrip(val); err != nil { + return err + } + return underlying.Write(op, path, *val) +} + +func (db *store) Build(ctx context.Context, txn storage.Transaction, ref ast.Ref) (storage.Index, error) { + underlying, err := db.underlying(txn) + if err != nil { + return nil, err + } + if underlying.write { + return nil, &storage.Error{ + Code: storage.IndexingNotSupportedErr, + Message: "in-memory store does not support indexing on write transactions", + } + } + return db.indices.Build(ctx, db, txn, ref) +} + +func (h *handle) Unregister(ctx context.Context, txn storage.Transaction) { + underlying, err := h.db.underlying(txn) + if err != nil { + panic(err) + } + if !underlying.write { + panic(&storage.Error{ + Code: storage.InvalidTransactionErr, + Message: "triggers must be unregistered with a write transaction", + }) + } + delete(h.db.triggers, h) +} + +func (db *store) runOnCommitTriggers(ctx context.Context, txn storage.Transaction, event storage.TriggerEvent) { + for _, t := range db.triggers { + t.OnCommit(ctx, txn, event) + } +} + +func (db *store) underlying(txn storage.Transaction) (*transaction, error) { + underlying, ok := txn.(*transaction) + if !ok { + return nil, &storage.Error{ + Code: storage.InvalidTransactionErr, + Message: fmt.Sprintf("unexpected transaction type %T", txn), + } + } + if underlying.db != db { + return nil, &storage.Error{ + Code: storage.InvalidTransactionErr, + Message: "unknown transaction", + } + } + if underlying.stale { + return nil, &storage.Error{ + Code: storage.InvalidTransactionErr, + Message: "stale transaction", + } + } + return underlying, nil +} + +var doesNotExistMsg = "document does not exist" +var rootMustBeObjectMsg = "root must be object" +var rootCannotBeRemovedMsg = "root cannot be removed" +var outOfRangeMsg = "array index out of range" +var arrayIndexTypeMsg = "array index must be integer" + +func invalidPatchError(f string, a ...interface{}) *storage.Error { + return &storage.Error{ + Code: storage.InvalidPatchErr, + Message: fmt.Sprintf(f, a...), + } +} + +func notFoundError(path storage.Path) *storage.Error { + return notFoundErrorHint(path, doesNotExistMsg) +} + +func notFoundErrorHint(path storage.Path, hint string) *storage.Error { + return notFoundErrorf("%v: %v", path.String(), hint) +} + +func notFoundErrorf(f string, a ...interface{}) *storage.Error { + msg := fmt.Sprintf(f, a...) + return &storage.Error{ + Code: storage.NotFoundErr, + Message: msg, + } +} diff --git a/vendor/github.com/open-policy-agent/opa/storage/inmem/txn.go b/vendor/github.com/open-policy-agent/opa/storage/inmem/txn.go new file mode 100644 index 000000000..5ef02bccb --- /dev/null +++ b/vendor/github.com/open-policy-agent/opa/storage/inmem/txn.go @@ -0,0 +1,444 @@ +// Copyright 2017 The OPA Authors. All rights reserved. +// Use of this source code is governed by an Apache2 +// license that can be found in the LICENSE file. + +package inmem + +import ( + "container/list" + "encoding/json" + "strconv" + + "github.com/open-policy-agent/opa/storage" +) + +// transaction implements the low-level read/write operations on the in-memory +// store and contains the state required for pending transactions. +// +// For write transactions, the struct contains a logical set of updates +// performed by write operations in the transaction. Each write operation +// compacts the set such that two updates never overlap: +// +// - If new update path is a prefix of existing update path, existing update is +// removed, new update is added. +// +// - If existing update path is a prefix of new update path, existing update is +// modified. +// +// - Otherwise, new update is added. +// +// Read transactions do not require any special handling and simply passthrough +// to the underlying store. Read transactions do not support upgrade. +type transaction struct { + xid uint64 + write bool + stale bool + db *store + updates *list.List + policies map[string]policyUpdate + context *storage.Context +} + +type policyUpdate struct { + value []byte + remove bool +} + +func newTransaction(xid uint64, write bool, context *storage.Context, db *store) *transaction { + return &transaction{ + xid: xid, + write: write, + db: db, + policies: map[string]policyUpdate{}, + updates: list.New(), + context: context, + } +} + +func (txn *transaction) ID() uint64 { + return txn.xid +} + +func (txn *transaction) Write(op storage.PatchOp, path storage.Path, value interface{}) error { + + if !txn.write { + return &storage.Error{ + Code: storage.InvalidTransactionErr, + Message: "data write during read transaction", + } + } + + if len(path) == 0 { + return txn.updateRoot(op, value) + } + + for curr := txn.updates.Front(); curr != nil; { + update := curr.Value.(*update) + + // Check if new update masks existing update exactly. In this case, the + // existing update can be removed and no other updates have to be + // visited (because no two updates overlap.) + if update.path.Equal(path) { + if update.remove { + if op != storage.AddOp { + return notFoundError(path) + } + } + txn.updates.Remove(curr) + break + } + + // Check if new update masks existing update. In this case, the + // existing update has to be removed but other updates may overlap, so + // we must continue. + if update.path.HasPrefix(path) { + remove := curr + curr = curr.Next() + txn.updates.Remove(remove) + continue + } + + // Check if new update modifies existing update. In this case, the + // existing update is mutated. + if path.HasPrefix(update.path) { + if update.remove { + return notFoundError(path) + } + suffix := path[len(update.path):] + newUpdate, err := newUpdate(update.value, op, suffix, 0, value) + if err != nil { + return err + } + update.value = newUpdate.Apply(update.value) + return nil + } + + curr = curr.Next() + } + + update, err := newUpdate(txn.db.data, op, path, 0, value) + if err != nil { + return err + } + + txn.updates.PushFront(update) + return nil +} + +func (txn *transaction) updateRoot(op storage.PatchOp, value interface{}) error { + if op == storage.RemoveOp { + return invalidPatchError(rootCannotBeRemovedMsg) + } + if _, ok := value.(map[string]interface{}); !ok { + return invalidPatchError(rootMustBeObjectMsg) + } + txn.updates.Init() + txn.updates.PushFront(&update{ + path: storage.Path{}, + remove: false, + value: value, + }) + return nil +} + +func (txn *transaction) Commit() (result storage.TriggerEvent) { + result.Context = txn.context + for curr := txn.updates.Front(); curr != nil; curr = curr.Next() { + action := curr.Value.(*update) + updated := action.Apply(txn.db.data) + txn.db.data = updated.(map[string]interface{}) + + result.Data = append(result.Data, storage.DataEvent{ + Path: action.path, + Data: action.value, + Removed: action.remove, + }) + } + for id, update := range txn.policies { + if update.remove { + delete(txn.db.policies, id) + } else { + txn.db.policies[id] = update.value + } + + result.Policy = append(result.Policy, storage.PolicyEvent{ + ID: id, + Data: update.value, + Removed: update.remove, + }) + } + return result +} + +func (txn *transaction) Read(path storage.Path) (interface{}, error) { + + if !txn.write { + return ptr(txn.db.data, path) + } + + merge := []*update{} + + for curr := txn.updates.Front(); curr != nil; curr = curr.Next() { + + update := curr.Value.(*update) + + if path.HasPrefix(update.path) { + if update.remove { + return nil, notFoundError(path) + } + return ptr(update.value, path[len(update.path):]) + } + + if update.path.HasPrefix(path) { + merge = append(merge, update) + } + } + + data, err := ptr(txn.db.data, path) + + if err != nil { + return nil, err + } + + if len(merge) == 0 { + return data, nil + } + + cpy := deepCopy(data) + + for _, update := range merge { + cpy = update.Relative(path).Apply(cpy) + } + + return cpy, nil +} + +func (txn *transaction) ListPolicies() []string { + var ids []string + for id := range txn.db.policies { + if _, ok := txn.policies[id]; !ok { + ids = append(ids, id) + } + } + for id, update := range txn.policies { + if !update.remove { + ids = append(ids, id) + } + } + return ids +} + +func (txn *transaction) GetPolicy(id string) ([]byte, error) { + if update, ok := txn.policies[id]; ok { + if !update.remove { + return update.value, nil + } + return nil, notFoundErrorf("policy id %q", id) + } + if exist, ok := txn.db.policies[id]; ok { + return exist, nil + } + return nil, notFoundErrorf("policy id %q", id) +} + +func (txn *transaction) UpsertPolicy(id string, bs []byte) error { + if !txn.write { + return &storage.Error{ + Code: storage.InvalidTransactionErr, + Message: "policy write during read transaction", + } + } + txn.policies[id] = policyUpdate{bs, false} + return nil +} + +func (txn *transaction) DeletePolicy(id string) error { + if !txn.write { + return &storage.Error{ + Code: storage.InvalidTransactionErr, + Message: "policy write during read transaction", + } + } + txn.policies[id] = policyUpdate{nil, true} + return nil +} + +// update contains state associated with an update to be applied to the +// in-memory data store. +type update struct { + path storage.Path // data path modified by update + remove bool // indicates whether update removes the value at path + value interface{} // value to add/replace at path (ignored if remove is true) +} + +func newUpdate(data interface{}, op storage.PatchOp, path storage.Path, idx int, value interface{}) (*update, error) { + + switch data := data.(type) { + case map[string]interface{}: + return newUpdateObject(data, op, path, idx, value) + + case []interface{}: + return newUpdateArray(data, op, path, idx, value) + + case nil, bool, json.Number, string: + return nil, notFoundError(path) + } + + return nil, &storage.Error{ + Code: storage.InternalErr, + Message: "invalid data value encountered", + } +} + +func newUpdateArray(data []interface{}, op storage.PatchOp, path storage.Path, idx int, value interface{}) (*update, error) { + + if idx == len(path)-1 { + if path[idx] == "-" { + if op != storage.AddOp { + return nil, invalidPatchError("%v: invalid patch path", path) + } + cpy := make([]interface{}, len(data)+1) + copy(cpy, data) + cpy[len(data)] = value + return &update{path[:len(path)-1], false, cpy}, nil + } + + pos, err := validateArrayIndex(data, path[idx], path) + if err != nil { + return nil, err + } + + if op == storage.AddOp { + cpy := make([]interface{}, len(data)+1) + copy(cpy[:pos], data[:pos]) + copy(cpy[pos+1:], data[pos:]) + cpy[pos] = value + return &update{path[:len(path)-1], false, cpy}, nil + + } else if op == storage.RemoveOp { + cpy := make([]interface{}, len(data)-1) + copy(cpy[:pos], data[:pos]) + copy(cpy[pos:], data[pos+1:]) + return &update{path[:len(path)-1], false, cpy}, nil + + } else { + cpy := make([]interface{}, len(data)) + copy(cpy, data) + cpy[pos] = value + return &update{path[:len(path)-1], false, cpy}, nil + } + } + + pos, err := validateArrayIndex(data, path[idx], path) + if err != nil { + return nil, err + } + + return newUpdate(data[pos], op, path, idx+1, value) +} + +func newUpdateObject(data map[string]interface{}, op storage.PatchOp, path storage.Path, idx int, value interface{}) (*update, error) { + + if idx == len(path)-1 { + switch op { + case storage.ReplaceOp, storage.RemoveOp: + if _, ok := data[path[idx]]; !ok { + return nil, notFoundError(path) + } + } + return &update{path, op == storage.RemoveOp, value}, nil + } + + if data, ok := data[path[idx]]; ok { + return newUpdate(data, op, path, idx+1, value) + } + + return nil, notFoundError(path) +} +func (u *update) Apply(data interface{}) interface{} { + if len(u.path) == 0 { + return u.value + } + parent, err := ptr(data, u.path[:len(u.path)-1]) + if err != nil { + panic(err) + } + key := u.path[len(u.path)-1] + if u.remove { + obj := parent.(map[string]interface{}) + delete(obj, key) + return data + } + switch parent := parent.(type) { + case map[string]interface{}: + parent[key] = u.value + case []interface{}: + idx, err := strconv.Atoi(key) + if err != nil { + panic(err) + } + parent[idx] = u.value + } + return data +} + +func (u *update) Relative(path storage.Path) *update { + cpy := *u + cpy.path = cpy.path[len(path):] + return &cpy +} + +func deepCopy(val interface{}) interface{} { + switch val := val.(type) { + case []interface{}: + cpy := make([]interface{}, len(val)) + for i := range cpy { + cpy[i] = deepCopy(val[i]) + } + return cpy + case map[string]interface{}: + cpy := make(map[string]interface{}, len(val)) + for k := range val { + cpy[k] = deepCopy(val[k]) + } + return cpy + default: + return val + } +} + +func ptr(data interface{}, path storage.Path) (interface{}, error) { + + node := data + for i := range path { + key := path[i] + switch curr := node.(type) { + case map[string]interface{}: + var ok bool + if node, ok = curr[key]; !ok { + return nil, notFoundError(path) + } + case []interface{}: + pos, err := validateArrayIndex(curr, key, path) + if err != nil { + return nil, err + } + node = curr[pos] + default: + return nil, notFoundError(path) + } + } + + return node, nil +} + +func validateArrayIndex(arr []interface{}, s string, path storage.Path) (int, error) { + idx, err := strconv.Atoi(s) + if err != nil { + return 0, notFoundErrorHint(path, arrayIndexTypeMsg) + } + if idx < 0 || idx >= len(arr) { + return 0, notFoundErrorHint(path, outOfRangeMsg) + } + return idx, nil +} diff --git a/vendor/github.com/open-policy-agent/opa/storage/interface.go b/vendor/github.com/open-policy-agent/opa/storage/interface.go new file mode 100644 index 000000000..8b1e1626e --- /dev/null +++ b/vendor/github.com/open-policy-agent/opa/storage/interface.go @@ -0,0 +1,219 @@ +// Copyright 2016 The OPA Authors. All rights reserved. +// Use of this source code is governed by an Apache2 +// license that can be found in the LICENSE file. + +package storage + +import ( + "context" + + "github.com/open-policy-agent/opa/ast" +) + +// Transaction defines the interface that identifies a consistent snapshot over +// the policy engine's storage layer. +type Transaction interface { + ID() uint64 +} + +// Store defines the interface for the storage layer's backend. +type Store interface { + Trigger + Policy + Indexing + + // NewTransaction is called create a new transaction in the store. + NewTransaction(ctx context.Context, params ...TransactionParams) (Transaction, error) + + // Read is called to fetch a document referred to by path. + Read(ctx context.Context, txn Transaction, path Path) (interface{}, error) + + // Write is called to modify a document referred to by path. + Write(ctx context.Context, txn Transaction, op PatchOp, path Path, value interface{}) error + + // Commit is called to finish the transaction. If Commit returns an error, the + // transaction must be automatically aborted by the Store implementation. + Commit(ctx context.Context, txn Transaction) error + + // Abort is called to cancel the transaction. + Abort(ctx context.Context, txn Transaction) +} + +// TransactionParams describes a new transaction. +type TransactionParams struct { + + // Write indicates if this transaction will perform any write operations. + Write bool + + // Context contains key/value pairs passed to triggers. + Context *Context +} + +// Context is a simple container for key/value pairs. +type Context struct { + values map[interface{}]interface{} +} + +// NewContext returns a new context object. +func NewContext() *Context { + return &Context{ + values: map[interface{}]interface{}{}, + } +} + +// Get returns the key value in the context. +func (ctx *Context) Get(key interface{}) interface{} { + if ctx == nil { + return nil + } + return ctx.values[key] +} + +// Put adds a key/value pair to the context. +func (ctx *Context) Put(key, value interface{}) { + ctx.values[key] = value +} + +// WriteParams specifies the TransactionParams for a write transaction. +var WriteParams = TransactionParams{ + Write: true, +} + +// PatchOp is the enumeration of supposed modifications. +type PatchOp int + +// Patch supports add, remove, and replace operations. +const ( + AddOp PatchOp = iota + RemoveOp = iota + ReplaceOp = iota +) + +// WritesNotSupported provides a default implementation of the write +// interface which may be used if the backend does not support writes. +type WritesNotSupported struct{} + +func (WritesNotSupported) Write(ctx context.Context, txn Transaction, op PatchOp, path Path, value interface{}) error { + return writesNotSupportedError() +} + +// Policy defines the interface for policy module storage. +type Policy interface { + ListPolicies(context.Context, Transaction) ([]string, error) + GetPolicy(context.Context, Transaction, string) ([]byte, error) + UpsertPolicy(context.Context, Transaction, string, []byte) error + DeletePolicy(context.Context, Transaction, string) error +} + +// PolicyNotSupported provides a default implementation of the policy interface +// which may be used if the backend does not support policy storage. +type PolicyNotSupported struct{} + +// ListPolicies always returns a PolicyNotSupportedErr. +func (PolicyNotSupported) ListPolicies(context.Context, Transaction) ([]string, error) { + return nil, policyNotSupportedError() +} + +// GetPolicy always returns a PolicyNotSupportedErr. +func (PolicyNotSupported) GetPolicy(context.Context, Transaction, string) ([]byte, error) { + return nil, policyNotSupportedError() +} + +// UpsertPolicy always returns a PolicyNotSupportedErr. +func (PolicyNotSupported) UpsertPolicy(context.Context, Transaction, string, []byte) error { + return policyNotSupportedError() +} + +// DeletePolicy always returns a PolicyNotSupportedErr. +func (PolicyNotSupported) DeletePolicy(context.Context, Transaction, string) error { + return policyNotSupportedError() +} + +// PolicyEvent describes a change to a policy. +type PolicyEvent struct { + ID string + Data []byte + Removed bool +} + +// DataEvent describes a change to a base data document. +type DataEvent struct { + Path Path + Data interface{} + Removed bool +} + +// TriggerEvent describes the changes that caused the trigger to be invoked. +type TriggerEvent struct { + Policy []PolicyEvent + Data []DataEvent + Context *Context +} + +// IsZero returns true if the TriggerEvent indicates no changes occurred. This +// function is primarily for test purposes. +func (e TriggerEvent) IsZero() bool { + return !e.PolicyChanged() && !e.DataChanged() +} + +// PolicyChanged returns true if the trigger was caused by a policy change. +func (e TriggerEvent) PolicyChanged() bool { + return len(e.Policy) > 0 +} + +// DataChanged returns true if the trigger was caused by a data change. +func (e TriggerEvent) DataChanged() bool { + return len(e.Data) > 0 +} + +// TriggerConfig contains the trigger registration configuration. +type TriggerConfig struct { + + // OnCommit is invoked when a transaction is successfully committed. The + // callback is invoked with a handle to the write transaction that + // successfully committed before other clients see the changes. + OnCommit func(ctx context.Context, txn Transaction, event TriggerEvent) +} + +// Trigger defines the interface that stores implement to register for change +// notifications when the store is changed. +type Trigger interface { + Register(ctx context.Context, txn Transaction, config TriggerConfig) (TriggerHandle, error) +} + +// TriggersNotSupported provides default implementations of the Trigger +// interface which may be used if the backend does not support triggers. +type TriggersNotSupported struct{} + +// Register always returns an error indicating triggers are not supported. +func (TriggersNotSupported) Register(context.Context, Transaction, TriggerConfig) (TriggerHandle, error) { + return nil, triggersNotSupportedError() +} + +// TriggerHandle defines the interface that can be used to unregister triggers that have +// been registered on a Store. +type TriggerHandle interface { + Unregister(ctx context.Context, txn Transaction) +} + +// IndexIterator defines the interface for iterating over index results. +type IndexIterator func(*ast.ValueMap) error + +// Indexing defines the interface for building an index. +type Indexing interface { + Build(ctx context.Context, txn Transaction, ref ast.Ref) (Index, error) +} + +// Index defines the interface for searching a pre-built index. +type Index interface { + Lookup(ctx context.Context, txn Transaction, value interface{}, iter IndexIterator) error +} + +// IndexingNotSupported provides default implementations of the Indexing +// interface which may be used if the backend does not support indexing. +type IndexingNotSupported struct{} + +// Build always returns an error indicating indexing is not supported. +func (IndexingNotSupported) Build(context.Context, Transaction, ast.Ref) (Index, error) { + return nil, indexingNotSupportedError() +} diff --git a/vendor/github.com/open-policy-agent/opa/storage/path.go b/vendor/github.com/open-policy-agent/opa/storage/path.go new file mode 100644 index 000000000..fcb0ebee1 --- /dev/null +++ b/vendor/github.com/open-policy-agent/opa/storage/path.go @@ -0,0 +1,154 @@ +// Copyright 2016 The OPA Authors. All rights reserved. +// Use of this source code is governed by an Apache2 +// license that can be found in the LICENSE file. + +package storage + +import ( + "fmt" + "net/url" + "strconv" + "strings" + + "github.com/open-policy-agent/opa/ast" +) + +// Path refers to a document in storage. +type Path []string + +// ParsePath returns a new path for the given str. +func ParsePath(str string) (path Path, ok bool) { + if len(str) == 0 { + return nil, false + } + if str[0] != '/' { + return nil, false + } + if len(str) == 1 { + return Path{}, true + } + parts := strings.Split(str[1:], "/") + return parts, true +} + +// ParsePathEscaped returns a new path for the given escaped str. +func ParsePathEscaped(str string) (path Path, ok bool) { + path, ok = ParsePath(str) + if !ok { + return + } + for i := range path { + segment, err := url.PathUnescape(path[i]) + if err == nil { + path[i] = segment + } + } + return +} + +// NewPathForRef returns a new path for the given ref. +func NewPathForRef(ref ast.Ref) (path Path, err error) { + + if len(ref) == 0 { + return nil, fmt.Errorf("empty reference (indicates error in caller)") + } + + if len(ref) == 1 { + return Path{}, nil + } + + path = make(Path, 0, len(ref)-1) + + for _, term := range ref[1:] { + switch v := term.Value.(type) { + case ast.String: + path = append(path, string(v)) + case ast.Number: + path = append(path, v.String()) + case ast.Boolean, ast.Null: + return nil, &Error{ + Code: NotFoundErr, + Message: fmt.Sprintf("%v: does not exist", ref), + } + case ast.Array, ast.Object, ast.Set: + return nil, fmt.Errorf("composites cannot be base document keys: %v", ref) + default: + return nil, fmt.Errorf("unresolved reference (indicates error in caller): %v", ref) + } + } + + return path, nil +} + +// Compare performs lexigraphical comparison on p and other and returns -1 if p +// is less than other, 0 if p is equal to other, or 1 if p is greater than +// other. +func (p Path) Compare(other Path) (cmp int) { + min := len(p) + if len(other) < min { + min = len(other) + } + for i := 0; i < min; i++ { + if cmp := strings.Compare(p[i], other[i]); cmp != 0 { + return cmp + } + } + if len(p) < len(other) { + return -1 + } + if len(p) == len(other) { + return 0 + } + return 1 +} + +// Equal returns true if p is the same as other. +func (p Path) Equal(other Path) bool { + return p.Compare(other) == 0 +} + +// HasPrefix returns true if p starts with other. +func (p Path) HasPrefix(other Path) bool { + if len(other) > len(p) { + return false + } + for i := range other { + if p[i] != other[i] { + return false + } + } + return true +} + +// Ref returns a ref that represents p rooted at head. +func (p Path) Ref(head *ast.Term) (ref ast.Ref) { + ref = make(ast.Ref, len(p)+1) + ref[0] = head + for i := range p { + idx, err := strconv.ParseInt(p[i], 10, 64) + if err == nil { + ref[i+1] = ast.IntNumberTerm(int(idx)) + } else { + ref[i+1] = ast.StringTerm(p[i]) + } + } + return ref +} + +func (p Path) String() string { + buf := make([]string, len(p)) + for i := range buf { + buf[i] = url.PathEscape(p[i]) + } + return "/" + strings.Join(buf, "/") +} + +// MustParsePath returns a new Path for s. If s cannot be parsed, this function +// will panic. This is mostly for test purposes. +func MustParsePath(s string) Path { + path, ok := ParsePath(s) + if !ok { + panic(s) + } + return path +} diff --git a/vendor/github.com/open-policy-agent/opa/storage/storage.go b/vendor/github.com/open-policy-agent/opa/storage/storage.go new file mode 100644 index 000000000..323a0dba7 --- /dev/null +++ b/vendor/github.com/open-policy-agent/opa/storage/storage.go @@ -0,0 +1,126 @@ +// Copyright 2016 The OPA Authors. All rights reserved. +// Use of this source code is governed by an Apache2 +// license that can be found in the LICENSE file. + +package storage + +import ( + "context" +) + +// NewTransactionOrDie is a helper function to create a new transaction. If the +// storage layer cannot create a new transaction, this function will panic. This +// function should only be used for tests. +func NewTransactionOrDie(ctx context.Context, store Store, params ...TransactionParams) Transaction { + txn, err := store.NewTransaction(ctx, params...) + if err != nil { + panic(err) + } + return txn +} + +// ReadOne is a convenience function to read a single value from the provided Store. It +// will create a new Transaction to perform the read with, and clean up after itself +// should an error occur. +func ReadOne(ctx context.Context, store Store, path Path) (interface{}, error) { + txn, err := store.NewTransaction(ctx) + if err != nil { + return nil, err + } + defer store.Abort(ctx, txn) + + return store.Read(ctx, txn, path) +} + +// WriteOne is a convenience function to write a single value to the provided Store. It +// will create a new Transaction to perform the write with, and clean up after itself +// should an error occur. +func WriteOne(ctx context.Context, store Store, op PatchOp, path Path, value interface{}) error { + txn, err := store.NewTransaction(ctx, WriteParams) + if err != nil { + return err + } + + if err := store.Write(ctx, txn, op, path, value); err != nil { + store.Abort(ctx, txn) + return err + } + + return store.Commit(ctx, txn) +} + +// MakeDir inserts an empty object at path. If the parent path does not exist, +// MakeDir will create it recursively. +func MakeDir(ctx context.Context, store Store, txn Transaction, path Path) (err error) { + + if len(path) == 0 { + return nil + } + + node, err := store.Read(ctx, txn, path) + + if err != nil { + if !IsNotFound(err) { + return err + } + + if err := MakeDir(ctx, store, txn, path[:len(path)-1]); err != nil { + return err + } else if err := store.Write(ctx, txn, AddOp, path, map[string]interface{}{}); err != nil { + return err + } + + return nil + } + + if _, ok := node.(map[string]interface{}); ok { + return nil + } + + return writeConflictError(path) + +} + +// Txn is a convenience function that executes f inside a new transaction +// opened on the store. If the function returns an error, the transaction is +// aborted and the error is returned. Otherwise, the transaction is committed +// and the result of the commit is returned. +func Txn(ctx context.Context, store Store, params TransactionParams, f func(Transaction) error) error { + + txn, err := store.NewTransaction(ctx, params) + if err != nil { + return err + } + + if err := f(txn); err != nil { + store.Abort(ctx, txn) + return err + } + + return store.Commit(ctx, txn) +} + +// NonEmpty returns a function that tests if a path is non-empty. A +// path is non-empty if a Read on the path returns a value or a Read +// on any of the path prefixes returns a non-object value. +func NonEmpty(ctx context.Context, store Store, txn Transaction) func([]string) (bool, error) { + return func(path []string) (bool, error) { + if _, err := store.Read(ctx, txn, Path(path)); err == nil { + return true, nil + } else if !IsNotFound(err) { + return false, err + } + for i := len(path) - 1; i > 0; i-- { + val, err := store.Read(ctx, txn, Path(path[:i])) + if err != nil && !IsNotFound(err) { + return false, err + } else if err == nil { + if _, ok := val.(map[string]interface{}); ok { + return false, nil + } + return true, nil + } + } + return false, nil + } +} diff --git a/vendor/github.com/open-policy-agent/opa/topdown/aggregates.go b/vendor/github.com/open-policy-agent/opa/topdown/aggregates.go new file mode 100644 index 000000000..0b59487b6 --- /dev/null +++ b/vendor/github.com/open-policy-agent/opa/topdown/aggregates.go @@ -0,0 +1,216 @@ +// Copyright 2016 The OPA Authors. All rights reserved. +// Use of this source code is governed by an Apache2 +// license that can be found in the LICENSE file. + +package topdown + +import ( + "math/big" + + "github.com/open-policy-agent/opa/ast" + "github.com/open-policy-agent/opa/topdown/builtins" +) + +func builtinCount(a ast.Value) (ast.Value, error) { + switch a := a.(type) { + case ast.Array: + return ast.IntNumberTerm(len(a)).Value, nil + case ast.Object: + return ast.IntNumberTerm(a.Len()).Value, nil + case ast.Set: + return ast.IntNumberTerm(a.Len()).Value, nil + case ast.String: + return ast.IntNumberTerm(len(a)).Value, nil + } + return nil, builtins.NewOperandTypeErr(1, a, "array", "object", "set") +} + +func builtinSum(a ast.Value) (ast.Value, error) { + switch a := a.(type) { + case ast.Array: + sum := big.NewFloat(0) + for _, x := range a { + n, ok := x.Value.(ast.Number) + if !ok { + return nil, builtins.NewOperandElementErr(1, a, x.Value, "number") + } + sum = new(big.Float).Add(sum, builtins.NumberToFloat(n)) + } + return builtins.FloatToNumber(sum), nil + case ast.Set: + sum := big.NewFloat(0) + err := a.Iter(func(x *ast.Term) error { + n, ok := x.Value.(ast.Number) + if !ok { + return builtins.NewOperandElementErr(1, a, x.Value, "number") + } + sum = new(big.Float).Add(sum, builtins.NumberToFloat(n)) + return nil + }) + return builtins.FloatToNumber(sum), err + } + return nil, builtins.NewOperandTypeErr(1, a, "set", "array") +} + +func builtinProduct(a ast.Value) (ast.Value, error) { + switch a := a.(type) { + case ast.Array: + product := big.NewFloat(1) + for _, x := range a { + n, ok := x.Value.(ast.Number) + if !ok { + return nil, builtins.NewOperandElementErr(1, a, x.Value, "number") + } + product = new(big.Float).Mul(product, builtins.NumberToFloat(n)) + } + return builtins.FloatToNumber(product), nil + case ast.Set: + product := big.NewFloat(1) + err := a.Iter(func(x *ast.Term) error { + n, ok := x.Value.(ast.Number) + if !ok { + return builtins.NewOperandElementErr(1, a, x.Value, "number") + } + product = new(big.Float).Mul(product, builtins.NumberToFloat(n)) + return nil + }) + return builtins.FloatToNumber(product), err + } + return nil, builtins.NewOperandTypeErr(1, a, "set", "array") +} + +func builtinMax(a ast.Value) (ast.Value, error) { + switch a := a.(type) { + case ast.Array: + if len(a) == 0 { + return nil, BuiltinEmpty{} + } + var max = ast.Value(ast.Null{}) + for i := range a { + if ast.Compare(max, a[i].Value) <= 0 { + max = a[i].Value + } + } + return max, nil + case ast.Set: + if a.Len() == 0 { + return nil, BuiltinEmpty{} + } + max, err := a.Reduce(ast.NullTerm(), func(max *ast.Term, elem *ast.Term) (*ast.Term, error) { + if ast.Compare(max, elem) <= 0 { + return elem, nil + } + return max, nil + }) + return max.Value, err + } + + return nil, builtins.NewOperandTypeErr(1, a, "set", "array") +} + +func builtinMin(a ast.Value) (ast.Value, error) { + switch a := a.(type) { + case ast.Array: + if len(a) == 0 { + return nil, BuiltinEmpty{} + } + min := a[0].Value + for i := range a { + if ast.Compare(min, a[i].Value) >= 0 { + min = a[i].Value + } + } + return min, nil + case ast.Set: + if a.Len() == 0 { + return nil, BuiltinEmpty{} + } + min, err := a.Reduce(ast.NullTerm(), func(min *ast.Term, elem *ast.Term) (*ast.Term, error) { + // The null term is considered to be less than any other term, + // so in order for min of a set to make sense, we need to check + // for it. + if min.Value.Compare(ast.Null{}) == 0 { + return elem, nil + } + + if ast.Compare(min, elem) >= 0 { + return elem, nil + } + return min, nil + }) + return min.Value, err + } + + return nil, builtins.NewOperandTypeErr(1, a, "set", "array") +} + +func builtinSort(a ast.Value) (ast.Value, error) { + switch a := a.(type) { + case ast.Array: + return a.Sorted(), nil + case ast.Set: + return a.Sorted(), nil + } + return nil, builtins.NewOperandTypeErr(1, a, "set", "array") +} + +func builtinAll(a ast.Value) (ast.Value, error) { + switch val := a.(type) { + case ast.Set: + res := true + match := ast.BooleanTerm(true) + val.Foreach(func(term *ast.Term) { + if !term.Equal(match) { + res = false + } + }) + return ast.Boolean(res), nil + case ast.Array: + res := true + match := ast.BooleanTerm(true) + for _, term := range val { + if !term.Equal(match) { + res = false + } + } + return ast.Boolean(res), nil + default: + return nil, builtins.NewOperandTypeErr(1, a, "array", "set") + } +} + +func builtinAny(a ast.Value) (ast.Value, error) { + switch val := a.(type) { + case ast.Set: + res := false + match := ast.BooleanTerm(true) + val.Foreach(func(term *ast.Term) { + if term.Equal(match) { + res = true + } + }) + return ast.Boolean(res), nil + case ast.Array: + res := false + match := ast.BooleanTerm(true) + for _, term := range val { + if term.Equal(match) { + res = true + } + } + return ast.Boolean(res), nil + default: + return nil, builtins.NewOperandTypeErr(1, a, "array", "set") + } +} + +func init() { + RegisterFunctionalBuiltin1(ast.Count.Name, builtinCount) + RegisterFunctionalBuiltin1(ast.Sum.Name, builtinSum) + RegisterFunctionalBuiltin1(ast.Product.Name, builtinProduct) + RegisterFunctionalBuiltin1(ast.Max.Name, builtinMax) + RegisterFunctionalBuiltin1(ast.Min.Name, builtinMin) + RegisterFunctionalBuiltin1(ast.Sort.Name, builtinSort) + RegisterFunctionalBuiltin1(ast.Any.Name, builtinAny) + RegisterFunctionalBuiltin1(ast.All.Name, builtinAll) +} diff --git a/vendor/github.com/open-policy-agent/opa/topdown/arithmetic.go b/vendor/github.com/open-policy-agent/opa/topdown/arithmetic.go new file mode 100644 index 000000000..e481cf17b --- /dev/null +++ b/vendor/github.com/open-policy-agent/opa/topdown/arithmetic.go @@ -0,0 +1,156 @@ +// Copyright 2016 The OPA Authors. All rights reserved. +// Use of this source code is governed by an Apache2 +// license that can be found in the LICENSE file. + +package topdown + +import ( + "math/big" + + "fmt" + + "github.com/open-policy-agent/opa/ast" + "github.com/open-policy-agent/opa/topdown/builtins" +) + +type arithArity1 func(a *big.Float) (*big.Float, error) +type arithArity2 func(a, b *big.Float) (*big.Float, error) + +func arithAbs(a *big.Float) (*big.Float, error) { + return a.Abs(a), nil +} + +var halfAwayFromZero = big.NewFloat(0.5) + +func arithRound(a *big.Float) (*big.Float, error) { + var i *big.Int + if a.Signbit() { + i, _ = new(big.Float).Sub(a, halfAwayFromZero).Int(nil) + } else { + i, _ = new(big.Float).Add(a, halfAwayFromZero).Int(nil) + } + return new(big.Float).SetInt(i), nil +} + +func arithPlus(a, b *big.Float) (*big.Float, error) { + return new(big.Float).Add(a, b), nil +} + +func arithMinus(a, b *big.Float) (*big.Float, error) { + return new(big.Float).Sub(a, b), nil +} + +func arithMultiply(a, b *big.Float) (*big.Float, error) { + return new(big.Float).Mul(a, b), nil +} + +func arithDivide(a, b *big.Float) (*big.Float, error) { + i, acc := b.Int64() + if acc == big.Exact && i == 0 { + return nil, fmt.Errorf("divide by zero") + } + return new(big.Float).Quo(a, b), nil +} + +func arithRem(a, b *big.Int) (*big.Int, error) { + if b.Int64() == 0 { + return nil, fmt.Errorf("modulo by zero") + } + return new(big.Int).Rem(a, b), nil +} + +func builtinArithArity1(fn arithArity1) FunctionalBuiltin1 { + return func(a ast.Value) (ast.Value, error) { + n, err := builtins.NumberOperand(a, 1) + if err != nil { + return nil, err + } + f, err := fn(builtins.NumberToFloat(n)) + if err != nil { + return nil, err + } + return builtins.FloatToNumber(f), nil + } +} + +func builtinArithArity2(fn arithArity2) FunctionalBuiltin2 { + return func(a, b ast.Value) (ast.Value, error) { + n1, err := builtins.NumberOperand(a, 1) + if err != nil { + return nil, err + } + n2, err := builtins.NumberOperand(b, 2) + if err != nil { + return nil, err + } + f, err := fn(builtins.NumberToFloat(n1), builtins.NumberToFloat(n2)) + if err != nil { + return nil, err + } + return builtins.FloatToNumber(f), nil + } +} + +func builtinMinus(a, b ast.Value) (ast.Value, error) { + + n1, ok1 := a.(ast.Number) + n2, ok2 := b.(ast.Number) + + if ok1 && ok2 { + f, err := arithMinus(builtins.NumberToFloat(n1), builtins.NumberToFloat(n2)) + if err != nil { + return nil, err + } + return builtins.FloatToNumber(f), nil + } + + s1, ok3 := a.(ast.Set) + s2, ok4 := b.(ast.Set) + + if ok3 && ok4 { + return s1.Diff(s2), nil + } + + if !ok1 && !ok3 { + return nil, builtins.NewOperandTypeErr(1, a, "number", "set") + } + + return nil, builtins.NewOperandTypeErr(2, b, "number", "set") +} + +func builtinRem(a, b ast.Value) (ast.Value, error) { + n1, ok1 := a.(ast.Number) + n2, ok2 := b.(ast.Number) + + if ok1 && ok2 { + + op1, err1 := builtins.NumberToInt(n1) + op2, err2 := builtins.NumberToInt(n2) + + if err1 != nil || err2 != nil { + return nil, fmt.Errorf("modulo on floating-point number") + } + + i, err := arithRem(op1, op2) + if err != nil { + return nil, err + } + return builtins.IntToNumber(i), nil + } + + if !ok1 { + return nil, builtins.NewOperandTypeErr(1, a, "number") + } + + return nil, builtins.NewOperandTypeErr(2, b, "number") +} + +func init() { + RegisterFunctionalBuiltin1(ast.Abs.Name, builtinArithArity1(arithAbs)) + RegisterFunctionalBuiltin1(ast.Round.Name, builtinArithArity1(arithRound)) + RegisterFunctionalBuiltin2(ast.Plus.Name, builtinArithArity2(arithPlus)) + RegisterFunctionalBuiltin2(ast.Minus.Name, builtinMinus) + RegisterFunctionalBuiltin2(ast.Multiply.Name, builtinArithArity2(arithMultiply)) + RegisterFunctionalBuiltin2(ast.Divide.Name, builtinArithArity2(arithDivide)) + RegisterFunctionalBuiltin2(ast.Rem.Name, builtinRem) +} diff --git a/vendor/github.com/open-policy-agent/opa/topdown/array.go b/vendor/github.com/open-policy-agent/opa/topdown/array.go new file mode 100644 index 000000000..ee0398704 --- /dev/null +++ b/vendor/github.com/open-policy-agent/opa/topdown/array.go @@ -0,0 +1,75 @@ +// Copyright 2018 The OPA Authors. All rights reserved. +// Use of this source code is governed by an Apache2 +// license that can be found in the LICENSE file. + +package topdown + +import ( + "github.com/open-policy-agent/opa/ast" + "github.com/open-policy-agent/opa/topdown/builtins" +) + +func builtinArrayConcat(a, b ast.Value) (ast.Value, error) { + arrA, err := builtins.ArrayOperand(a, 1) + if err != nil { + return nil, err + } + + arrB, err := builtins.ArrayOperand(b, 2) + if err != nil { + return nil, err + } + + arrC := make(ast.Array, 0, len(arrA)+len(arrB)) + + for _, elemA := range arrA { + arrC = append(arrC, elemA) + } + + for _, elemB := range arrB { + arrC = append(arrC, elemB) + } + + return arrC, nil +} + +func builtinArraySlice(a, i, j ast.Value) (ast.Value, error) { + arr, err := builtins.ArrayOperand(a, 1) + if err != nil { + return nil, err + } + + startIndex, err := builtins.IntOperand(i, 2) + if err != nil { + return nil, err + } + + stopIndex, err := builtins.IntOperand(j, 3) + if err != nil { + return nil, err + } + + // Return empty array if bounds cannot be clamped sensibly. + if (startIndex >= stopIndex) || (startIndex <= 0 && stopIndex <= 0) { + return arr[0:0], nil + } + + // Clamp bounds to avoid out-of-range errors. + if startIndex < 0 { + startIndex = 0 + } + + if stopIndex > len(arr) { + stopIndex = len(arr) + } + + arrb := arr[startIndex:stopIndex] + + return arrb, nil + +} + +func init() { + RegisterFunctionalBuiltin2(ast.ArrayConcat.Name, builtinArrayConcat) + RegisterFunctionalBuiltin3(ast.ArraySlice.Name, builtinArraySlice) +} diff --git a/vendor/github.com/open-policy-agent/opa/topdown/binary.go b/vendor/github.com/open-policy-agent/opa/topdown/binary.go new file mode 100644 index 000000000..3cab5def1 --- /dev/null +++ b/vendor/github.com/open-policy-agent/opa/topdown/binary.go @@ -0,0 +1,45 @@ +// Copyright 2017 The OPA Authors. All rights reserved. +// Use of this source code is governed by an Apache2 +// license that can be found in the LICENSE file. + +package topdown + +import ( + "github.com/open-policy-agent/opa/ast" + "github.com/open-policy-agent/opa/topdown/builtins" +) + +func builtinBinaryAnd(a ast.Value, b ast.Value) (ast.Value, error) { + + s1, err := builtins.SetOperand(a, 1) + if err != nil { + return nil, err + } + + s2, err := builtins.SetOperand(b, 2) + if err != nil { + return nil, err + } + + return s1.Intersect(s2), nil +} + +func builtinBinaryOr(a ast.Value, b ast.Value) (ast.Value, error) { + + s1, err := builtins.SetOperand(a, 1) + if err != nil { + return nil, err + } + + s2, err := builtins.SetOperand(b, 2) + if err != nil { + return nil, err + } + + return s1.Union(s2), nil +} + +func init() { + RegisterFunctionalBuiltin2(ast.And.Name, builtinBinaryAnd) + RegisterFunctionalBuiltin2(ast.Or.Name, builtinBinaryOr) +} diff --git a/vendor/github.com/open-policy-agent/opa/topdown/bindings.go b/vendor/github.com/open-policy-agent/opa/topdown/bindings.go new file mode 100644 index 000000000..35e399415 --- /dev/null +++ b/vendor/github.com/open-policy-agent/opa/topdown/bindings.go @@ -0,0 +1,387 @@ +// Copyright 2017 The OPA Authors. All rights reserved. +// Use of this source code is governed by an Apache2 +// license that can be found in the LICENSE file. + +package topdown + +import ( + "fmt" + "strings" + + "github.com/open-policy-agent/opa/ast" +) + +type undo struct { + k *ast.Term + u *bindings + next *undo +} + +func (u *undo) Undo() { + if u == nil { + // Allow call on zero value of Undo for ease-of-use. + return + } + if u.u == nil { + // Call on empty unifier undos a no-op unify operation. + return + } + u.u.delete(u.k) + u.next.Undo() +} + +type bindings struct { + id uint64 + values bindingsArrayHashmap + instr *Instrumentation +} + +func newBindings(id uint64, instr *Instrumentation) *bindings { + values := newBindingsArrayHashmap() + return &bindings{id, values, instr} +} + +func (u *bindings) Iter(caller *bindings, iter func(*ast.Term, *ast.Term) error) error { + + var err error + + u.values.Iter(func(k *ast.Term, v value) bool { + if err != nil { + return true + } + err = iter(k, u.PlugNamespaced(k, caller)) + + return false + }) + + return err +} + +func (u *bindings) Namespace(x ast.Node, caller *bindings) { + vis := namespacingVisitor{ + b: u, + caller: caller, + } + ast.NewGenericVisitor(vis.Visit).Walk(x) +} + +func (u *bindings) Plug(a *ast.Term) *ast.Term { + return u.PlugNamespaced(a, nil) +} + +func (u *bindings) PlugNamespaced(a *ast.Term, caller *bindings) *ast.Term { + if u != nil { + u.instr.startTimer(evalOpPlug) + t := u.plugNamespaced(a, caller) + u.instr.stopTimer(evalOpPlug) + return t + } + + return u.plugNamespaced(a, caller) +} + +func (u *bindings) plugNamespaced(a *ast.Term, caller *bindings) *ast.Term { + switch v := a.Value.(type) { + case ast.Var: + b, next := u.apply(a) + if a != b || u != next { + return next.plugNamespaced(b, caller) + } + return u.namespaceVar(b, caller) + case ast.Array: + cpy := *a + arr := make(ast.Array, len(v)) + for i := 0; i < len(arr); i++ { + arr[i] = u.plugNamespaced(v[i], caller) + } + cpy.Value = arr + return &cpy + case ast.Object: + if a.IsGround() { + return a + } + cpy := *a + cpy.Value, _ = v.Map(func(k, v *ast.Term) (*ast.Term, *ast.Term, error) { + return u.plugNamespaced(k, caller), u.plugNamespaced(v, caller), nil + }) + return &cpy + case ast.Set: + cpy := *a + cpy.Value, _ = v.Map(func(x *ast.Term) (*ast.Term, error) { + return u.plugNamespaced(x, caller), nil + }) + return &cpy + case ast.Ref: + cpy := *a + ref := make(ast.Ref, len(v)) + for i := 0; i < len(ref); i++ { + ref[i] = u.plugNamespaced(v[i], caller) + } + cpy.Value = ref + return &cpy + } + return a +} + +func (u *bindings) bind(a *ast.Term, b *ast.Term, other *bindings) *undo { + u.values.Put(a, value{ + u: other, + v: b, + }) + return &undo{a, u, nil} +} + +func (u *bindings) apply(a *ast.Term) (*ast.Term, *bindings) { + // Early exit for non-var terms. Only vars are bound in the binding list, + // so the lookup below will always fail for non-var terms. In some cases, + // the lookup may be expensive as it has to hash the term (which for large + // inputs can be costly.) + _, ok := a.Value.(ast.Var) + if !ok { + return a, u + } + val, ok := u.get(a) + if !ok { + return a, u + } + return val.u.apply(val.v) +} + +func (u *bindings) delete(v *ast.Term) { + u.values.Delete(v) +} + +func (u *bindings) get(v *ast.Term) (value, bool) { + if u == nil { + return value{}, false + } + return u.values.Get(v) +} + +func (u *bindings) String() string { + if u == nil { + return "()" + } + var buf []string + u.values.Iter(func(a *ast.Term, b value) bool { + buf = append(buf, fmt.Sprintf("%v: %v", a, b)) + return false + }) + return fmt.Sprintf("({%v}, %v)", strings.Join(buf, ", "), u.id) +} + +func (u *bindings) namespaceVar(v *ast.Term, caller *bindings) *ast.Term { + name, ok := v.Value.(ast.Var) + if !ok { + panic("illegal value") + } + if caller != nil && caller != u { + // Root documents (i.e., data, input) should never be namespaced because they + // are globally unique. + if !ast.RootDocumentNames.Contains(v) { + return ast.NewTerm(ast.Var(string(name) + fmt.Sprint(u.id))) + } + } + return v +} + +type value struct { + u *bindings + v *ast.Term +} + +func (v value) String() string { + return fmt.Sprintf("(%v, %d)", v.v, v.u.id) +} + +func (v value) equal(other *value) bool { + if v.u == other.u { + return v.v.Equal(other.v) + } + return false +} + +type namespacingVisitor struct { + b *bindings + caller *bindings +} + +func (vis namespacingVisitor) Visit(x interface{}) bool { + switch x := x.(type) { + case *ast.ArrayComprehension: + x.Term = vis.namespaceTerm(x.Term) + ast.NewGenericVisitor(vis.Visit).Walk(x.Body) + return true + case *ast.SetComprehension: + x.Term = vis.namespaceTerm(x.Term) + ast.NewGenericVisitor(vis.Visit).Walk(x.Body) + return true + case *ast.ObjectComprehension: + x.Key = vis.namespaceTerm(x.Key) + x.Value = vis.namespaceTerm(x.Value) + ast.NewGenericVisitor(vis.Visit).Walk(x.Body) + return true + case *ast.Expr: + switch terms := x.Terms.(type) { + case []*ast.Term: + for i := 1; i < len(terms); i++ { + terms[i] = vis.namespaceTerm(terms[i]) + } + case *ast.Term: + x.Terms = vis.namespaceTerm(terms) + } + for _, w := range x.With { + w.Target = vis.namespaceTerm(w.Target) + w.Value = vis.namespaceTerm(w.Value) + } + } + return false +} + +func (vis namespacingVisitor) namespaceTerm(a *ast.Term) *ast.Term { + switch v := a.Value.(type) { + case ast.Var: + return vis.b.namespaceVar(a, vis.caller) + case ast.Array: + cpy := *a + arr := make(ast.Array, len(v)) + for i := 0; i < len(arr); i++ { + arr[i] = vis.namespaceTerm(v[i]) + } + cpy.Value = arr + return &cpy + case ast.Object: + if a.IsGround() { + return a + } + cpy := *a + cpy.Value, _ = v.Map(func(k, v *ast.Term) (*ast.Term, *ast.Term, error) { + return vis.namespaceTerm(k), vis.namespaceTerm(v), nil + }) + return &cpy + case ast.Set: + cpy := *a + cpy.Value, _ = v.Map(func(x *ast.Term) (*ast.Term, error) { + return vis.namespaceTerm(x), nil + }) + return &cpy + case ast.Ref: + cpy := *a + ref := make(ast.Ref, len(v)) + for i := 0; i < len(ref); i++ { + ref[i] = vis.namespaceTerm(v[i]) + } + cpy.Value = ref + return &cpy + } + return a +} + +const maxLinearScan = 16 + +// bindingsArrayHashMap uses an array with linear scan instead of a hash map for smaller # of entries. Hash maps start to show off their performance advantage only after 16 keys. +type bindingsArrayHashmap struct { + n int // Entries in the array. + a *[maxLinearScan]bindingArrayKeyValue + m map[ast.Var]bindingArrayKeyValue +} + +type bindingArrayKeyValue struct { + key *ast.Term + value value +} + +func newBindingsArrayHashmap() bindingsArrayHashmap { + return bindingsArrayHashmap{} +} + +func (b *bindingsArrayHashmap) Put(key *ast.Term, value value) { + if b.m == nil { + if b.a == nil { + b.a = new([maxLinearScan]bindingArrayKeyValue) + } else if i := b.find(key); i >= 0 { + (*b.a)[i].value = value + return + } + + if b.n < maxLinearScan { + (*b.a)[b.n] = bindingArrayKeyValue{key, value} + b.n++ + return + } + + // Array is full, revert to using the hash map instead. + + b.m = make(map[ast.Var]bindingArrayKeyValue, maxLinearScan+1) + for _, kv := range *b.a { + b.m[kv.key.Value.(ast.Var)] = bindingArrayKeyValue{kv.key, kv.value} + } + b.m[key.Value.(ast.Var)] = bindingArrayKeyValue{key, value} + + b.n = 0 + return + } + + b.m[key.Value.(ast.Var)] = bindingArrayKeyValue{key, value} +} + +func (b *bindingsArrayHashmap) Get(key *ast.Term) (value, bool) { + if b.m == nil { + if i := b.find(key); i >= 0 { + return (*b.a)[i].value, true + } + + return value{}, false + } + + v, ok := b.m[key.Value.(ast.Var)] + if ok { + return v.value, true + } + + return value{}, false +} + +func (b *bindingsArrayHashmap) Delete(key *ast.Term) { + if b.m == nil { + if i := b.find(key); i >= 0 { + n := b.n - 1 + if i < n { + (*b.a)[i] = (*b.a)[n] + } + + b.n = n + } + return + } + + delete(b.m, key.Value.(ast.Var)) +} + +func (b *bindingsArrayHashmap) Iter(f func(k *ast.Term, v value) bool) { + if b.m == nil { + for i := 0; i < b.n; i++ { + if f((*b.a)[i].key, (*b.a)[i].value) { + return + } + } + return + } + + for _, v := range b.m { + if f(v.key, v.value) { + return + } + } +} + +func (b *bindingsArrayHashmap) find(key *ast.Term) int { + v := key.Value.(ast.Var) + for i := 0; i < b.n; i++ { + if (*b.a)[i].key.Value.(ast.Var) == v { + return i + } + } + + return -1 +} diff --git a/vendor/github.com/open-policy-agent/opa/topdown/bits.go b/vendor/github.com/open-policy-agent/opa/topdown/bits.go new file mode 100644 index 000000000..7a63c0df1 --- /dev/null +++ b/vendor/github.com/open-policy-agent/opa/topdown/bits.go @@ -0,0 +1,88 @@ +// Copyright 2020 The OPA Authors. All rights reserved. +// Use of this source code is governed by an Apache2 +// license that can be found in the LICENSE file. + +package topdown + +import ( + "math/big" + + "github.com/open-policy-agent/opa/ast" + "github.com/open-policy-agent/opa/topdown/builtins" +) + +type bitsArity1 func(a *big.Int) (*big.Int, error) +type bitsArity2 func(a, b *big.Int) (*big.Int, error) + +func bitsOr(a, b *big.Int) (*big.Int, error) { + return new(big.Int).Or(a, b), nil +} + +func bitsAnd(a, b *big.Int) (*big.Int, error) { + return new(big.Int).And(a, b), nil +} + +func bitsNegate(a *big.Int) (*big.Int, error) { + return new(big.Int).Not(a), nil +} + +func bitsXOr(a, b *big.Int) (*big.Int, error) { + return new(big.Int).Xor(a, b), nil +} + +func bitsShiftLeft(a, b *big.Int) (*big.Int, error) { + if b.Sign() == -1 { + return nil, builtins.NewOperandErr(2, "must be an unsigned integer number but got a negative integer") + } + shift := uint(b.Uint64()) + return new(big.Int).Lsh(a, shift), nil +} + +func bitsShiftRight(a, b *big.Int) (*big.Int, error) { + if b.Sign() == -1 { + return nil, builtins.NewOperandErr(2, "must be an unsigned integer number but got a negative integer") + } + shift := uint(b.Uint64()) + return new(big.Int).Rsh(a, shift), nil +} + +func builtinBitsArity1(fn bitsArity1) BuiltinFunc { + return func(_ BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error { + i, err := builtins.BigIntOperand(operands[0].Value, 1) + if err != nil { + return err + } + iOut, err := fn(i) + if err != nil { + return err + } + return iter(ast.NewTerm(builtins.IntToNumber(iOut))) + } +} + +func builtinBitsArity2(fn bitsArity2) BuiltinFunc { + return func(_ BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error { + i1, err := builtins.BigIntOperand(operands[0].Value, 1) + if err != nil { + return err + } + i2, err := builtins.BigIntOperand(operands[1].Value, 2) + if err != nil { + return err + } + iOut, err := fn(i1, i2) + if err != nil { + return err + } + return iter(ast.NewTerm(builtins.IntToNumber(iOut))) + } +} + +func init() { + RegisterBuiltinFunc(ast.BitsOr.Name, builtinBitsArity2(bitsOr)) + RegisterBuiltinFunc(ast.BitsAnd.Name, builtinBitsArity2(bitsAnd)) + RegisterBuiltinFunc(ast.BitsNegate.Name, builtinBitsArity1(bitsNegate)) + RegisterBuiltinFunc(ast.BitsXOr.Name, builtinBitsArity2(bitsXOr)) + RegisterBuiltinFunc(ast.BitsShiftLeft.Name, builtinBitsArity2(bitsShiftLeft)) + RegisterBuiltinFunc(ast.BitsShiftRight.Name, builtinBitsArity2(bitsShiftRight)) +} diff --git a/vendor/github.com/open-policy-agent/opa/topdown/builtins.go b/vendor/github.com/open-policy-agent/opa/topdown/builtins.go new file mode 100644 index 000000000..36e333f88 --- /dev/null +++ b/vendor/github.com/open-policy-agent/opa/topdown/builtins.go @@ -0,0 +1,160 @@ +// Copyright 2016 The OPA Authors. All rights reserved. +// Use of this source code is governed by an Apache2 +// license that can be found in the LICENSE file. + +package topdown + +import ( + "context" + "fmt" + + "github.com/open-policy-agent/opa/ast" + "github.com/open-policy-agent/opa/topdown/builtins" +) + +type ( + // FunctionalBuiltin1 is deprecated. Use BuiltinFunc instead. + FunctionalBuiltin1 func(op1 ast.Value) (output ast.Value, err error) + + // FunctionalBuiltin2 is deprecated. Use BuiltinFunc instead. + FunctionalBuiltin2 func(op1, op2 ast.Value) (output ast.Value, err error) + + // FunctionalBuiltin3 is deprecated. Use BuiltinFunc instead. + FunctionalBuiltin3 func(op1, op2, op3 ast.Value) (output ast.Value, err error) + + // FunctionalBuiltin4 is deprecated. Use BuiltinFunc instead. + FunctionalBuiltin4 func(op1, op2, op3, op4 ast.Value) (output ast.Value, err error) + + // BuiltinContext contains context from the evaluator that may be used by + // built-in functions. + BuiltinContext struct { + Context context.Context // request context that was passed when query started + Cancel Cancel // atomic value that signals evaluation to halt + Runtime *ast.Term // runtime information on the OPA instance + Cache builtins.Cache // built-in function state cache + Location *ast.Location // location of built-in call + Tracers []Tracer // tracer objects for trace() built-in function + QueryID uint64 // identifies query being evaluated + ParentID uint64 // identifies parent of query being evaluated + } + + // BuiltinFunc defines an interface for implementing built-in functions. + // The built-in function is called with the plugged operands from the call + // (including the output operands.) The implementation should evaluate the + // operands and invoke the iterator for each successful/defined output + // value. + BuiltinFunc func(bctx BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error +) + +// RegisterBuiltinFunc adds a new built-in function to the evaluation engine. +func RegisterBuiltinFunc(name string, f BuiltinFunc) { + builtinFunctions[name] = builtinErrorWrapper(name, f) +} + +// RegisterFunctionalBuiltin1 is deprecated use RegisterBuiltinFunc instead. +func RegisterFunctionalBuiltin1(name string, fun FunctionalBuiltin1) { + builtinFunctions[name] = functionalWrapper1(name, fun) +} + +// RegisterFunctionalBuiltin2 is deprecated use RegisterBuiltinFunc instead. +func RegisterFunctionalBuiltin2(name string, fun FunctionalBuiltin2) { + builtinFunctions[name] = functionalWrapper2(name, fun) +} + +// RegisterFunctionalBuiltin3 is deprecated use RegisterBuiltinFunc instead. +func RegisterFunctionalBuiltin3(name string, fun FunctionalBuiltin3) { + builtinFunctions[name] = functionalWrapper3(name, fun) +} + +// RegisterFunctionalBuiltin4 is deprecated use RegisterBuiltinFunc instead. +func RegisterFunctionalBuiltin4(name string, fun FunctionalBuiltin4) { + builtinFunctions[name] = functionalWrapper4(name, fun) +} + +// GetBuiltin returns a built-in function implementation, nil if no built-in found. +func GetBuiltin(name string) BuiltinFunc { + return builtinFunctions[name] +} + +// BuiltinEmpty is deprecated. +type BuiltinEmpty struct{} + +func (BuiltinEmpty) Error() string { + return "" +} + +var builtinFunctions = map[string]BuiltinFunc{} + +func builtinErrorWrapper(name string, fn BuiltinFunc) BuiltinFunc { + return func(bctx BuiltinContext, args []*ast.Term, iter func(*ast.Term) error) error { + err := fn(bctx, args, iter) + if err == nil { + return nil + } + return handleBuiltinErr(name, bctx.Location, err) + } +} + +func functionalWrapper1(name string, fn FunctionalBuiltin1) BuiltinFunc { + return func(bctx BuiltinContext, args []*ast.Term, iter func(*ast.Term) error) error { + result, err := fn(args[0].Value) + if err == nil { + return iter(ast.NewTerm(result)) + } + return handleBuiltinErr(name, bctx.Location, err) + } +} + +func functionalWrapper2(name string, fn FunctionalBuiltin2) BuiltinFunc { + return func(bctx BuiltinContext, args []*ast.Term, iter func(*ast.Term) error) error { + result, err := fn(args[0].Value, args[1].Value) + if err == nil { + return iter(ast.NewTerm(result)) + } + return handleBuiltinErr(name, bctx.Location, err) + } +} + +func functionalWrapper3(name string, fn FunctionalBuiltin3) BuiltinFunc { + return func(bctx BuiltinContext, args []*ast.Term, iter func(*ast.Term) error) error { + result, err := fn(args[0].Value, args[1].Value, args[2].Value) + if err == nil { + return iter(ast.NewTerm(result)) + } + return handleBuiltinErr(name, bctx.Location, err) + } +} + +func functionalWrapper4(name string, fn FunctionalBuiltin4) BuiltinFunc { + return func(bctx BuiltinContext, args []*ast.Term, iter func(*ast.Term) error) error { + result, err := fn(args[0].Value, args[1].Value, args[2].Value, args[3].Value) + if err == nil { + return iter(ast.NewTerm(result)) + } + if _, empty := err.(BuiltinEmpty); empty { + return nil + } + return handleBuiltinErr(name, bctx.Location, err) + } +} + +func handleBuiltinErr(name string, loc *ast.Location, err error) error { + switch err := err.(type) { + case BuiltinEmpty: + return nil + case *Error: + return err + case builtins.ErrOperand: + return &Error{ + Code: TypeErr, + Message: fmt.Sprintf("%v: %v", string(name), err.Error()), + Location: loc, + } + default: + return &Error{ + Code: BuiltinErr, + Message: fmt.Sprintf("%v: %v", string(name), err.Error()), + Location: loc, + } + } +} diff --git a/vendor/github.com/open-policy-agent/opa/topdown/builtins/builtins.go b/vendor/github.com/open-policy-agent/opa/topdown/builtins/builtins.go new file mode 100644 index 000000000..861167f39 --- /dev/null +++ b/vendor/github.com/open-policy-agent/opa/topdown/builtins/builtins.go @@ -0,0 +1,235 @@ +// Copyright 2016 The OPA Authors. All rights reserved. +// Use of this source code is governed by an Apache2 +// license that can be found in the LICENSE file. + +// Package builtins contains utilities for implementing built-in functions. +package builtins + +import ( + "fmt" + "math/big" + "strings" + + "github.com/open-policy-agent/opa/ast" +) + +// Cache defines the built-in cache used by the top-down evaluation. The keys +// must be comparable and should not be of type string. +type Cache map[interface{}]interface{} + +// Put updates the cache for the named built-in. +func (c Cache) Put(k, v interface{}) { + c[k] = v +} + +// Get returns the cached value for k. +func (c Cache) Get(k interface{}) (interface{}, bool) { + v, ok := c[k] + return v, ok +} + +// ErrOperand represents an invalid operand has been passed to a built-in +// function. Built-ins should return ErrOperand to indicate a type error has +// occurred. +type ErrOperand string + +func (err ErrOperand) Error() string { + return string(err) +} + +// NewOperandErr returns a generic operand error. +func NewOperandErr(pos int, f string, a ...interface{}) error { + f = fmt.Sprintf("operand %v ", pos) + f + return ErrOperand(fmt.Sprintf(f, a...)) +} + +// NewOperandTypeErr returns an operand error indicating the operand's type was wrong. +func NewOperandTypeErr(pos int, got ast.Value, expected ...string) error { + + if len(expected) == 1 { + return NewOperandErr(pos, "must be %v but got %v", expected[0], ast.TypeName(got)) + } + + return NewOperandErr(pos, "must be one of {%v} but got %v", strings.Join(expected, ", "), ast.TypeName(got)) +} + +// NewOperandElementErr returns an operand error indicating an element in the +// composite operand was wrong. +func NewOperandElementErr(pos int, composite ast.Value, got ast.Value, expected ...string) error { + + tpe := ast.TypeName(composite) + + if len(expected) == 1 { + return NewOperandErr(pos, "must be %v of %vs but got %v containing %v", tpe, expected[0], tpe, ast.TypeName(got)) + } + + return NewOperandErr(pos, "must be %v of (any of) {%v} but got %v containing %v", tpe, strings.Join(expected, ", "), tpe, ast.TypeName(got)) +} + +// NewOperandEnumErr returns an operand error indicating a value was wrong. +func NewOperandEnumErr(pos int, expected ...string) error { + + if len(expected) == 1 { + return NewOperandErr(pos, "must be %v", expected[0]) + } + + return NewOperandErr(pos, "must be one of {%v}", strings.Join(expected, ", ")) +} + +// IntOperand converts x to an int. If the cast fails, a descriptive error is +// returned. +func IntOperand(x ast.Value, pos int) (int, error) { + n, ok := x.(ast.Number) + if !ok { + return 0, NewOperandTypeErr(pos, x, "number") + } + + i, ok := n.Int() + if !ok { + return 0, NewOperandErr(pos, "must be integer number but got floating-point number") + } + + return i, nil +} + +// BigIntOperand converts x to a big int. If the cast fails, a descriptive error +// is returned. +func BigIntOperand(x ast.Value, pos int) (*big.Int, error) { + n, err := NumberOperand(x, 1) + if err != nil { + return nil, NewOperandTypeErr(pos, x, "integer") + } + bi, err := NumberToInt(n) + if err != nil { + return nil, NewOperandErr(pos, "must be integer number but got floating-point number") + } + + return bi, nil +} + +// NumberOperand converts x to a number. If the cast fails, a descriptive error is +// returned. +func NumberOperand(x ast.Value, pos int) (ast.Number, error) { + n, ok := x.(ast.Number) + if !ok { + return ast.Number(""), NewOperandTypeErr(pos, x, "number") + } + return n, nil +} + +// SetOperand converts x to a set. If the cast fails, a descriptive error is +// returned. +func SetOperand(x ast.Value, pos int) (ast.Set, error) { + s, ok := x.(ast.Set) + if !ok { + return nil, NewOperandTypeErr(pos, x, "set") + } + return s, nil +} + +// StringOperand converts x to a string. If the cast fails, a descriptive error is +// returned. +func StringOperand(x ast.Value, pos int) (ast.String, error) { + s, ok := x.(ast.String) + if !ok { + return ast.String(""), NewOperandTypeErr(pos, x, "string") + } + return s, nil +} + +// ObjectOperand converts x to an object. If the cast fails, a descriptive +// error is returned. +func ObjectOperand(x ast.Value, pos int) (ast.Object, error) { + o, ok := x.(ast.Object) + if !ok { + return nil, NewOperandTypeErr(pos, x, "object") + } + return o, nil +} + +// ArrayOperand converts x to an array. If the cast fails, a descriptive +// error is returned. +func ArrayOperand(x ast.Value, pos int) (ast.Array, error) { + a, ok := x.(ast.Array) + if !ok { + return nil, NewOperandTypeErr(pos, x, "array") + } + return a, nil +} + +// NumberToFloat converts n to a big float. +func NumberToFloat(n ast.Number) *big.Float { + r, ok := new(big.Float).SetString(string(n)) + if !ok { + panic("illegal value") + } + return r +} + +// FloatToNumber converts f to a number. +func FloatToNumber(f *big.Float) ast.Number { + return ast.Number(f.String()) +} + +// NumberToInt converts n to a big int. +// If n cannot be converted to an big int, an error is returned. +func NumberToInt(n ast.Number) (*big.Int, error) { + f := NumberToFloat(n) + r, accuracy := f.Int(nil) + if accuracy != big.Exact { + return nil, fmt.Errorf("illegal value") + } + return r, nil +} + +// IntToNumber converts i to a number. +func IntToNumber(i *big.Int) ast.Number { + return ast.Number(i.String()) +} + +// StringSliceOperand converts x to a []string. If the cast fails, a descriptive error is +// returned. +func StringSliceOperand(x ast.Value, pos int) ([]string, error) { + a, err := ArrayOperand(x, pos) + if err != nil { + return nil, err + } + + var f = make([]string, len(a)) + for k, b := range a { + c, ok := b.Value.(ast.String) + if !ok { + return nil, NewOperandElementErr(pos, x, b.Value, "[]string") + } + + f[k] = string(c) + } + + return f, nil +} + +// RuneSliceOperand converts x to a []rune. If the cast fails, a descriptive error is +// returned. +func RuneSliceOperand(x ast.Value, pos int) ([]rune, error) { + a, err := ArrayOperand(x, pos) + if err != nil { + return nil, err + } + + var f = make([]rune, len(a)) + for k, b := range a { + c, ok := b.Value.(ast.String) + if !ok { + return nil, NewOperandElementErr(pos, x, b.Value, "string") + } + + d := []rune(string(c)) + if len(d) != 1 { + return nil, NewOperandElementErr(pos, x, b.Value, "rune") + } + + f[k] = d[0] + } + + return f, nil +} diff --git a/vendor/github.com/open-policy-agent/opa/topdown/cache.go b/vendor/github.com/open-policy-agent/opa/topdown/cache.go new file mode 100644 index 000000000..33a8b16f2 --- /dev/null +++ b/vendor/github.com/open-policy-agent/opa/topdown/cache.go @@ -0,0 +1,166 @@ +// Copyright 2017 The OPA Authors. All rights reserved. +// Use of this source code is governed by an Apache2 +// license that can be found in the LICENSE file. + +package topdown + +import ( + "github.com/open-policy-agent/opa/ast" + "github.com/open-policy-agent/opa/util" +) + +type virtualCache struct { + stack []*virtualCacheElem +} + +type virtualCacheElem struct { + value *ast.Term + children *util.HashMap +} + +func newVirtualCache() *virtualCache { + cache := &virtualCache{} + cache.Push() + return cache +} + +func (c *virtualCache) Push() { + c.stack = append(c.stack, newVirtualCacheElem()) +} + +func (c *virtualCache) Pop() { + c.stack = c.stack[:len(c.stack)-1] +} + +func (c *virtualCache) Get(ref ast.Ref) *ast.Term { + node := c.stack[len(c.stack)-1] + for i := 0; i < len(ref); i++ { + x, ok := node.children.Get(ref[i]) + if !ok { + return nil + } + node = x.(*virtualCacheElem) + } + return node.value +} + +func (c *virtualCache) Put(ref ast.Ref, value *ast.Term) { + node := c.stack[len(c.stack)-1] + for i := 0; i < len(ref); i++ { + x, ok := node.children.Get(ref[i]) + if ok { + node = x.(*virtualCacheElem) + } else { + next := newVirtualCacheElem() + node.children.Put(ref[i], next) + node = next + } + } + node.value = value +} + +func newVirtualCacheElem() *virtualCacheElem { + return &virtualCacheElem{children: newVirtualCacheHashMap()} +} + +func newVirtualCacheHashMap() *util.HashMap { + return util.NewHashMap(func(a, b util.T) bool { + return a.(*ast.Term).Equal(b.(*ast.Term)) + }, func(x util.T) int { + return x.(*ast.Term).Hash() + }) +} + +// baseCache implements a trie structure to cache base documents read out of +// storage. Values inserted into the cache may contain other values that were +// previously inserted. In this case, the previous values are erased from the +// structure. +type baseCache struct { + root *baseCacheElem +} + +func newBaseCache() *baseCache { + return &baseCache{ + root: newBaseCacheElem(), + } +} + +func (c *baseCache) Get(ref ast.Ref) ast.Value { + node := c.root + for i := 0; i < len(ref); i++ { + node = node.children[ref[i].Value] + if node == nil { + return nil + } else if node.value != nil { + result, err := node.value.Find(ref[i+1:]) + if err != nil { + return nil + } + return result + } + } + return nil +} + +func (c *baseCache) Put(ref ast.Ref, value ast.Value) { + node := c.root + for i := 0; i < len(ref); i++ { + if child, ok := node.children[ref[i].Value]; ok { + node = child + } else { + child := newBaseCacheElem() + node.children[ref[i].Value] = child + node = child + } + } + node.set(value) +} + +type baseCacheElem struct { + value ast.Value + children map[ast.Value]*baseCacheElem +} + +func newBaseCacheElem() *baseCacheElem { + return &baseCacheElem{ + children: map[ast.Value]*baseCacheElem{}, + } +} + +func (e *baseCacheElem) set(value ast.Value) { + e.value = value + e.children = map[ast.Value]*baseCacheElem{} +} + +type refStack struct { + sl []refStackElem +} + +type refStackElem struct { + refs []ast.Ref +} + +func newRefStack() *refStack { + return &refStack{} +} + +func (s *refStack) Push(refs []ast.Ref) { + s.sl = append(s.sl, refStackElem{refs: refs}) +} + +func (s *refStack) Pop() { + s.sl = s.sl[:len(s.sl)-1] +} + +func (s *refStack) Prefixed(ref ast.Ref) bool { + if s != nil { + for i := len(s.sl) - 1; i >= 0; i-- { + for j := range s.sl[i].refs { + if ref.HasPrefix(s.sl[i].refs[j]) { + return true + } + } + } + } + return false +} diff --git a/vendor/github.com/open-policy-agent/opa/topdown/cancel.go b/vendor/github.com/open-policy-agent/opa/topdown/cancel.go new file mode 100644 index 000000000..534e0799a --- /dev/null +++ b/vendor/github.com/open-policy-agent/opa/topdown/cancel.go @@ -0,0 +1,33 @@ +// Copyright 2017 The OPA Authors. All rights reserved. +// Use of this source code is governed by an Apache2 +// license that can be found in the LICENSE file. + +package topdown + +import ( + "sync/atomic" +) + +// Cancel defines the interface for cancelling topdown queries. Cancel +// operations are thread-safe and idempotent. +type Cancel interface { + Cancel() + Cancelled() bool +} + +type cancel struct { + flag int32 +} + +// NewCancel returns a new Cancel object. +func NewCancel() Cancel { + return &cancel{} +} + +func (c *cancel) Cancel() { + atomic.StoreInt32(&c.flag, 1) +} + +func (c *cancel) Cancelled() bool { + return atomic.LoadInt32(&c.flag) != 0 +} diff --git a/vendor/github.com/open-policy-agent/opa/topdown/casts.go b/vendor/github.com/open-policy-agent/opa/topdown/casts.go new file mode 100644 index 000000000..d79aac321 --- /dev/null +++ b/vendor/github.com/open-policy-agent/opa/topdown/casts.go @@ -0,0 +1,113 @@ +// Copyright 2018 The OPA Authors. All rights reserved. +// Use of this source code is governed by an Apache2 +// license that can be found in the LICENSE file. + +package topdown + +import ( + "strconv" + + "github.com/open-policy-agent/opa/ast" + "github.com/open-policy-agent/opa/topdown/builtins" +) + +func builtinToNumber(a ast.Value) (ast.Value, error) { + switch a := a.(type) { + case ast.Null: + return ast.Number("0"), nil + case ast.Boolean: + if a { + return ast.Number("1"), nil + } + return ast.Number("0"), nil + case ast.Number: + return a, nil + case ast.String: + _, err := strconv.ParseFloat(string(a), 64) + if err != nil { + return nil, err + } + return ast.Number(a), nil + } + return nil, builtins.NewOperandTypeErr(1, a, "null", "boolean", "number", "string") +} + +// Deprecated in v0.13.0. +func builtinToArray(a ast.Value) (ast.Value, error) { + switch val := a.(type) { + case ast.Array: + return val, nil + case ast.Set: + arr := make(ast.Array, val.Len()) + i := 0 + val.Foreach(func(term *ast.Term) { + arr[i] = term + i++ + }) + return arr, nil + default: + return nil, builtins.NewOperandTypeErr(1, a, "array", "set") + } +} + +// Deprecated in v0.13.0. +func builtinToSet(a ast.Value) (ast.Value, error) { + switch val := a.(type) { + case ast.Array: + return ast.NewSet(val...), nil + case ast.Set: + return val, nil + default: + return nil, builtins.NewOperandTypeErr(1, a, "array", "set") + } +} + +// Deprecated in v0.13.0. +func builtinToString(a ast.Value) (ast.Value, error) { + switch val := a.(type) { + case ast.String: + return val, nil + default: + return nil, builtins.NewOperandTypeErr(1, a, "string") + } +} + +// Deprecated in v0.13.0. +func builtinToBoolean(a ast.Value) (ast.Value, error) { + switch val := a.(type) { + case ast.Boolean: + return val, nil + default: + return nil, builtins.NewOperandTypeErr(1, a, "boolean") + } +} + +// Deprecated in v0.13.0. +func builtinToNull(a ast.Value) (ast.Value, error) { + switch val := a.(type) { + case ast.Null: + return val, nil + default: + return nil, builtins.NewOperandTypeErr(1, a, "null") + } +} + +// Deprecated in v0.13.0. +func builtinToObject(a ast.Value) (ast.Value, error) { + switch val := a.(type) { + case ast.Object: + return val, nil + default: + return nil, builtins.NewOperandTypeErr(1, a, "object") + } +} + +func init() { + RegisterFunctionalBuiltin1(ast.ToNumber.Name, builtinToNumber) + RegisterFunctionalBuiltin1(ast.CastArray.Name, builtinToArray) + RegisterFunctionalBuiltin1(ast.CastSet.Name, builtinToSet) + RegisterFunctionalBuiltin1(ast.CastString.Name, builtinToString) + RegisterFunctionalBuiltin1(ast.CastBoolean.Name, builtinToBoolean) + RegisterFunctionalBuiltin1(ast.CastNull.Name, builtinToNull) + RegisterFunctionalBuiltin1(ast.CastObject.Name, builtinToObject) +} diff --git a/vendor/github.com/open-policy-agent/opa/topdown/cidr.go b/vendor/github.com/open-policy-agent/opa/topdown/cidr.go new file mode 100644 index 000000000..b65628dae --- /dev/null +++ b/vendor/github.com/open-policy-agent/opa/topdown/cidr.go @@ -0,0 +1,157 @@ +package topdown + +import ( + "fmt" + "math/big" + "net" + + "github.com/open-policy-agent/opa/ast" + "github.com/open-policy-agent/opa/topdown/builtins" +) + +func getNetFromOperand(v ast.Value) (*net.IPNet, error) { + subnetStringA, err := builtins.StringOperand(v, 1) + if err != nil { + return nil, err + } + + _, cidrnet, err := net.ParseCIDR(string(subnetStringA)) + if err != nil { + return nil, err + } + + return cidrnet, nil +} + +func getLastIP(cidr *net.IPNet) (net.IP, error) { + prefixLen, bits := cidr.Mask.Size() + if prefixLen == 0 && bits == 0 { + // non-standard mask, see https://golang.org/pkg/net/#IPMask.Size + return nil, fmt.Errorf("CIDR mask is in non-standard format") + } + var lastIP []byte + if prefixLen == bits { + // Special case for single ip address ranges ex: 192.168.1.1/32 + // We can just use the starting IP as the last IP + lastIP = cidr.IP + } else { + // Use big.Int's so we can handle ipv6 addresses + firstIPInt := new(big.Int) + firstIPInt.SetBytes(cidr.IP) + hostLen := uint(bits) - uint(prefixLen) + lastIPInt := big.NewInt(1) + lastIPInt.Lsh(lastIPInt, hostLen) + lastIPInt.Sub(lastIPInt, big.NewInt(1)) + lastIPInt.Or(lastIPInt, firstIPInt) + + ipBytes := lastIPInt.Bytes() + lastIP = make([]byte, bits/8) + + // Pack our IP bytes into the end of the return array, + // since big.Int.Bytes() removes front zero padding. + for i := 1; i <= len(lastIPInt.Bytes()); i++ { + lastIP[len(lastIP)-i] = ipBytes[len(ipBytes)-i] + } + } + + return lastIP, nil +} + +func builtinNetCIDRIntersects(a, b ast.Value) (ast.Value, error) { + cidrnetA, err := getNetFromOperand(a) + if err != nil { + return nil, err + } + + cidrnetB, err := getNetFromOperand(b) + if err != nil { + return nil, err + } + + // If either net contains the others starting IP they are overlapping + cidrsOverlap := (cidrnetA.Contains(cidrnetB.IP) || cidrnetB.Contains(cidrnetA.IP)) + + return ast.Boolean(cidrsOverlap), nil +} + +func builtinNetCIDRContains(a, b ast.Value) (ast.Value, error) { + cidrnetA, err := getNetFromOperand(a) + if err != nil { + return nil, err + } + + // b could be either an IP addressor CIDR string, try to parse it as an IP first, fall back to CIDR + bStr, err := builtins.StringOperand(b, 1) + if err != nil { + return nil, err + } + + ip := net.ParseIP(string(bStr)) + if ip != nil { + return ast.Boolean(cidrnetA.Contains(ip)), nil + } + + // It wasn't an IP, try and parse it as a CIDR + cidrnetB, err := getNetFromOperand(b) + if err != nil { + return nil, fmt.Errorf("not a valid textual representation of an IP address or CIDR: %s", string(bStr)) + } + + // We can determine if cidr A contains cidr B iff A contains the starting address of B and the last address in B. + cidrContained := false + if cidrnetA.Contains(cidrnetB.IP) { + // Only spend time calculating the last IP if the starting IP is already verified to be in cidr A + lastIP, err := getLastIP(cidrnetB) + if err != nil { + return nil, err + } + cidrContained = cidrnetA.Contains(lastIP) + } + + return ast.Boolean(cidrContained), nil +} + +func builtinNetCIDRExpand(bctx BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error { + + s, err := builtins.StringOperand(operands[0].Value, 1) + if err != nil { + return err + } + + ip, ipNet, err := net.ParseCIDR(string(s)) + if err != nil { + return err + } + + result := ast.NewSet() + + for ip := ip.Mask(ipNet.Mask); ipNet.Contains(ip); incIP(ip) { + + if bctx.Cancel != nil && bctx.Cancel.Cancelled() { + return &Error{ + Code: CancelErr, + Message: "net.cidr_expand: timed out before generating all IP addresses", + } + } + + result.Add(ast.StringTerm(ip.String())) + } + + return iter(ast.NewTerm(result)) +} + +func incIP(ip net.IP) { + for j := len(ip) - 1; j >= 0; j-- { + ip[j]++ + if ip[j] > 0 { + break + } + } +} + +func init() { + RegisterFunctionalBuiltin2(ast.NetCIDROverlap.Name, builtinNetCIDRContains) + RegisterFunctionalBuiltin2(ast.NetCIDRIntersects.Name, builtinNetCIDRIntersects) + RegisterFunctionalBuiltin2(ast.NetCIDRContains.Name, builtinNetCIDRContains) + RegisterBuiltinFunc(ast.NetCIDRExpand.Name, builtinNetCIDRExpand) +} diff --git a/vendor/github.com/open-policy-agent/opa/topdown/comparison.go b/vendor/github.com/open-policy-agent/opa/topdown/comparison.go new file mode 100644 index 000000000..96be984ac --- /dev/null +++ b/vendor/github.com/open-policy-agent/opa/topdown/comparison.go @@ -0,0 +1,48 @@ +// Copyright 2016 The OPA Authors. All rights reserved. +// Use of this source code is governed by an Apache2 +// license that can be found in the LICENSE file. + +package topdown + +import "github.com/open-policy-agent/opa/ast" + +type compareFunc func(a, b ast.Value) bool + +func compareGreaterThan(a, b ast.Value) bool { + return ast.Compare(a, b) > 0 +} + +func compareGreaterThanEq(a, b ast.Value) bool { + return ast.Compare(a, b) >= 0 +} + +func compareLessThan(a, b ast.Value) bool { + return ast.Compare(a, b) < 0 +} + +func compareLessThanEq(a, b ast.Value) bool { + return ast.Compare(a, b) <= 0 +} + +func compareNotEq(a, b ast.Value) bool { + return ast.Compare(a, b) != 0 +} + +func compareEq(a, b ast.Value) bool { + return ast.Compare(a, b) == 0 +} + +func builtinCompare(cmp compareFunc) FunctionalBuiltin2 { + return func(a, b ast.Value) (ast.Value, error) { + return ast.Boolean(cmp(a, b)), nil + } +} + +func init() { + RegisterFunctionalBuiltin2(ast.GreaterThan.Name, builtinCompare(compareGreaterThan)) + RegisterFunctionalBuiltin2(ast.GreaterThanEq.Name, builtinCompare(compareGreaterThanEq)) + RegisterFunctionalBuiltin2(ast.LessThan.Name, builtinCompare(compareLessThan)) + RegisterFunctionalBuiltin2(ast.LessThanEq.Name, builtinCompare(compareLessThanEq)) + RegisterFunctionalBuiltin2(ast.NotEqual.Name, builtinCompare(compareNotEq)) + RegisterFunctionalBuiltin2(ast.Equal.Name, builtinCompare(compareEq)) +} diff --git a/vendor/github.com/open-policy-agent/opa/topdown/copypropagation/copypropagation.go b/vendor/github.com/open-policy-agent/opa/topdown/copypropagation/copypropagation.go new file mode 100644 index 000000000..820184b66 --- /dev/null +++ b/vendor/github.com/open-policy-agent/opa/topdown/copypropagation/copypropagation.go @@ -0,0 +1,484 @@ +// Copyright 2018 The OPA Authors. All rights reserved. +// Use of this source code is governed by an Apache2 +// license that can be found in the LICENSE file. + +package copypropagation + +import ( + "sort" + + "github.com/open-policy-agent/opa/ast" +) + +// CopyPropagator implements a simple copy propagation optimization to remove +// intermediate variables in partial evaluation results. +// +// For example, given the query: input.x > 1 where 'input' is unknown, the +// compiled query would become input.x = a; a > 1 which would remain in the +// partial evaluation result. The CopyPropagator will remove the variable +// assignment so that partial evaluation simply outputs input.x > 1. +// +// In many cases, copy propagation can remove all variables from the result of +// partial evaluation which simplifies evaluation for non-OPA consumers. +// +// In some cases, copy propagation cannot remove all variables. If the output of +// a built-in call is subsequently used as a ref head, the output variable must +// be kept. For example. sort(input, x); x[0] == 1. In this case, copy +// propagation cannot replace x[0] == 1 with sort(input, x)[0] == 1 as this is +// not legal. +type CopyPropagator struct { + livevars ast.VarSet // vars that must be preserved in the resulting query + sorted []ast.Var // sorted copy of vars to ensure deterministic result + ensureNonEmptyBody bool +} + +// New returns a new CopyPropagator that optimizes queries while preserving vars +// in the livevars set. +func New(livevars ast.VarSet) *CopyPropagator { + + sorted := make([]ast.Var, 0, len(livevars)) + for v := range livevars { + sorted = append(sorted, v) + } + + sort.Slice(sorted, func(i, j int) bool { + return sorted[i].Compare(sorted[j]) < 0 + }) + + return &CopyPropagator{livevars: livevars, sorted: sorted} +} + +// WithEnsureNonEmptyBody configures p to ensure that results are always non-empty. +func (p *CopyPropagator) WithEnsureNonEmptyBody(yes bool) *CopyPropagator { + p.ensureNonEmptyBody = yes + return p +} + +// Apply executes the copy propagation optimization and returns a new query. +func (p *CopyPropagator) Apply(query ast.Body) (result ast.Body) { + + uf, ok := makeDisjointSets(p.livevars, query) + if !ok { + return query + } + + // Compute set of vars that appear in the head of refs in the query. If a var + // is dereferenced, we cannot plug it with a constant value so the constant on + // the union-find root must be unset (e.g., [1][0] is not legal.) + headvars := ast.NewVarSet() + ast.WalkRefs(query, func(x ast.Ref) bool { + if v, ok := x[0].Value.(ast.Var); ok { + if root, ok := uf.Find(v); ok { + root.constant = nil + headvars.Add(root.key) + } else { + headvars.Add(v) + } + } + return false + }) + + bindings := map[ast.Var]*binding{} + + for _, expr := range query { + + pctx := &plugContext{ + bindings: bindings, + uf: uf, + negated: expr.Negated, + headvars: headvars, + } + + if expr, keep := p.plugBindings(pctx, expr); keep { + if p.updateBindings(pctx, expr) { + result.Append(expr) + } + } + } + + // Run post-processing step on the query to ensure that all live vars are bound + // in the result. The plugging that happens above substitutes all vars in the + // same set with the root. + // + // This step should run before the next step to prevent unnecessary bindings + // from being added to the result. For example: + // + // - Given the following result: + // - Given the following bindings: x/input.x and y/input + // - Given the following liveset: {x} + // + // If this step were to run AFTER the following step, the output would be: + // + // x = input.x; y = input + // + // Even though y = input is not required. + for _, v := range p.sorted { + if root, ok := uf.Find(v); ok { + if root.constant != nil { + result.Append(ast.Equality.Expr(ast.NewTerm(v), root.constant)) + } else if b, ok := bindings[root.key]; ok { + result.Append(ast.Equality.Expr(ast.NewTerm(v), ast.NewTerm(b.v))) + } else if root.key != v { + result.Append(ast.Equality.Expr(ast.NewTerm(v), ast.NewTerm(root.key))) + } + } + } + + // Run post-processing step on query to ensure that all killed exprs are + // accounted for. If an expr is killed but the binding is never used, the query + // must still include the expr. For example, given the query 'input.x = a' and + // an empty livevar set, the result must include the ref input.x otherwise the + // query could be satisfied without input.x being defined. When exprs are + // killed we initialize the binding counter to zero and then increment it each + // time the binding is substituted. if the binding was never substituted it + // means the binding value must be added back into the query. + for _, b := range sortbindings(bindings) { + if !b.containedIn(result) { + result.Append(ast.Equality.Expr(ast.NewTerm(b.k), ast.NewTerm(b.v))) + } + } + + if p.ensureNonEmptyBody && len(result) == 0 { + result = append(result, ast.NewExpr(ast.BooleanTerm(true))) + } + + return result +} + +// plugBindings applies the binding list and union-find to x. This process +// removes as many variables as possible. +func (p *CopyPropagator) plugBindings(pctx *plugContext, expr *ast.Expr) (*ast.Expr, bool) { + + // Kill single term expressions that are in the binding list. They will be + // re-added during post-processing if needed. + if term, ok := expr.Terms.(*ast.Term); ok { + if v, ok := term.Value.(ast.Var); ok { + if root, ok := pctx.uf.Find(v); ok { + if _, ok := pctx.bindings[root.key]; ok { + return nil, false + } + } + } + } + + xform := bindingPlugTransform{ + pctx: pctx, + } + + // Deep copy the expression as it may be mutated during the transform and + // the caller running copy propagation may have references to the + // expression. Note, the transform does not contain any error paths and + // should never return a non-expression value for the root so consider + // errors unreachable. + x, err := ast.Transform(xform, expr.Copy()) + + if expr, ok := x.(*ast.Expr); !ok || err != nil { + panic("unreachable") + } else { + return expr, true + } +} + +type bindingPlugTransform struct { + pctx *plugContext +} + +func (t bindingPlugTransform) Transform(x interface{}) (interface{}, error) { + switch x := x.(type) { + case ast.Var: + return t.plugBindingsVar(t.pctx, x), nil + case ast.Ref: + return t.plugBindingsRef(t.pctx, x), nil + default: + return x, nil + } +} + +func (t bindingPlugTransform) plugBindingsVar(pctx *plugContext, v ast.Var) (result ast.Value) { + + result = v + + // Apply union-find to remove redundant variables from input. + if root, ok := pctx.uf.Find(v); ok { + result = root.Value() + } + + // Apply binding list to substitute remaining vars. + if v, ok := result.(ast.Var); ok { + if b, ok := pctx.bindings[v]; ok { + if !pctx.negated || b.v.IsGround() { + result = b.v + } + } + } + + return result +} + +func (t bindingPlugTransform) plugBindingsRef(pctx *plugContext, v ast.Ref) ast.Ref { + + // Apply union-find to remove redundant variables from input. + if root, ok := pctx.uf.Find(v[0].Value.(ast.Var)); ok { + v[0].Value = root.Value() + } + + result := v + + // Refs require special handling. If the head of the ref was killed, then + // the rest of the ref must be concatenated with the new base. + // + // Invariant: ref heads can only be replaced by refs (not calls). + if b, ok := pctx.bindings[v[0].Value.(ast.Var)]; ok { + if !pctx.negated || b.v.IsGround() { + result = b.v.(ast.Ref).Concat(v[1:]) + } + } + + return result +} + +// updateBindings returns false if the expression can be killed. If the +// expression is killed, the binding list is updated to map a var to value. +func (p *CopyPropagator) updateBindings(pctx *plugContext, expr *ast.Expr) bool { + if pctx.negated || len(expr.With) > 0 { + return true + } + if expr.IsEquality() { + a, b := expr.Operand(0), expr.Operand(1) + if a.Equal(b) { + return false + } + k, v, keep := p.updateBindingsEq(a, b) + if !keep { + if v != nil { + pctx.bindings[k] = newbinding(k, v) + } + return false + } + } else if expr.IsCall() { + terms := expr.Terms.([]*ast.Term) + output := terms[len(terms)-1] + if k, ok := output.Value.(ast.Var); ok && !p.livevars.Contains(k) && !pctx.headvars.Contains(k) { + pctx.bindings[k] = newbinding(k, ast.CallTerm(terms[:len(terms)-1]...).Value) + return false + } + } + return !isNoop(expr) +} + +func (p *CopyPropagator) updateBindingsEq(a, b *ast.Term) (ast.Var, ast.Value, bool) { + k, v, keep := p.updateBindingsEqAsymmetric(a, b) + if !keep { + return k, v, keep + } + return p.updateBindingsEqAsymmetric(b, a) +} + +func (p *CopyPropagator) updateBindingsEqAsymmetric(a, b *ast.Term) (ast.Var, ast.Value, bool) { + k, ok := a.Value.(ast.Var) + if !ok || p.livevars.Contains(k) { + return "", nil, true + } + + switch b.Value.(type) { + case ast.Ref, ast.Call: + return k, b.Value, false + } + + return "", nil, true +} + +type plugContext struct { + bindings map[ast.Var]*binding + uf *unionFind + headvars ast.VarSet + negated bool +} + +type binding struct { + k ast.Var + v ast.Value +} + +func newbinding(k ast.Var, v ast.Value) *binding { + return &binding{k: k, v: v} +} + +func (b *binding) containedIn(query ast.Body) bool { + var stop bool + switch v := b.v.(type) { + case ast.Ref: + ast.WalkRefs(query, func(other ast.Ref) bool { + if stop || other.HasPrefix(v) { + stop = true + return stop + } + return false + }) + default: + ast.WalkTerms(query, func(other *ast.Term) bool { + if stop || other.Value.Compare(v) == 0 { + stop = true + return stop + } + return false + }) + } + return stop +} + +func sortbindings(bindings map[ast.Var]*binding) []*binding { + sorted := make([]*binding, 0, len(bindings)) + for _, b := range bindings { + sorted = append(sorted, b) + } + sort.Slice(sorted, func(i, j int) bool { + return sorted[i].k.Compare(sorted[j].k) < 0 + }) + return sorted +} + +type unionFind struct { + roots map[ast.Var]*unionFindRoot + parents map[ast.Var]ast.Var + rank rankFunc +} + +// makeDisjointSets builds the union-find structure for the query. The structure +// is built by processing all of the equality exprs in the query. Sets represent +// vars that must be equal to each other. In addition to vars, each set can have +// at most one constant. If the query contains expressions that cannot be +// satisfied (e.g., because a set has multiple constants) this function returns +// false. +func makeDisjointSets(livevars ast.VarSet, query ast.Body) (*unionFind, bool) { + uf := newUnionFind(func(r1, r2 *unionFindRoot) (*unionFindRoot, *unionFindRoot) { + if livevars.Contains(r1.key) { + return r1, r2 + } + return r2, r1 + }) + for _, expr := range query { + if expr.IsEquality() && !expr.Negated && len(expr.With) == 0 { + a, b := expr.Operand(0), expr.Operand(1) + varA, ok1 := a.Value.(ast.Var) + varB, ok2 := b.Value.(ast.Var) + if ok1 && ok2 { + if _, ok := uf.Merge(varA, varB); !ok { + return nil, false + } + } else if ok1 && ast.IsConstant(b.Value) { + root := uf.MakeSet(varA) + if root.constant != nil && !root.constant.Equal(b) { + return nil, false + } + root.constant = b + } else if ok2 && ast.IsConstant(a.Value) { + root := uf.MakeSet(varB) + if root.constant != nil && !root.constant.Equal(a) { + return nil, false + } + root.constant = a + } + } + } + + return uf, true +} + +type rankFunc func(*unionFindRoot, *unionFindRoot) (*unionFindRoot, *unionFindRoot) + +func newUnionFind(rank rankFunc) *unionFind { + return &unionFind{ + roots: map[ast.Var]*unionFindRoot{}, + parents: map[ast.Var]ast.Var{}, + rank: rank, + } +} + +func (uf *unionFind) MakeSet(v ast.Var) *unionFindRoot { + + root, ok := uf.Find(v) + if ok { + return root + } + + root = newUnionFindRoot(v) + uf.parents[v] = v + uf.roots[v] = root + return uf.roots[v] +} + +func (uf *unionFind) Find(v ast.Var) (*unionFindRoot, bool) { + + parent, ok := uf.parents[v] + if !ok { + return nil, false + } + + if parent == v { + return uf.roots[v], true + } + + return uf.Find(parent) +} + +func (uf *unionFind) Merge(a, b ast.Var) (*unionFindRoot, bool) { + + r1 := uf.MakeSet(a) + r2 := uf.MakeSet(b) + + if r1 != r2 { + + r1, r2 = uf.rank(r1, r2) + + uf.parents[r2.key] = r1.key + delete(uf.roots, r2.key) + + // Sets can have at most one constant value associated with them. When + // unioning, we must preserve this invariant. If a set has two constants, + // there will be no way to prove the query. + if r1.constant != nil && r2.constant != nil && !r1.constant.Equal(r2.constant) { + return nil, false + } else if r1.constant == nil { + r1.constant = r2.constant + } + } + + return r1, true +} + +type unionFindRoot struct { + key ast.Var + constant *ast.Term +} + +func newUnionFindRoot(key ast.Var) *unionFindRoot { + return &unionFindRoot{ + key: key, + } +} + +func (r *unionFindRoot) Value() ast.Value { + if r.constant != nil { + return r.constant.Value + } + return r.key +} + +func isNoop(expr *ast.Expr) bool { + + if !expr.IsCall() { + term := expr.Terms.(*ast.Term) + if !ast.IsConstant(term.Value) { + return false + } + return !ast.Boolean(false).Equal(term.Value) + } + + // A==A can be ignored + if expr.Operator().Equal(ast.Equal.Ref()) { + return expr.Operand(0).Equal(expr.Operand(1)) + } + + return false +} diff --git a/vendor/github.com/open-policy-agent/opa/topdown/crypto.go b/vendor/github.com/open-policy-agent/opa/topdown/crypto.go new file mode 100644 index 000000000..c0b10c603 --- /dev/null +++ b/vendor/github.com/open-policy-agent/opa/topdown/crypto.go @@ -0,0 +1,128 @@ +// Copyright 2018 The OPA Authors. All rights reserved. +// Use of this source code is governed by an Apache2 +// license that can be found in the LICENSE file. + +package topdown + +import ( + "crypto/md5" + "crypto/sha1" + "crypto/sha256" + "crypto/x509" + "encoding/json" + "fmt" + "io/ioutil" + + "github.com/open-policy-agent/opa/ast" + "github.com/open-policy-agent/opa/topdown/builtins" + "github.com/open-policy-agent/opa/util" +) + +func builtinCryptoX509ParseCertificates(a ast.Value) (ast.Value, error) { + + str, err := builtinBase64Decode(a) + if err != nil { + return nil, err + } + + certs, err := x509.ParseCertificates([]byte(str.(ast.String))) + if err != nil { + return nil, err + } + + bs, err := json.Marshal(certs) + if err != nil { + return nil, err + } + + var x interface{} + + if err := util.UnmarshalJSON(bs, &x); err != nil { + return nil, err + } + + return ast.InterfaceToValue(x) +} + +func hashHelper(a ast.Value, h func(ast.String) string) (ast.Value, error) { + s, err := builtins.StringOperand(a, 1) + if err != nil { + return nil, err + } + return ast.String(h(s)), nil +} + +func builtinCryptoMd5(a ast.Value) (ast.Value, error) { + return hashHelper(a, func(s ast.String) string { return fmt.Sprintf("%x", md5.Sum([]byte(s))) }) +} + +func builtinCryptoSha1(a ast.Value) (ast.Value, error) { + return hashHelper(a, func(s ast.String) string { return fmt.Sprintf("%x", sha1.Sum([]byte(s))) }) +} + +func builtinCryptoSha256(a ast.Value) (ast.Value, error) { + return hashHelper(a, func(s ast.String) string { return fmt.Sprintf("%x", sha256.Sum256([]byte(s))) }) +} + +func init() { + RegisterFunctionalBuiltin1(ast.CryptoX509ParseCertificates.Name, builtinCryptoX509ParseCertificates) + RegisterFunctionalBuiltin1(ast.CryptoMd5.Name, builtinCryptoMd5) + RegisterFunctionalBuiltin1(ast.CryptoSha1.Name, builtinCryptoSha1) + RegisterFunctionalBuiltin1(ast.CryptoSha256.Name, builtinCryptoSha256) +} + +// createRootCAs creates a new Cert Pool from scratch or adds to a copy of System Certs +func createRootCAs(tlsCACertFile string, tlsCACertEnvVar []byte, tlsUseSystemCerts bool) (*x509.CertPool, error) { + + var newRootCAs *x509.CertPool + + if tlsUseSystemCerts { + systemCertPool, err := x509.SystemCertPool() + if err != nil { + return nil, err + } + newRootCAs = systemCertPool + } else { + newRootCAs = x509.NewCertPool() + } + + if len(tlsCACertFile) > 0 { + // Append our cert to the system pool + caCert, err := readCertFromFile(tlsCACertFile) + if err != nil { + return nil, err + } + if ok := newRootCAs.AppendCertsFromPEM(caCert); !ok { + return nil, fmt.Errorf("could not append CA cert from %q", tlsCACertFile) + } + } + + if len(tlsCACertEnvVar) > 0 { + // Append our cert to the system pool + if ok := newRootCAs.AppendCertsFromPEM(tlsCACertEnvVar); !ok { + return nil, fmt.Errorf("error appending cert from env var %q into system certs", tlsCACertEnvVar) + } + } + + return newRootCAs, nil +} + +// ReadCertFromFile reads a cert from file +func readCertFromFile(localCertFile string) ([]byte, error) { + // Read in the cert file + certPEM, err := ioutil.ReadFile(localCertFile) + if err != nil { + return nil, err + } + return certPEM, nil +} + +// ReadKeyFromFile reads a key from file +func readKeyFromFile(localKeyFile string) ([]byte, error) { + // Read in the cert file + key, err := ioutil.ReadFile(localKeyFile) + if err != nil { + return nil, err + } + return key, nil +} diff --git a/vendor/github.com/open-policy-agent/opa/topdown/doc.go b/vendor/github.com/open-policy-agent/opa/topdown/doc.go new file mode 100644 index 000000000..9aa7aa45c --- /dev/null +++ b/vendor/github.com/open-policy-agent/opa/topdown/doc.go @@ -0,0 +1,10 @@ +// Copyright 2016 The OPA Authors. All rights reserved. +// Use of this source code is governed by an Apache2 +// license that can be found in the LICENSE file. + +// Package topdown provides low-level query evaluation support. +// +// The topdown implementation is a modified version of the standard top-down +// evaluation algorithm used in Datalog. References and comprehensions are +// evaluated eagerly while all other terms are evaluated lazily. +package topdown diff --git a/vendor/github.com/open-policy-agent/opa/topdown/encoding.go b/vendor/github.com/open-policy-agent/opa/topdown/encoding.go new file mode 100644 index 000000000..f2074bb46 --- /dev/null +++ b/vendor/github.com/open-policy-agent/opa/topdown/encoding.go @@ -0,0 +1,217 @@ +// Copyright 2017 The OPA Authors. All rights reserved. +// Use of this source code is governed by an Apache2 +// license that can be found in the LICENSE file. + +package topdown + +import ( + "bytes" + "encoding/base64" + "encoding/json" + "fmt" + "net/url" + "strings" + + ghodss "github.com/ghodss/yaml" + + "github.com/open-policy-agent/opa/ast" + "github.com/open-policy-agent/opa/topdown/builtins" + "github.com/open-policy-agent/opa/util" +) + +func builtinJSONMarshal(a ast.Value) (ast.Value, error) { + + asJSON, err := ast.JSON(a) + if err != nil { + return nil, err + } + + bs, err := json.Marshal(asJSON) + if err != nil { + return nil, err + } + + return ast.String(string(bs)), nil +} + +func builtinJSONUnmarshal(a ast.Value) (ast.Value, error) { + + str, err := builtins.StringOperand(a, 1) + if err != nil { + return nil, err + } + + var x interface{} + + if err := util.UnmarshalJSON([]byte(str), &x); err != nil { + return nil, err + } + + return ast.InterfaceToValue(x) +} + +func builtinBase64Encode(a ast.Value) (ast.Value, error) { + str, err := builtins.StringOperand(a, 1) + if err != nil { + return nil, err + } + + return ast.String(base64.StdEncoding.EncodeToString([]byte(str))), nil +} + +func builtinBase64Decode(a ast.Value) (ast.Value, error) { + str, err := builtins.StringOperand(a, 1) + if err != nil { + return nil, err + } + + result, err := base64.StdEncoding.DecodeString(string(str)) + return ast.String(result), err +} + +func builtinBase64UrlEncode(a ast.Value) (ast.Value, error) { + str, err := builtins.StringOperand(a, 1) + if err != nil { + return nil, err + } + + return ast.String(base64.URLEncoding.EncodeToString([]byte(str))), nil +} + +func builtinBase64UrlDecode(a ast.Value) (ast.Value, error) { + str, err := builtins.StringOperand(a, 1) + if err != nil { + return nil, err + } + s := string(str) + + // Some base64url encoders omit the padding at the end, so this case + // corrects such representations using the method given in RFC 7515 + // Appendix C: https://tools.ietf.org/html/rfc7515#appendix-C + if !strings.HasSuffix(s, "=") { + switch len(s) % 4 { + case 0: + case 2: + s += "==" + case 3: + s += "=" + default: + return nil, fmt.Errorf("illegal base64url string: %s", s) + } + } + result, err := base64.URLEncoding.DecodeString(s) + return ast.String(result), err +} + +func builtinURLQueryEncode(a ast.Value) (ast.Value, error) { + str, err := builtins.StringOperand(a, 1) + if err != nil { + return nil, err + } + return ast.String(url.QueryEscape(string(str))), nil +} + +func builtinURLQueryDecode(a ast.Value) (ast.Value, error) { + str, err := builtins.StringOperand(a, 1) + if err != nil { + return nil, err + } + s, err := url.QueryUnescape(string(str)) + if err != nil { + return nil, err + } + return ast.String(s), nil +} + +var encodeObjectErr = builtins.NewOperandErr(1, "values must be string, array[string], or set[string]") + +func builtinURLQueryEncodeObject(a ast.Value) (ast.Value, error) { + asJSON, err := ast.JSON(a) + if err != nil { + return nil, err + } + + inputs, ok := asJSON.(map[string]interface{}) + if !ok { + return nil, builtins.NewOperandTypeErr(1, a, "object") + } + + query := url.Values{} + + for k, v := range inputs { + switch vv := v.(type) { + case string: + query.Set(k, vv) + case []interface{}: + for _, val := range vv { + strVal, ok := val.(string) + if !ok { + return nil, encodeObjectErr + } + query.Add(k, strVal) + } + default: + return nil, encodeObjectErr + } + } + + return ast.String(query.Encode()), nil +} + +func builtinYAMLMarshal(a ast.Value) (ast.Value, error) { + + asJSON, err := ast.JSON(a) + if err != nil { + return nil, err + } + + var buf bytes.Buffer + encoder := json.NewEncoder(&buf) + if err := encoder.Encode(asJSON); err != nil { + return nil, err + } + + bs, err := ghodss.JSONToYAML(buf.Bytes()) + if err != nil { + return nil, err + } + + return ast.String(string(bs)), nil +} + +func builtinYAMLUnmarshal(a ast.Value) (ast.Value, error) { + + str, err := builtins.StringOperand(a, 1) + if err != nil { + return nil, err + } + + bs, err := ghodss.YAMLToJSON([]byte(str)) + if err != nil { + return nil, err + } + + buf := bytes.NewBuffer(bs) + decoder := util.NewJSONDecoder(buf) + var val interface{} + err = decoder.Decode(&val) + if err != nil { + return nil, err + } + + return ast.InterfaceToValue(val) +} + +func init() { + RegisterFunctionalBuiltin1(ast.JSONMarshal.Name, builtinJSONMarshal) + RegisterFunctionalBuiltin1(ast.JSONUnmarshal.Name, builtinJSONUnmarshal) + RegisterFunctionalBuiltin1(ast.Base64Encode.Name, builtinBase64Encode) + RegisterFunctionalBuiltin1(ast.Base64Decode.Name, builtinBase64Decode) + RegisterFunctionalBuiltin1(ast.Base64UrlEncode.Name, builtinBase64UrlEncode) + RegisterFunctionalBuiltin1(ast.Base64UrlDecode.Name, builtinBase64UrlDecode) + RegisterFunctionalBuiltin1(ast.URLQueryDecode.Name, builtinURLQueryDecode) + RegisterFunctionalBuiltin1(ast.URLQueryEncode.Name, builtinURLQueryEncode) + RegisterFunctionalBuiltin1(ast.URLQueryEncodeObject.Name, builtinURLQueryEncodeObject) + RegisterFunctionalBuiltin1(ast.YAMLMarshal.Name, builtinYAMLMarshal) + RegisterFunctionalBuiltin1(ast.YAMLUnmarshal.Name, builtinYAMLUnmarshal) +} diff --git a/vendor/github.com/open-policy-agent/opa/topdown/errors.go b/vendor/github.com/open-policy-agent/opa/topdown/errors.go new file mode 100644 index 000000000..d0457f1a8 --- /dev/null +++ b/vendor/github.com/open-policy-agent/opa/topdown/errors.go @@ -0,0 +1,119 @@ +// Copyright 2017 The OPA Authors. All rights reserved. +// Use of this source code is governed by an Apache2 +// license that can be found in the LICENSE file. + +package topdown + +import ( + "fmt" + + "github.com/open-policy-agent/opa/ast" +) + +// Error is the error type returned by the Eval and Query functions when +// an evaluation error occurs. +type Error struct { + Code string `json:"code"` + Message string `json:"message"` + Location *ast.Location `json:"location,omitempty"` +} + +const ( + + // InternalErr represents an unknown evaluation error. + InternalErr string = "eval_internal_error" + + // CancelErr indicates the evaluation process was cancelled. + CancelErr string = "eval_cancel_error" + + // ConflictErr indicates a conflict was encountered during evaluation. For + // instance, a conflict occurs if a rule produces multiple, differing values + // for the same key in an object. Conflict errors indicate the policy does + // not account for the data loaded into the policy engine. + ConflictErr string = "eval_conflict_error" + + // TypeErr indicates evaluation stopped because an expression was applied to + // a value of an inappropriate type. + TypeErr string = "eval_type_error" + + // BuiltinErr indicates a built-in function received a semantically invalid + // input or encountered some kind of runtime error, e.g., connection + // timeout, connection refused, etc. + BuiltinErr string = "eval_builtin_error" + + // WithMergeErr indicates that the real and replacement data could not be merged. + WithMergeErr string = "eval_with_merge_error" +) + +// IsError returns true if the err is an Error. +func IsError(err error) bool { + _, ok := err.(*Error) + return ok +} + +// IsCancel returns true if err was caused by cancellation. +func IsCancel(err error) bool { + if e, ok := err.(*Error); ok { + return e.Code == CancelErr + } + return false +} + +func (e *Error) Error() string { + + msg := fmt.Sprintf("%v: %v", e.Code, e.Message) + + if e.Location != nil { + msg = e.Location.String() + ": " + msg + } + + return msg +} + +func functionConflictErr(loc *ast.Location) error { + return &Error{ + Code: ConflictErr, + Location: loc, + Message: "functions must not produce multiple outputs for same inputs", + } +} + +func completeDocConflictErr(loc *ast.Location) error { + return &Error{ + Code: ConflictErr, + Location: loc, + Message: "complete rules must not produce multiple outputs", + } +} + +func objectDocKeyConflictErr(loc *ast.Location) error { + return &Error{ + Code: ConflictErr, + Location: loc, + Message: "object keys must be unique", + } +} + +func documentConflictErr(loc *ast.Location) error { + return &Error{ + Code: ConflictErr, + Location: loc, + Message: "base and virtual document keys must be disjoint", + } +} + +func unsupportedBuiltinErr(loc *ast.Location) error { + return &Error{ + Code: InternalErr, + Location: loc, + Message: "unsupported built-in", + } +} + +func mergeConflictErr(loc *ast.Location) error { + return &Error{ + Code: WithMergeErr, + Location: loc, + Message: "real and replacement data could not be merged", + } +} diff --git a/vendor/github.com/open-policy-agent/opa/topdown/eval.go b/vendor/github.com/open-policy-agent/opa/topdown/eval.go new file mode 100644 index 000000000..1e32f8d45 --- /dev/null +++ b/vendor/github.com/open-policy-agent/opa/topdown/eval.go @@ -0,0 +1,2382 @@ +package topdown + +import ( + "context" + "fmt" + "sort" + "strconv" + "strings" + + "github.com/open-policy-agent/opa/ast" + "github.com/open-policy-agent/opa/storage" + "github.com/open-policy-agent/opa/topdown/builtins" + "github.com/open-policy-agent/opa/topdown/copypropagation" +) + +type evalIterator func(*eval) error + +type unifyIterator func() error + +type queryIDFactory struct { + curr uint64 +} + +// Note: The first call to Next() returns 0. +func (f *queryIDFactory) Next() uint64 { + curr := f.curr + f.curr++ + return curr +} + +type eval struct { + ctx context.Context + queryID uint64 + queryIDFact *queryIDFactory + parent *eval + caller *eval + cancel Cancel + query ast.Body + queryCompiler ast.QueryCompiler + index int + indexing bool + bindings *bindings + store storage.Store + baseCache *baseCache + txn storage.Transaction + compiler *ast.Compiler + input *ast.Term + data *ast.Term + targetStack *refStack + tracers []Tracer + instr *Instrumentation + builtins map[string]*Builtin + builtinCache builtins.Cache + virtualCache *virtualCache + saveSet *saveSet + saveStack *saveStack + saveSupport *saveSupport + saveNamespace *ast.Term + disableInlining [][]ast.Ref + genvarprefix string + runtime *ast.Term +} + +func (e *eval) Run(iter evalIterator) error { + e.traceEnter(e.query) + return e.eval(func(e *eval) error { + e.traceExit(e.query) + err := iter(e) + e.traceRedo(e.query) + return err + }) +} + +func (e *eval) builtinFunc(name string) (*ast.Builtin, BuiltinFunc, bool) { + decl, ok := ast.BuiltinMap[name] + if !ok { + bi, ok := e.builtins[name] + if ok { + return bi.Decl, bi.Func, true + } + } else { + f, ok := builtinFunctions[name] + if ok { + return decl, f, true + } + } + return nil, nil, false +} + +func (e *eval) closure(query ast.Body) *eval { + cpy := *e + cpy.index = 0 + cpy.query = query + cpy.queryID = cpy.queryIDFact.Next() + cpy.parent = e + return &cpy +} + +func (e *eval) child(query ast.Body) *eval { + cpy := *e + cpy.index = 0 + cpy.query = query + cpy.queryID = cpy.queryIDFact.Next() + cpy.bindings = newBindings(cpy.queryID, e.instr) + cpy.parent = e + return &cpy +} + +func (e *eval) next(iter evalIterator) error { + e.index++ + err := e.evalExpr(iter) + e.index-- + return err +} + +func (e *eval) partial() bool { + return e.saveSet != nil +} + +func (e *eval) unknown(x interface{}, b *bindings) bool { + if !e.partial() { + return false + } + + // If the caller provided an ast.Value directly (e.g., an ast.Ref) wrap + // it as an ast.Term because the saveSet Contains() function expects + // ast.Term. + if v, ok := x.(ast.Value); ok { + x = ast.NewTerm(v) + } + + return saveRequired(e.compiler, e.saveSet, b, x, false) +} + +func (e *eval) traceEnter(x ast.Node) { + e.traceEvent(EnterOp, x, "") +} + +func (e *eval) traceExit(x ast.Node) { + e.traceEvent(ExitOp, x, "") +} + +func (e *eval) traceEval(x ast.Node) { + e.traceEvent(EvalOp, x, "") +} + +func (e *eval) traceFail(x ast.Node) { + e.traceEvent(FailOp, x, "") +} + +func (e *eval) traceRedo(x ast.Node) { + e.traceEvent(RedoOp, x, "") +} + +func (e *eval) traceSave(x ast.Node) { + e.traceEvent(SaveOp, x, "") +} + +func (e *eval) traceIndex(x ast.Node, msg string) { + e.traceEvent(IndexOp, x, msg) +} + +func (e *eval) traceEvent(op Op, x ast.Node, msg string) { + + if !traceIsEnabled(e.tracers) { + return + } + + locals := ast.NewValueMap() + localMeta := map[ast.Var]VarMetadata{} + + e.bindings.Iter(nil, func(k, v *ast.Term) error { + original := k.Value.(ast.Var) + rewritten, _ := e.rewrittenVar(original) + localMeta[original] = VarMetadata{ + Name: rewritten, + Location: k.Loc(), + } + + // For backwards compatibility save a copy of the values too.. + locals.Put(k.Value, v.Value) + return nil + }) + + ast.WalkTerms(x, func(term *ast.Term) bool { + if v, ok := term.Value.(ast.Var); ok { + if _, ok := localMeta[v]; !ok { + if rewritten, ok := e.rewrittenVar(v); ok { + localMeta[v] = VarMetadata{ + Name: rewritten, + Location: term.Loc(), + } + } + } + } + return false + }) + + var parentID uint64 + if e.parent != nil { + parentID = e.parent.queryID + } + + evt := &Event{ + QueryID: e.queryID, + ParentID: parentID, + Op: op, + Node: x, + Location: x.Loc(), + Locals: locals, + LocalMetadata: localMeta, + Message: msg, + } + + for i := range e.tracers { + if e.tracers[i].Enabled() { + e.tracers[i].Trace(evt) + } + } +} + +func (e *eval) eval(iter evalIterator) error { + return e.evalExpr(iter) +} + +func (e *eval) evalExpr(iter evalIterator) error { + + if e.cancel != nil && e.cancel.Cancelled() { + return &Error{ + Code: CancelErr, + Message: "caller cancelled query execution", + } + } + + if e.index >= len(e.query) { + return iter(e) + } + + expr := e.query[e.index] + + e.traceEval(expr) + + if len(expr.With) > 0 { + return e.evalWith(iter) + } + + return e.evalStep(func(e *eval) error { + return e.next(iter) + }) +} + +func (e *eval) evalStep(iter evalIterator) error { + + expr := e.query[e.index] + + if expr.Negated { + return e.evalNot(iter) + } + + var defined bool + var err error + + switch terms := expr.Terms.(type) { + case []*ast.Term: + if expr.IsEquality() { + err = e.unify(terms[1], terms[2], func() error { + defined = true + err := iter(e) + e.traceRedo(expr) + return err + }) + } else { + err = e.evalCall(terms, func() error { + defined = true + err := iter(e) + e.traceRedo(expr) + return err + }) + } + case *ast.Term: + rterm := e.generateVar(fmt.Sprintf("term_%d_%d", e.queryID, e.index)) + err = e.unify(terms, rterm, func() error { + if e.saveSet.Contains(rterm, e.bindings) { + return e.saveExpr(ast.NewExpr(rterm), e.bindings, func() error { + return iter(e) + }) + } + if !e.bindings.Plug(rterm).Equal(ast.BooleanTerm(false)) { + defined = true + err := iter(e) + e.traceRedo(expr) + return err + } + return nil + }) + } + + if err != nil { + return err + } + + if !defined { + e.traceFail(expr) + } + + return nil +} + +func (e *eval) evalNot(iter evalIterator) error { + + expr := e.query[e.index] + + if e.unknown(expr, e.bindings) { + return e.evalNotPartial(iter) + } + + negation := ast.NewBody(expr.Complement().NoWith()) + child := e.closure(negation) + + var defined bool + child.traceEnter(negation) + + err := child.eval(func(*eval) error { + child.traceExit(negation) + defined = true + child.traceRedo(negation) + return nil + }) + + if err != nil { + return err + } + + if !defined { + return iter(e) + } + + e.traceFail(expr) + return nil +} + +func (e *eval) evalWith(iter evalIterator) error { + + expr := e.query[e.index] + var disable []ast.Ref + + if e.partial() { + + // If the value is unknown the with statement cannot be evaluated and so + // the entire expression should be saved to be safe. In the future this + // could be relaxed in certain cases (e.g., if the with statement would + // have no affect.) + for _, with := range expr.With { + if e.saveSet.ContainsRecursive(with.Value, e.bindings) { + return e.saveExpr(expr, e.bindings, func() error { + return e.next(iter) + }) + } + } + + // Disable inlining on all references in the expression so the result of + // partial evaluation has the same semamntics w/ the with statements + // preserved. + ast.WalkRefs(expr, func(x ast.Ref) bool { + disable = append(disable, x.GroundPrefix()) + return false + }) + } + + pairsInput := [][2]*ast.Term{} + pairsData := [][2]*ast.Term{} + targets := []ast.Ref{} + + for i := range expr.With { + plugged := e.bindings.Plug(expr.With[i].Value) + if isInputRef(expr.With[i].Target) { + pairsInput = append(pairsInput, [...]*ast.Term{expr.With[i].Target, plugged}) + } else if isDataRef(expr.With[i].Target) { + pairsData = append(pairsData, [...]*ast.Term{expr.With[i].Target, plugged}) + } + targets = append(targets, expr.With[i].Target.Value.(ast.Ref)) + } + + input, err := mergeTermWithValues(e.input, pairsInput) + + if err != nil { + return &Error{ + Code: ConflictErr, + Location: expr.Location, + Message: err.Error(), + } + } + + data, err := mergeTermWithValues(e.data, pairsData) + if err != nil { + return &Error{ + Code: ConflictErr, + Location: expr.Location, + Message: err.Error(), + } + } + + oldInput, oldData := e.evalWithPush(input, data, targets, disable) + + err = e.evalStep(func(e *eval) error { + e.evalWithPop(oldInput, oldData) + err := e.next(iter) + oldInput, oldData = e.evalWithPush(input, data, targets, disable) + return err + }) + + e.evalWithPop(oldInput, oldData) + + return err +} + +func (e *eval) evalWithPush(input *ast.Term, data *ast.Term, targets []ast.Ref, disable []ast.Ref) (*ast.Term, *ast.Term) { + + var oldInput *ast.Term + + if input != nil { + oldInput = e.input + e.input = input + } + + var oldData *ast.Term + + if data != nil { + oldData = e.data + e.data = data + } + + e.virtualCache.Push() + e.targetStack.Push(targets) + e.disableInlining = append(e.disableInlining, disable) + + return oldInput, oldData +} + +func (e *eval) evalWithPop(input *ast.Term, data *ast.Term) { + e.disableInlining = e.disableInlining[:len(e.disableInlining)-1] + e.targetStack.Pop() + e.virtualCache.Pop() + e.data = data + e.input = input +} + +func (e *eval) evalNotPartial(iter evalIterator) error { + + // Prepare query normally. + expr := e.query[e.index] + negation := expr.Complement().NoWith() + child := e.closure(ast.NewBody(negation)) + + // Unknowns is the set of variables that are marked as unknown. The variables + // are namespaced with the query ID that they originate in. This ensures that + // variables across two or more queries are identified uniquely. + // + // NOTE(tsandall): this is greedy in the sense that we only need variable + // dependencies of the negation. + unknowns := e.saveSet.Vars(e.caller.bindings) + + // Run partial evaluation, plugging the result and applying copy propagation to + // each result. Since the result may require support, push a new query onto the + // save stack to avoid mutating the current save query. + p := copypropagation.New(unknowns).WithEnsureNonEmptyBody(true) + var savedQueries []ast.Body + e.saveStack.PushQuery(nil) + + child.eval(func(*eval) error { + query := e.saveStack.Peek() + plugged := query.Plug(e.caller.bindings) + result := applyCopyPropagation(p, e.instr, plugged) + savedQueries = append(savedQueries, result) + return nil + }) + + e.saveStack.PopQuery() + + // If partial evaluation produced no results, the expression is always undefined + // so it does not have to be saved. + if len(savedQueries) == 0 { + return iter(e) + } + + // Check if the partial evaluation result can be inlined in this query. If not, + // generate support rules for the result. Depending on the size of the partial + // evaluation result and the contents, it may or may not be inlinable. We treat + // the unknowns as safe because vars in the save set will either be known to + // the caller or made safe by an expression on the save stack. + if !canInlineNegation(unknowns, savedQueries) { + return e.evalNotPartialSupport(expr, unknowns, savedQueries, iter) + } + + // If we can inline the result, we have to generate the cross product of the + // queries. For example: + // + // (A && B) || (C && D) + // + // Becomes: + // + // (!A && !C) || (!A && !D) || (!B && !C) || (!B && !D) + return complementedCartesianProduct(savedQueries, 0, nil, func(q ast.Body) error { + return e.saveInlinedNegatedExprs(q, func() error { + return iter(e) + }) + }) +} + +func (e *eval) evalNotPartialSupport(expr *ast.Expr, unknowns ast.VarSet, queries []ast.Body, iter evalIterator) error { + + // Prepare support rule head. + supportName := fmt.Sprintf("__not%d_%d__", e.queryID, e.index) + term := ast.RefTerm(ast.DefaultRootDocument, e.saveNamespace, ast.StringTerm(supportName)) + path := term.Value.(ast.Ref) + head := ast.NewHead(ast.Var(supportName), nil, ast.BooleanTerm(true)) + + bodyVars := ast.NewVarSet() + + for _, q := range queries { + bodyVars.Update(q.Vars(ast.VarVisitorParams{})) + } + + unknowns = unknowns.Intersect(bodyVars) + + // Make rule args. Sort them to ensure order is deterministic. + args := make([]*ast.Term, 0, len(unknowns)) + + for v := range unknowns { + args = append(args, ast.NewTerm(v)) + } + + sort.Slice(args, func(i, j int) bool { + return args[i].Value.Compare(args[j].Value) < 0 + }) + + if len(args) > 0 { + head.Args = ast.Args(args) + } + + // Save support rules. + for _, query := range queries { + e.saveSupport.Insert(path, &ast.Rule{ + Head: head, + Body: query, + }) + } + + // Save expression that refers to support rule set. + expr = expr.Copy() + if len(args) > 0 { + terms := make([]*ast.Term, len(args)+1) + terms[0] = term + for i := 0; i < len(args); i++ { + terms[i+1] = args[i] + } + expr.Terms = terms + } else { + expr.Terms = term + } + + return e.saveInlinedNegatedExprs([]*ast.Expr{expr}, func() error { + return e.next(iter) + }) +} + +func (e *eval) evalCall(terms []*ast.Term, iter unifyIterator) error { + + ref := terms[0].Value.(ast.Ref) + + if ref[0].Equal(ast.DefaultRootDocument) { + eval := evalFunc{ + e: e, + ref: ref, + terms: terms, + } + return eval.eval(iter) + } + + bi, f, ok := e.builtinFunc(ref.String()) + if !ok { + return unsupportedBuiltinErr(e.query[e.index].Location) + } + + if e.unknown(e.query[e.index], e.bindings) { + return e.saveCall(len(bi.Decl.Args()), terms, iter) + } + + var parentID uint64 + if e.parent != nil { + parentID = e.parent.queryID + } + + bctx := BuiltinContext{ + Context: e.ctx, + Cancel: e.cancel, + Runtime: e.runtime, + Cache: e.builtinCache, + Location: e.query[e.index].Location, + Tracers: e.tracers, + QueryID: e.queryID, + ParentID: parentID, + } + + eval := evalBuiltin{ + e: e, + bi: bi, + bctx: bctx, + f: f, + terms: terms[1:], + } + return eval.eval(iter) +} + +func (e *eval) unify(a, b *ast.Term, iter unifyIterator) error { + return e.biunify(a, b, e.bindings, e.bindings, iter) +} + +func (e *eval) biunify(a, b *ast.Term, b1, b2 *bindings, iter unifyIterator) error { + a, b1 = b1.apply(a) + b, b2 = b2.apply(b) + switch vA := a.Value.(type) { + case ast.Var, ast.Ref, *ast.ArrayComprehension, *ast.SetComprehension, *ast.ObjectComprehension: + return e.biunifyValues(a, b, b1, b2, iter) + case ast.Null: + switch b.Value.(type) { + case ast.Var, ast.Null, ast.Ref: + return e.biunifyValues(a, b, b1, b2, iter) + } + case ast.Boolean: + switch b.Value.(type) { + case ast.Var, ast.Boolean, ast.Ref: + return e.biunifyValues(a, b, b1, b2, iter) + } + case ast.Number: + switch b.Value.(type) { + case ast.Var, ast.Number, ast.Ref: + return e.biunifyValues(a, b, b1, b2, iter) + } + case ast.String: + switch b.Value.(type) { + case ast.Var, ast.String, ast.Ref: + return e.biunifyValues(a, b, b1, b2, iter) + } + case ast.Array: + switch vB := b.Value.(type) { + case ast.Var, ast.Ref, *ast.ArrayComprehension: + return e.biunifyValues(a, b, b1, b2, iter) + case ast.Array: + return e.biunifyArrays(vA, vB, b1, b2, iter) + } + case ast.Object: + switch vB := b.Value.(type) { + case ast.Var, ast.Ref, *ast.ObjectComprehension: + return e.biunifyValues(a, b, b1, b2, iter) + case ast.Object: + return e.biunifyObjects(vA, vB, b1, b2, iter) + } + case ast.Set: + return e.biunifyValues(a, b, b1, b2, iter) + } + return nil +} + +func (e *eval) biunifyArrays(a, b ast.Array, b1, b2 *bindings, iter unifyIterator) error { + if len(a) != len(b) { + return nil + } + return e.biunifyArraysRec(a, b, b1, b2, iter, 0) +} + +func (e *eval) biunifyArraysRec(a, b ast.Array, b1, b2 *bindings, iter unifyIterator, idx int) error { + if idx == len(a) { + return iter() + } + return e.biunify(a[idx], b[idx], b1, b2, func() error { + return e.biunifyArraysRec(a, b, b1, b2, iter, idx+1) + }) +} + +func (e *eval) biunifyObjects(a, b ast.Object, b1, b2 *bindings, iter unifyIterator) error { + if a.Len() != b.Len() { + return nil + } + + // Objects must not contain unbound variables as keys at this point as we + // cannot unify them. Similar to sets, plug both sides before comparing the + // keys and unifying the values. + if nonGroundKeys(a) { + a = plugKeys(a, b1) + } + + if nonGroundKeys(b) { + b = plugKeys(b, b2) + } + + return e.biunifyObjectsRec(a, b, b1, b2, iter, a.Keys(), 0) +} + +func (e *eval) biunifyObjectsRec(a, b ast.Object, b1, b2 *bindings, iter unifyIterator, keys []*ast.Term, idx int) error { + if idx == len(keys) { + return iter() + } + v2 := b.Get(keys[idx]) + if v2 == nil { + return nil + } + return e.biunify(a.Get(keys[idx]), v2, b1, b2, func() error { + return e.biunifyObjectsRec(a, b, b1, b2, iter, keys, idx+1) + }) +} + +func (e *eval) biunifyValues(a, b *ast.Term, b1, b2 *bindings, iter unifyIterator) error { + // Try to evaluate refs and comprehensions. If partial evaluation is + // enabled, then skip evaluation (and save the expression) if the term is + // in the save set. Currently, comprehensions are not evaluated during + // partial eval. This could be improved in the future. + + var saveA, saveB bool + + if _, ok := a.Value.(ast.Set); ok { + saveA = e.saveSet.ContainsRecursive(a, b1) + } else { + saveA = e.saveSet.Contains(a, b1) + if !saveA { + if _, refA := a.Value.(ast.Ref); refA { + return e.biunifyRef(a, b, b1, b2, iter) + } + } + } + + if _, ok := b.Value.(ast.Set); ok { + saveB = e.saveSet.ContainsRecursive(b, b2) + } else { + saveB = e.saveSet.Contains(b, b2) + if !saveB { + if _, refB := b.Value.(ast.Ref); refB { + return e.biunifyRef(b, a, b2, b1, iter) + } + } + } + + if saveA || saveB { + return e.saveUnify(a, b, b1, b2, iter) + } + + if ast.IsComprehension(a.Value) { + return e.biunifyComprehension(a, b, b1, b2, false, iter) + } else if ast.IsComprehension(b.Value) { + return e.biunifyComprehension(b, a, b2, b1, true, iter) + } + + // Perform standard unification. + _, varA := a.Value.(ast.Var) + _, varB := b.Value.(ast.Var) + + if varA && varB { + if b1 == b2 && a.Equal(b) { + return iter() + } + undo := b1.bind(a, b, b2) + err := iter() + undo.Undo() + return err + } else if varA && !varB { + undo := b1.bind(a, b, b2) + err := iter() + undo.Undo() + return err + } else if varB && !varA { + undo := b2.bind(b, a, b1) + err := iter() + undo.Undo() + return err + } + + // Sets must not contain unbound variables at this point as we cannot unify + // them. So simply plug both sides (to substitute any bound variables with + // values) and then check for equality. + switch a.Value.(type) { + case ast.Set: + a = b1.Plug(a) + b = b2.Plug(b) + } + + if a.Equal(b) { + return iter() + } + + return nil +} + +func (e *eval) biunifyRef(a, b *ast.Term, b1, b2 *bindings, iter unifyIterator) error { + + ref := a.Value.(ast.Ref) + + if ref[0].Equal(ast.DefaultRootDocument) { + node := e.compiler.RuleTree.Child(ref[0].Value) + + eval := evalTree{ + e: e, + ref: ref, + pos: 1, + plugged: ref.Copy(), + bindings: b1, + rterm: b, + rbindings: b2, + node: node, + } + return eval.eval(iter) + } + + var term *ast.Term + var termbindings *bindings + + if ref[0].Equal(ast.InputRootDocument) { + term = e.input + termbindings = b1 + } else { + term, termbindings = b1.apply(ref[0]) + if term == ref[0] { + term = nil + } + } + + if term == nil { + return nil + } + + eval := evalTerm{ + e: e, + ref: ref, + pos: 1, + bindings: b1, + term: term, + termbindings: termbindings, + rterm: b, + rbindings: b2, + } + + return eval.eval(iter) +} + +func (e *eval) biunifyComprehension(a, b *ast.Term, b1, b2 *bindings, swap bool, iter unifyIterator) error { + + if e.unknown(a, b1) { + return e.biunifyComprehensionPartial(a, b, b1, b2, swap, iter) + } + + switch a := a.Value.(type) { + case *ast.ArrayComprehension: + return e.biunifyComprehensionArray(a, b, b1, b2, iter) + case *ast.SetComprehension: + return e.biunifyComprehensionSet(a, b, b1, b2, iter) + case *ast.ObjectComprehension: + return e.biunifyComprehensionObject(a, b, b1, b2, iter) + } + + return fmt.Errorf("illegal comprehension %T", a) +} + +func (e *eval) biunifyComprehensionPartial(a, b *ast.Term, b1, b2 *bindings, swap bool, iter unifyIterator) error { + + // Capture bindings available to the comprehension. We will add expressions + // to the comprehension body that ensure the comprehension body is safe. + // Currently this process adds _all_ bindings (even if they are not + // needed.) Eventually we may want to make the logic a bit smarter. + var extras []*ast.Expr + + err := b1.Iter(e.caller.bindings, func(k, v *ast.Term) error { + extras = append(extras, ast.Equality.Expr(k, v)) + return nil + }) + + if err != nil { + return err + } + + // Namespace the variables in the body to avoid collision when the final + // queries returned by partial evaluation. + var body *ast.Body + + switch a := a.Value.(type) { + case *ast.ArrayComprehension: + body = &a.Body + case *ast.SetComprehension: + body = &a.Body + case *ast.ObjectComprehension: + body = &a.Body + default: + return fmt.Errorf("illegal comprehension %T", a) + } + + for _, e := range extras { + body.Append(e) + } + + b1.Namespace(a, e.caller.bindings) + + // The other term might need to be plugged so include the bindings. The + // bindings for the comprehension term are saved (for compatibility) but + // the eventual plug operation on the comprehension will be a no-op. + if !swap { + return e.saveUnify(a, b, b1, b2, iter) + } + + return e.saveUnify(b, a, b2, b1, iter) +} + +func (e *eval) biunifyComprehensionArray(x *ast.ArrayComprehension, b *ast.Term, b1, b2 *bindings, iter unifyIterator) error { + result := ast.Array{} + child := e.closure(x.Body) + err := child.Run(func(child *eval) error { + result = append(result, child.bindings.Plug(x.Term)) + return nil + }) + if err != nil { + return err + } + return e.biunify(ast.NewTerm(result), b, b1, b2, iter) +} + +func (e *eval) biunifyComprehensionSet(x *ast.SetComprehension, b *ast.Term, b1, b2 *bindings, iter unifyIterator) error { + result := ast.NewSet() + child := e.closure(x.Body) + err := child.Run(func(child *eval) error { + result.Add(child.bindings.Plug(x.Term)) + return nil + }) + if err != nil { + return err + } + return e.biunify(ast.NewTerm(result), b, b1, b2, iter) +} + +func (e *eval) biunifyComprehensionObject(x *ast.ObjectComprehension, b *ast.Term, b1, b2 *bindings, iter unifyIterator) error { + result := ast.NewObject() + child := e.closure(x.Body) + err := child.Run(func(child *eval) error { + key := child.bindings.Plug(x.Key) + value := child.bindings.Plug(x.Value) + exist := result.Get(key) + if exist != nil && !exist.Equal(value) { + return objectDocKeyConflictErr(x.Key.Location) + } + result.Insert(key, value) + return nil + }) + if err != nil { + return err + } + return e.biunify(ast.NewTerm(result), b, b1, b2, iter) +} + +type savePair struct { + term *ast.Term + b *bindings +} + +func getSavePairs(x *ast.Term, b *bindings, result []savePair) []savePair { + if _, ok := x.Value.(ast.Var); ok { + result = append(result, savePair{x, b}) + return result + } + vis := ast.NewVarVisitor().WithParams(ast.VarVisitorParams{ + SkipClosures: true, + SkipRefHead: true, + }) + vis.Walk(x) + for v := range vis.Vars() { + y, next := b.apply(ast.NewTerm(v)) + result = getSavePairs(y, next, result) + } + return result +} + +func (e *eval) saveExpr(expr *ast.Expr, b *bindings, iter unifyIterator) error { + expr.With = e.query[e.index].With + e.saveStack.Push(expr, b, b) + e.traceSave(expr) + err := iter() + e.saveStack.Pop() + return err +} + +func (e *eval) saveUnify(a, b *ast.Term, b1, b2 *bindings, iter unifyIterator) error { + e.instr.startTimer(partialOpSaveUnify) + expr := ast.Equality.Expr(a, b) + expr.With = e.query[e.index].With + pops := 0 + if pairs := getSavePairs(a, b1, nil); len(pairs) > 0 { + pops += len(pairs) + for _, p := range pairs { + e.saveSet.Push([]*ast.Term{p.term}, p.b) + } + + } + if pairs := getSavePairs(b, b2, nil); len(pairs) > 0 { + pops += len(pairs) + for _, p := range pairs { + e.saveSet.Push([]*ast.Term{p.term}, p.b) + } + } + e.saveStack.Push(expr, b1, b2) + e.traceSave(expr) + e.instr.stopTimer(partialOpSaveUnify) + err := iter() + + e.saveStack.Pop() + for i := 0; i < pops; i++ { + e.saveSet.Pop() + } + + return err +} + +func (e *eval) saveCall(declArgsLen int, terms []*ast.Term, iter unifyIterator) error { + expr := ast.NewExpr(terms) + expr.With = e.query[e.index].With + + // If call-site includes output value then partial eval must add vars in output + // position to the save set. + pops := 0 + if declArgsLen == len(terms)-2 { + if pairs := getSavePairs(terms[len(terms)-1], e.bindings, nil); len(pairs) > 0 { + pops += len(pairs) + for _, p := range pairs { + e.saveSet.Push([]*ast.Term{p.term}, p.b) + } + } + } + e.saveStack.Push(expr, e.bindings, nil) + e.traceSave(expr) + err := iter() + + e.saveStack.Pop() + for i := 0; i < pops; i++ { + e.saveSet.Pop() + } + return err +} + +func (e *eval) saveInlinedNegatedExprs(exprs []*ast.Expr, iter unifyIterator) error { + + // This function does not include with statements on the exprs because + // they will have already been saved and therefore had their any relevant + // with statements set. + for _, expr := range exprs { + e.saveStack.Push(expr, nil, nil) + e.traceSave(expr) + } + err := iter() + for i := 0; i < len(exprs); i++ { + e.saveStack.Pop() + } + return err +} + +func (e *eval) getRules(ref ast.Ref) (*ast.IndexResult, error) { + e.instr.startTimer(evalOpRuleIndex) + defer e.instr.stopTimer(evalOpRuleIndex) + + index := e.compiler.RuleIndex(ref) + if index == nil { + return nil, nil + } + + var result *ast.IndexResult + var err error + if e.indexing { + result, err = index.Lookup(e) + } else { + result, err = index.AllRules(e) + } + + if err != nil { + return nil, err + } + + var msg string + if len(result.Rules) == 1 { + msg = "(matched 1 rule)" + } else { + var b strings.Builder + b.Grow(len("(matched NNNN rules)")) + b.WriteString("matched ") + b.WriteString(strconv.FormatInt(int64(len(result.Rules)), 10)) + b.WriteString(" rules)") + msg = b.String() + } + e.traceIndex(e.query[e.index], msg) + return result, err +} + +func (e *eval) Resolve(ref ast.Ref) (ast.Value, error) { + e.instr.startTimer(evalOpResolve) + + if e.saveSet.Contains(ast.NewTerm(ref), nil) { + e.instr.stopTimer(evalOpResolve) + return nil, ast.UnknownValueErr{} + } + + if ref[0].Equal(ast.InputRootDocument) { + if e.input != nil { + v, err := e.input.Value.Find(ref[1:]) + if err != nil { + v = nil + } + e.instr.stopTimer(evalOpResolve) + return v, nil + } + e.instr.stopTimer(evalOpResolve) + return nil, nil + } + + if ref[0].Equal(ast.DefaultRootDocument) { + + var repValue ast.Value + + if e.data != nil { + if v, err := e.data.Value.Find(ref[1:]); err == nil { + repValue = v + } else { + repValue = nil + } + } + + if e.targetStack.Prefixed(ref) { + e.instr.stopTimer(evalOpResolve) + return repValue, nil + } + + var merged ast.Value + var err error + + // Converting large JSON values into AST values can be fairly expensive. For + // example, a 2MB JSON value can take upwards of 30 millisceonds to convert. + // We cache the result of conversion here in case the same base document is + // being read multiple times during evaluation. + realValue := e.baseCache.Get(ref) + if realValue != nil { + e.instr.counterIncr(evalOpBaseCacheHit) + if repValue == nil { + e.instr.stopTimer(evalOpResolve) + return realValue, nil + } + var ok bool + merged, ok = merge(repValue, realValue) + if !ok { + err = mergeConflictErr(ref[0].Location) + } + } else { + e.instr.counterIncr(evalOpBaseCacheMiss) + merged, err = e.resolveReadFromStorage(ref, repValue) + } + e.instr.stopTimer(evalOpResolve) + return merged, err + } + e.instr.stopTimer(evalOpResolve) + return nil, fmt.Errorf("illegal ref") +} + +func (e *eval) resolveReadFromStorage(ref ast.Ref, a ast.Value) (ast.Value, error) { + if refContainsNonScalar(ref) { + return a, nil + } + + path, err := storage.NewPathForRef(ref) + if err != nil { + if !storage.IsNotFound(err) { + return nil, err + } + return a, nil + } + + blob, err := e.store.Read(e.ctx, e.txn, path) + if err != nil { + if !storage.IsNotFound(err) { + return nil, err + } + return a, nil + } + + if len(path) == 0 { + obj := blob.(map[string]interface{}) + if len(obj) > 0 { + cpy := make(map[string]interface{}, len(obj)-1) + for k, v := range obj { + if string(ast.SystemDocumentKey) == k { + continue + } + cpy[k] = v + } + blob = cpy + } + } + + v, err := ast.InterfaceToValue(blob) + if err != nil { + return nil, err + } + + e.baseCache.Put(ref, v) + + if a == nil { + return v, nil + } + + merged, ok := merge(a, v) + if !ok { + return nil, mergeConflictErr(ref[0].Location) + } + return merged, nil +} + +func (e *eval) generateVar(suffix string) *ast.Term { + return ast.VarTerm(fmt.Sprintf("%v_%v", e.genvarprefix, suffix)) +} + +func (e *eval) rewrittenVar(v ast.Var) (ast.Var, bool) { + if e.compiler != nil { + if rw, ok := e.compiler.RewrittenVars[v]; ok { + return rw, true + } + } + if e.queryCompiler != nil { + if rw, ok := e.queryCompiler.RewrittenVars()[v]; ok { + return rw, true + } + } + return v, false +} + +type evalBuiltin struct { + e *eval + bi *ast.Builtin + bctx BuiltinContext + f BuiltinFunc + terms []*ast.Term +} + +func (e evalBuiltin) eval(iter unifyIterator) error { + + operands := make([]*ast.Term, len(e.terms)) + + for i := 0; i < len(e.terms); i++ { + operands[i] = e.e.bindings.Plug(e.terms[i]) + } + + numDeclArgs := len(e.bi.Decl.Args()) + + e.e.instr.startTimer(evalOpBuiltinCall) + + err := e.f(e.bctx, operands, func(output *ast.Term) error { + + e.e.instr.stopTimer(evalOpBuiltinCall) + + var err error + if len(operands) == numDeclArgs { + if output.Value.Compare(ast.Boolean(false)) != 0 { + err = iter() + } + } else { + err = e.e.unify(e.terms[len(e.terms)-1], output, iter) + } + e.e.instr.startTimer(evalOpBuiltinCall) + return err + }) + + e.e.instr.stopTimer(evalOpBuiltinCall) + return err +} + +type evalFunc struct { + e *eval + ref ast.Ref + terms []*ast.Term +} + +func (e evalFunc) eval(iter unifyIterator) error { + + ir, err := e.e.getRules(e.ref) + if err != nil { + return err + } + + if ir.Empty() { + return nil + } + + if len(ir.Else) > 0 && e.e.unknown(e.e.query[e.e.index], e.e.bindings) { + // Partial evaluation of ordered rules is not supported currently. Save the + // expression and continue. This could be revisited in the future. + return e.e.saveCall(len(ir.Rules[0].Head.Args), e.terms, iter) + } + + var prev *ast.Term + + for i := range ir.Rules { + next, err := e.evalOneRule(iter, ir.Rules[i], prev) + if err != nil { + return err + } + if next == nil { + for _, rule := range ir.Else[ir.Rules[i]] { + next, err = e.evalOneRule(iter, rule, prev) + if err != nil { + return err + } + if next != nil { + break + } + } + } + if next != nil { + prev = next + } + } + + return nil +} + +func (e evalFunc) evalOneRule(iter unifyIterator, rule *ast.Rule, prev *ast.Term) (*ast.Term, error) { + + child := e.e.child(rule.Body) + + args := make(ast.Array, len(e.terms)-1) + + for i := range rule.Head.Args { + args[i] = rule.Head.Args[i] + } + + if len(args) == len(rule.Head.Args)+1 { + args[len(args)-1] = rule.Head.Value + } + + var result *ast.Term + + child.traceEnter(rule) + + err := child.biunifyArrays(e.terms[1:], args, e.e.bindings, child.bindings, func() error { + return child.eval(func(child *eval) error { + child.traceExit(rule) + result = child.bindings.Plug(rule.Head.Value) + + if len(rule.Head.Args) == len(e.terms)-1 { + if result.Value.Compare(ast.Boolean(false)) == 0 { + return nil + } + } + + // Partial evaluation should explore all rules and may not produce + // a ground result so we do not perform conflict detection or + // deduplication. See "ignore conflicts: functions" test case for + // an example. + if !e.e.partial() { + if prev != nil { + if ast.Compare(prev, result) != 0 { + return functionConflictErr(rule.Location) + } + child.traceRedo(rule) + return nil + } + } + + prev = result + + if err := iter(); err != nil { + return err + } + + child.traceRedo(rule) + return nil + }) + }) + + return result, err +} + +type evalTree struct { + e *eval + ref ast.Ref + plugged ast.Ref + pos int + bindings *bindings + rterm *ast.Term + rbindings *bindings + node *ast.TreeNode +} + +func (e evalTree) eval(iter unifyIterator) error { + + if len(e.ref) == e.pos { + return e.finish(iter) + } + + plugged := e.bindings.Plug(e.ref[e.pos]) + + if plugged.IsGround() { + return e.next(iter, plugged) + } + + return e.enumerate(iter) +} + +func (e evalTree) finish(iter unifyIterator) error { + + // During partial evaluation it may not be possible to compute the value + // for this reference if it refers to a virtual document so save the entire + // expression. See "save: full extent" test case for an example. + if e.node != nil && e.e.unknown(e.ref, e.e.bindings) { + return e.e.saveUnify(ast.NewTerm(e.plugged), e.rterm, e.bindings, e.rbindings, iter) + } + + v, err := e.extent() + if err != nil || v == nil { + return err + } + + return e.e.biunify(e.rterm, v, e.rbindings, e.bindings, func() error { + return iter() + }) +} + +func (e evalTree) next(iter unifyIterator, plugged *ast.Term) error { + + var node *ast.TreeNode + + cpy := e + cpy.plugged[e.pos] = plugged + cpy.pos++ + + if !e.e.targetStack.Prefixed(cpy.plugged[:cpy.pos]) { + if e.node != nil { + node = e.node.Child(plugged.Value) + if node != nil && len(node.Values) > 0 { + r := evalVirtual{ + e: e.e, + ref: e.ref, + plugged: e.plugged, + pos: e.pos, + bindings: e.bindings, + rterm: e.rterm, + rbindings: e.rbindings, + } + r.plugged[e.pos] = plugged + return r.eval(iter) + } + } + } + + cpy.node = node + return cpy.eval(iter) +} + +func (e evalTree) enumerate(iter unifyIterator) error { + doc, err := e.e.Resolve(e.plugged[:e.pos]) + if err != nil { + return err + } + + if doc != nil { + switch doc := doc.(type) { + case ast.Array: + for i := range doc { + k := ast.IntNumberTerm(i) + err := e.e.biunify(k, e.ref[e.pos], e.bindings, e.bindings, func() error { + return e.next(iter, k) + }) + if err != nil { + return err + } + } + case ast.Object: + err := doc.Iter(func(k, _ *ast.Term) error { + return e.e.biunify(k, e.ref[e.pos], e.bindings, e.bindings, func() error { + return e.next(iter, k) + }) + }) + if err != nil { + return err + } + case ast.Set: + err := doc.Iter(func(elem *ast.Term) error { + return e.e.biunify(elem, e.ref[e.pos], e.bindings, e.bindings, func() error { + return e.next(iter, elem) + }) + }) + if err != nil { + return err + } + } + } + + if e.node == nil { + return nil + } + + for k := range e.node.Children { + key := ast.NewTerm(k) + if err := e.e.biunify(key, e.ref[e.pos], e.bindings, e.bindings, func() error { + return e.next(iter, key) + }); err != nil { + return err + } + } + + return nil +} + +func (e evalTree) extent() (*ast.Term, error) { + base, err := e.e.Resolve(e.plugged) + if err != nil { + return nil, err + } + + virtual, err := e.leaves(e.plugged, e.node) + if err != nil { + return nil, err + } + + if virtual == nil { + if base == nil { + return nil, nil + } + return ast.NewTerm(base), nil + } + + if base != nil { + merged, ok := merge(base, virtual) + if !ok { + return nil, mergeConflictErr(e.plugged[0].Location) + } + return ast.NewTerm(merged), nil + } + + return ast.NewTerm(virtual), nil +} + +func (e evalTree) leaves(plugged ast.Ref, node *ast.TreeNode) (ast.Object, error) { + + if e.node == nil { + return nil, nil + } + + result := ast.NewObject() + + for _, child := range node.Children { + if child.Hide { + continue + } + + plugged = append(plugged, ast.NewTerm(child.Key)) + + var save ast.Value + var err error + + if len(child.Values) > 0 { + rterm := e.e.generateVar("leaf") + err = e.e.unify(ast.NewTerm(plugged), rterm, func() error { + save = e.e.bindings.Plug(rterm).Value + return nil + }) + } else { + save, err = e.leaves(plugged, child) + } + + if err != nil { + return nil, err + } + + if save != nil { + v := ast.NewObject([2]*ast.Term{plugged[len(plugged)-1], ast.NewTerm(save)}) + result, _ = result.Merge(v) + } + + plugged = plugged[:len(plugged)-1] + } + + return result, nil +} + +type evalVirtual struct { + e *eval + ref ast.Ref + plugged ast.Ref + pos int + bindings *bindings + rterm *ast.Term + rbindings *bindings +} + +func (e evalVirtual) eval(iter unifyIterator) error { + + ir, err := e.e.getRules(e.plugged[:e.pos+1]) + if err != nil { + return err + } + + // Partial evaluation of ordered rules is not supported currently. Save the + // expression and continue. This could be revisited in the future. + if len(ir.Else) > 0 && e.e.unknown(e.ref, e.bindings) { + return e.e.saveUnify(ast.NewTerm(e.ref), e.rterm, e.bindings, e.rbindings, iter) + } + + switch ir.Kind { + case ast.PartialSetDoc: + eval := evalVirtualPartial{ + e: e.e, + ref: e.ref, + plugged: e.plugged, + pos: e.pos, + ir: ir, + bindings: e.bindings, + rterm: e.rterm, + rbindings: e.rbindings, + empty: ast.SetTerm(), + } + return eval.eval(iter) + case ast.PartialObjectDoc: + eval := evalVirtualPartial{ + e: e.e, + ref: e.ref, + plugged: e.plugged, + pos: e.pos, + ir: ir, + bindings: e.bindings, + rterm: e.rterm, + rbindings: e.rbindings, + empty: ast.ObjectTerm(), + } + return eval.eval(iter) + default: + eval := evalVirtualComplete{ + e: e.e, + ref: e.ref, + plugged: e.plugged, + pos: e.pos, + ir: ir, + bindings: e.bindings, + rterm: e.rterm, + rbindings: e.rbindings, + } + return eval.eval(iter) + } +} + +type evalVirtualPartial struct { + e *eval + ref ast.Ref + plugged ast.Ref + pos int + ir *ast.IndexResult + bindings *bindings + rterm *ast.Term + rbindings *bindings + empty *ast.Term +} + +func (e evalVirtualPartial) eval(iter unifyIterator) error { + + if len(e.ref) == e.pos+1 { + // During partial evaluation, it may not be possible to produce a value + // for this reference so save the entire expression. See "save: full + // extent: partial object" test case for an example. + if e.e.unknown(e.ref, e.bindings) { + return e.e.saveUnify(ast.NewTerm(e.ref), e.rterm, e.bindings, e.rbindings, iter) + } + return e.evalAllRules(iter, e.ir.Rules) + } + + var cacheKey ast.Ref + + if e.ir.Kind == ast.PartialObjectDoc { + plugged := e.bindings.Plug(e.ref[e.pos+1]) + + if plugged.IsGround() { + path := e.plugged[:e.pos+2] + path[len(path)-1] = plugged + cached := e.e.virtualCache.Get(path) + + if cached != nil { + e.e.instr.counterIncr(evalOpVirtualCacheHit) + return e.evalTerm(iter, cached, e.bindings) + } + + e.e.instr.counterIncr(evalOpVirtualCacheMiss) + cacheKey = path + } + } + + generateSupport := anyRefSetContainsPrefix(e.e.disableInlining, e.plugged[:e.pos+1]) + + if generateSupport { + return e.partialEvalSupport(iter) + } + + for _, rule := range e.ir.Rules { + if err := e.evalOneRule(iter, rule, cacheKey); err != nil { + return err + } + } + + return nil +} + +func (e evalVirtualPartial) evalAllRules(iter unifyIterator, rules []*ast.Rule) error { + + result := e.empty + + for _, rule := range rules { + child := e.e.child(rule.Body) + child.traceEnter(rule) + + err := child.eval(func(*eval) error { + child.traceExit(rule) + var err error + result, err = e.reduce(rule.Head, child.bindings, result) + if err != nil { + return err + } + + child.traceRedo(rule) + return nil + }) + + if err != nil { + return err + } + } + + return e.e.biunify(result, e.rterm, e.bindings, e.bindings, iter) +} + +func (e evalVirtualPartial) evalOneRule(iter unifyIterator, rule *ast.Rule, cacheKey ast.Ref) error { + + key := e.ref[e.pos+1] + child := e.e.child(rule.Body) + + child.traceEnter(rule) + var defined bool + + err := child.biunify(rule.Head.Key, key, child.bindings, e.bindings, func() error { + defined = true + return child.eval(func(child *eval) error { + child.traceExit(rule) + + term := rule.Head.Value + if term == nil { + term = rule.Head.Key + } + + if cacheKey != nil { + result := child.bindings.Plug(term) + e.e.virtualCache.Put(cacheKey, result) + } + + term, termbindings := child.bindings.apply(term) + err := e.evalTerm(iter, term, termbindings) + if err != nil { + return err + } + + child.traceRedo(rule) + return nil + }) + }) + + if err != nil { + return err + } + + if !defined { + child.traceFail(rule) + } + + return nil +} + +func (e evalVirtualPartial) partialEvalSupport(iter unifyIterator) error { + + path := e.plugged[:e.pos+1].Insert(e.e.saveNamespace, 1) + + if !e.e.saveSupport.Exists(path) { + for i := range e.ir.Rules { + err := e.partialEvalSupportRule(iter, e.ir.Rules[i], path) + if err != nil { + return err + } + } + } + + rewritten := ast.NewTerm(e.ref.Insert(e.e.saveNamespace, 1)) + return e.e.saveUnify(rewritten, e.rterm, e.bindings, e.rbindings, iter) +} + +func (e evalVirtualPartial) partialEvalSupportRule(iter unifyIterator, rule *ast.Rule, path ast.Ref) error { + + child := e.e.child(rule.Body) + child.traceEnter(rule) + + e.e.saveStack.PushQuery(nil) + + err := child.eval(func(child *eval) error { + child.traceExit(rule) + + current := e.e.saveStack.PopQuery() + plugged := current.Plug(e.e.caller.bindings) + + var key, value *ast.Term + + if rule.Head.Key != nil { + key = child.bindings.PlugNamespaced(rule.Head.Key, e.e.caller.bindings) + } + + if rule.Head.Value != nil { + value = child.bindings.PlugNamespaced(rule.Head.Value, e.e.caller.bindings) + } + + head := ast.NewHead(rule.Head.Name, key, value) + p := copypropagation.New(head.Vars()).WithEnsureNonEmptyBody(true) + + e.e.saveSupport.Insert(path, &ast.Rule{ + Head: head, + Body: p.Apply(plugged), + Default: rule.Default, + }) + + child.traceRedo(rule) + e.e.saveStack.PushQuery(current) + return nil + }) + e.e.saveStack.PopQuery() + return err +} + +func (e evalVirtualPartial) evalTerm(iter unifyIterator, term *ast.Term, termbindings *bindings) error { + eval := evalTerm{ + e: e.e, + ref: e.ref, + pos: e.pos + 2, + bindings: e.bindings, + term: term, + termbindings: termbindings, + rterm: e.rterm, + rbindings: e.rbindings, + } + return eval.eval(iter) +} + +func (e evalVirtualPartial) reduce(head *ast.Head, b *bindings, result *ast.Term) (*ast.Term, error) { + + switch v := result.Value.(type) { + case ast.Set: + v.Add(b.Plug(head.Key)) + case ast.Object: + key := b.Plug(head.Key) + value := b.Plug(head.Value) + exist := v.Get(key) + if exist != nil && !exist.Equal(value) { + return nil, objectDocKeyConflictErr(head.Location) + } + v.Insert(key, value) + result.Value = v + } + + return result, nil +} + +type evalVirtualComplete struct { + e *eval + ref ast.Ref + plugged ast.Ref + pos int + ir *ast.IndexResult + bindings *bindings + rterm *ast.Term + rbindings *bindings +} + +func (e evalVirtualComplete) eval(iter unifyIterator) error { + + if e.ir.Empty() { + return nil + } + + if len(e.ir.Rules) > 0 && len(e.ir.Rules[0].Head.Args) > 0 { + return nil + } + + if !e.e.unknown(e.ref, e.bindings) { + return e.evalValue(iter) + } + + var generateSupport bool + + if e.ir.Default != nil { + // If the other term is not constant OR it's equal to the default value, then + // a support rule must be produced as the default value _may_ be required. On + // the other hand, if the other term is constant (i.e., it does not require + // evaluation) and it differs from the default value then the default value is + // _not_ required, so partially evaluate the rule normally. + rterm := e.rbindings.Plug(e.rterm) + generateSupport = !ast.IsConstant(rterm.Value) || e.ir.Default.Head.Value.Equal(rterm) + } + + generateSupport = generateSupport || anyRefSetContainsPrefix(e.e.disableInlining, e.plugged[:e.pos+1]) + + if generateSupport { + return e.partialEvalSupport(iter) + } + + return e.partialEval(iter) +} + +func (e evalVirtualComplete) evalValue(iter unifyIterator) error { + cached := e.e.virtualCache.Get(e.plugged[:e.pos+1]) + if cached != nil { + e.e.instr.counterIncr(evalOpVirtualCacheHit) + return e.evalTerm(iter, cached, e.bindings) + } + + e.e.instr.counterIncr(evalOpVirtualCacheMiss) + + var prev *ast.Term + + for i := range e.ir.Rules { + next, err := e.evalValueRule(iter, e.ir.Rules[i], prev) + if err != nil { + return err + } + if next == nil { + for _, rule := range e.ir.Else[e.ir.Rules[i]] { + next, err = e.evalValueRule(iter, rule, prev) + if err != nil { + return err + } + if next != nil { + break + } + } + } + if next != nil { + prev = next + } + } + + if e.ir.Default != nil && prev == nil { + _, err := e.evalValueRule(iter, e.ir.Default, prev) + return err + } + + return nil +} + +func (e evalVirtualComplete) evalValueRule(iter unifyIterator, rule *ast.Rule, prev *ast.Term) (*ast.Term, error) { + + child := e.e.child(rule.Body) + child.traceEnter(rule) + var result *ast.Term + + err := child.eval(func(child *eval) error { + child.traceExit(rule) + result = child.bindings.Plug(rule.Head.Value) + + if prev != nil { + if ast.Compare(result, prev) != 0 { + return completeDocConflictErr(rule.Location) + } + child.traceRedo(rule) + return nil + } + + prev = result + e.e.virtualCache.Put(e.plugged[:e.pos+1], result) + term, termbindings := child.bindings.apply(rule.Head.Value) + + err := e.evalTerm(iter, term, termbindings) + if err != nil { + return err + } + + child.traceRedo(rule) + return nil + }) + + return result, err +} + +func (e evalVirtualComplete) partialEval(iter unifyIterator) error { + + for _, rule := range e.ir.Rules { + child := e.e.child(rule.Body) + child.traceEnter(rule) + + err := child.eval(func(child *eval) error { + child.traceExit(rule) + term, termbindings := child.bindings.apply(rule.Head.Value) + + err := e.evalTerm(iter, term, termbindings) + if err != nil { + return err + } + + child.traceRedo(rule) + return nil + }) + + if err != nil { + return err + } + } + + return nil +} + +func (e evalVirtualComplete) partialEvalSupport(iter unifyIterator) error { + + path := e.plugged[:e.pos+1].Insert(e.e.saveNamespace, 1) + + if !e.e.saveSupport.Exists(path) { + + for i := range e.ir.Rules { + err := e.partialEvalSupportRule(iter, e.ir.Rules[i], path) + if err != nil { + return err + } + } + + if e.ir.Default != nil { + err := e.partialEvalSupportRule(iter, e.ir.Default, path) + if err != nil { + return err + } + } + } + + rewritten := ast.NewTerm(e.ref.Insert(e.e.saveNamespace, 1)) + return e.e.saveUnify(rewritten, e.rterm, e.bindings, e.rbindings, iter) +} + +func (e evalVirtualComplete) partialEvalSupportRule(iter unifyIterator, rule *ast.Rule, path ast.Ref) error { + + child := e.e.child(rule.Body) + child.traceEnter(rule) + + e.e.saveStack.PushQuery(nil) + + err := child.eval(func(child *eval) error { + child.traceExit(rule) + + current := e.e.saveStack.PopQuery() + plugged := current.Plug(e.e.caller.bindings) + + head := ast.NewHead(rule.Head.Name, nil, child.bindings.PlugNamespaced(rule.Head.Value, e.e.caller.bindings)) + p := copypropagation.New(head.Vars()).WithEnsureNonEmptyBody(true) + + e.e.saveSupport.Insert(path, &ast.Rule{ + Head: head, + Body: applyCopyPropagation(p, e.e.instr, plugged), + Default: rule.Default, + }) + + child.traceRedo(rule) + e.e.saveStack.PushQuery(current) + return nil + }) + e.e.saveStack.PopQuery() + return err +} + +func (e evalVirtualComplete) evalTerm(iter unifyIterator, term *ast.Term, termbindings *bindings) error { + eval := evalTerm{ + e: e.e, + ref: e.ref, + pos: e.pos + 1, + bindings: e.bindings, + term: term, + termbindings: termbindings, + rterm: e.rterm, + rbindings: e.rbindings, + } + return eval.eval(iter) +} + +type evalTerm struct { + e *eval + ref ast.Ref + pos int + bindings *bindings + term *ast.Term + termbindings *bindings + rterm *ast.Term + rbindings *bindings +} + +func (e evalTerm) eval(iter unifyIterator) error { + + if len(e.ref) == e.pos { + return e.e.biunify(e.term, e.rterm, e.termbindings, e.rbindings, iter) + } + + if e.e.saveSet.Contains(e.term, e.termbindings) { + return e.save(iter) + } + + plugged := e.bindings.Plug(e.ref[e.pos]) + + if plugged.IsGround() { + return e.next(iter, plugged) + } + + return e.enumerate(iter) +} + +func (e evalTerm) next(iter unifyIterator, plugged *ast.Term) error { + + term, bindings := e.get(plugged) + if term == nil { + return nil + } + + cpy := e + cpy.term = term + cpy.termbindings = bindings + cpy.pos++ + return cpy.eval(iter) +} + +func (e evalTerm) enumerate(iter unifyIterator) error { + + switch v := e.term.Value.(type) { + case ast.Array: + for i := range v { + k := ast.IntNumberTerm(i) + err := e.e.biunify(k, e.ref[e.pos], e.bindings, e.bindings, func() error { + return e.next(iter, k) + }) + if err != nil { + return err + } + } + case ast.Object: + return v.Iter(func(k, _ *ast.Term) error { + return e.e.biunify(k, e.ref[e.pos], e.termbindings, e.bindings, func() error { + return e.next(iter, e.termbindings.Plug(k)) + }) + }) + case ast.Set: + return v.Iter(func(elem *ast.Term) error { + return e.e.biunify(elem, e.ref[e.pos], e.termbindings, e.bindings, func() error { + return e.next(iter, e.termbindings.Plug(elem)) + }) + }) + } + + return nil +} + +func (e evalTerm) get(plugged *ast.Term) (*ast.Term, *bindings) { + switch v := e.term.Value.(type) { + case ast.Set: + if v.IsGround() { + if v.Contains(plugged) { + return e.termbindings.apply(plugged) + } + } else { + var t *ast.Term + var b *bindings + stop := v.Until(func(elem *ast.Term) bool { + if e.termbindings.Plug(elem).Equal(plugged) { + t, b = e.termbindings.apply(plugged) + return true + } + return false + }) + if stop { + return t, b + } + } + case ast.Object: + if v.IsGround() { + term := v.Get(plugged) + if term != nil { + return e.termbindings.apply(term) + } + } else { + var t *ast.Term + var b *bindings + stop := v.Until(func(k, v *ast.Term) bool { + if e.termbindings.Plug(k).Equal(plugged) { + t, b = e.termbindings.apply(v) + return true + } + return false + }) + if stop { + return t, b + } + } + case ast.Array: + term := v.Get(plugged) + if term != nil { + return e.termbindings.apply(term) + } + } + return nil, nil +} + +func (e evalTerm) save(iter unifyIterator) error { + + suffix := e.ref[e.pos:] + ref := make(ast.Ref, len(suffix)+1) + ref[0] = e.term + + for i := 0; i < len(suffix); i++ { + ref[i+1] = suffix[i] + } + + return e.e.biunify(ast.NewTerm(ref), e.rterm, e.termbindings, e.rbindings, iter) +} + +func applyCopyPropagation(p *copypropagation.CopyPropagator, instr *Instrumentation, body ast.Body) ast.Body { + instr.startTimer(partialOpCopyPropagation) + result := p.Apply(body) + instr.stopTimer(partialOpCopyPropagation) + return result +} + +func nonGroundKeys(a ast.Object) bool { + return a.Until(func(k, _ *ast.Term) bool { + return !k.IsGround() + }) +} + +func plugKeys(a ast.Object, b *bindings) ast.Object { + plugged, _ := a.Map(func(k, v *ast.Term) (*ast.Term, *ast.Term, error) { + return b.Plug(k), v, nil + }) + return plugged +} + +func plugSlice(xs []*ast.Term, b *bindings) []*ast.Term { + cpy := make([]*ast.Term, len(xs)) + for i := range cpy { + cpy[i] = b.Plug(xs[i]) + } + return cpy +} + +func canInlineNegation(safe ast.VarSet, queries []ast.Body) bool { + + size := 1 + + for _, query := range queries { + size *= len(query) + for _, expr := range query { + if !expr.Negated { + // Positive expressions containing variables cannot be trivially negated + // because they become unsafe (e.g., "x = 1" negated is "not x = 1" making x + // unsafe.) We check if the vars in the expr are already safe. + vis := ast.NewVarVisitor().WithParams(ast.VarVisitorParams{ + SkipRefCallHead: true, + SkipClosures: true, + }) + vis.Walk(expr) + unsafe := vis.Vars().Diff(safe).Diff(ast.ReservedVars) + if len(unsafe) > 0 { + return false + } + } + } + } + + // NOTE(tsandall): this limit is arbitrary–it's only in place to prevent the + // partial evaluation result from blowing up. In the future, we could make this + // configurable or do something more clever. + if size > 16 { + return false + } + + return true +} + +func complementedCartesianProduct(queries []ast.Body, idx int, curr ast.Body, iter func(ast.Body) error) error { + if idx == len(queries) { + return iter(curr) + } + for _, expr := range queries[idx] { + curr = append(curr, expr.Complement()) + if err := complementedCartesianProduct(queries, idx+1, curr, iter); err != nil { + return err + } + curr = curr[:len(curr)-1] + } + return nil +} + +func isInputRef(term *ast.Term) bool { + if ref, ok := term.Value.(ast.Ref); ok { + if ref.HasPrefix(ast.InputRootRef) { + return true + } + } + return false +} + +func isDataRef(term *ast.Term) bool { + if ref, ok := term.Value.(ast.Ref); ok { + if ref.HasPrefix(ast.DefaultRootRef) { + return true + } + } + return false +} + +func merge(a, b ast.Value) (ast.Value, bool) { + aObj, ok1 := a.(ast.Object) + bObj, ok2 := b.(ast.Object) + + if ok1 && ok2 { + return mergeObjects(aObj, bObj) + } + return nil, false +} + +// mergeObjects returns a new Object containing the non-overlapping keys of +// the objA and objB. If there are overlapping keys between objA and objB, +// the values of associated with the keys are merged. Only +// objects can be merged with other objects. If the values cannot be merged, +// objB value will be overwritten by objA value. +func mergeObjects(objA, objB ast.Object) (result ast.Object, ok bool) { + result = ast.NewObject() + stop := objA.Until(func(k, v *ast.Term) bool { + if v2 := objB.Get(k); v2 == nil { + result.Insert(k, v) + } else { + obj1, ok1 := v.Value.(ast.Object) + obj2, ok2 := v2.Value.(ast.Object) + + if !ok1 || !ok2 { + result.Insert(k, v) + return false + } + obj3, ok := mergeObjects(obj1, obj2) + if !ok { + return true + } + result.Insert(k, ast.NewTerm(obj3)) + } + return false + }) + if stop { + return nil, false + } + objB.Foreach(func(k, v *ast.Term) { + if v2 := objA.Get(k); v2 == nil { + result.Insert(k, v) + } + }) + return result, true +} + +func anyRefSetContainsPrefix(s [][]ast.Ref, prefix ast.Ref) bool { + for _, refs := range s { + for _, ref := range refs { + if ref.HasPrefix(prefix) { + return true + } + } + } + return false +} + +func refContainsNonScalar(ref ast.Ref) bool { + for _, term := range ref[1:] { + if !ast.IsScalar(term.Value) { + return true + } + } + return false +} diff --git a/vendor/github.com/open-policy-agent/opa/topdown/glob.go b/vendor/github.com/open-policy-agent/opa/topdown/glob.go new file mode 100644 index 000000000..98052a0c6 --- /dev/null +++ b/vendor/github.com/open-policy-agent/opa/topdown/glob.go @@ -0,0 +1,65 @@ +package topdown + +import ( + "fmt" + "sync" + + "github.com/gobwas/glob" + + "github.com/open-policy-agent/opa/ast" + "github.com/open-policy-agent/opa/topdown/builtins" +) + +var globCacheLock = sync.Mutex{} +var globCache map[string]glob.Glob + +func builtinGlobMatch(a, b, c ast.Value) (ast.Value, error) { + pattern, err := builtins.StringOperand(a, 1) + if err != nil { + return nil, err + } + + delimiters, err := builtins.RuneSliceOperand(b, 2) + if err != nil { + return nil, err + } + + if len(delimiters) == 0 { + delimiters = []rune{'.'} + } + + match, err := builtins.StringOperand(c, 3) + if err != nil { + return nil, err + } + + id := fmt.Sprintf("%s-%v", pattern, delimiters) + + globCacheLock.Lock() + defer globCacheLock.Unlock() + p, ok := globCache[id] + if !ok { + var err error + if p, err = glob.Compile(string(pattern), delimiters...); err != nil { + return nil, err + } + globCache[id] = p + } + + return ast.Boolean(p.Match(string(match))), nil +} + +func builtinGlobQuoteMeta(a ast.Value) (ast.Value, error) { + pattern, err := builtins.StringOperand(a, 1) + if err != nil { + return nil, err + } + + return ast.String(glob.QuoteMeta(string(pattern))), nil +} + +func init() { + globCache = map[string]glob.Glob{} + RegisterFunctionalBuiltin3(ast.GlobMatch.Name, builtinGlobMatch) + RegisterFunctionalBuiltin1(ast.GlobQuoteMeta.Name, builtinGlobQuoteMeta) +} diff --git a/vendor/github.com/open-policy-agent/opa/topdown/http.go b/vendor/github.com/open-policy-agent/opa/topdown/http.go new file mode 100644 index 000000000..7c6cae2ad --- /dev/null +++ b/vendor/github.com/open-policy-agent/opa/topdown/http.go @@ -0,0 +1,466 @@ +// Copyright 2018 The OPA Authors. All rights reserved. +// Use of this source code is governed by an Apache2 +// license that can be found in the LICENSE file. + +package topdown + +import ( + "bytes" + "crypto/tls" + "encoding/json" + "fmt" + "io" + "io/ioutil" + "net/url" + "strconv" + + "github.com/open-policy-agent/opa/internal/version" + + "net/http" + "os" + "strings" + "time" + + "github.com/open-policy-agent/opa/ast" + "github.com/open-policy-agent/opa/topdown/builtins" +) + +const defaultHTTPRequestTimeoutEnv = "HTTP_SEND_TIMEOUT" + +var defaultHTTPRequestTimeout = time.Second * 5 + +var allowedKeyNames = [...]string{ + "method", + "url", + "body", + "enable_redirect", + "force_json_decode", + "headers", + "raw_body", + "tls_use_system_certs", + "tls_ca_cert_file", + "tls_ca_cert_env_variable", + "tls_client_cert_env_variable", + "tls_client_key_env_variable", + "tls_client_cert_file", + "tls_client_key_file", + "tls_insecure_skip_verify", + "timeout", +} +var allowedKeys = ast.NewSet() + +var requiredKeys = ast.NewSet(ast.StringTerm("method"), ast.StringTerm("url")) + +type httpSendKey string + +// httpSendBuiltinCacheKey is the key in the builtin context cache that +// points to the http.send() specific cache resides at. +const httpSendBuiltinCacheKey httpSendKey = "HTTP_SEND_CACHE_KEY" + +func builtinHTTPSend(bctx BuiltinContext, args []*ast.Term, iter func(*ast.Term) error) error { + + req, err := validateHTTPRequestOperand(args[0], 1) + if err != nil { + return handleBuiltinErr(ast.HTTPSend.Name, bctx.Location, err) + } + + // check if cache already has a response for this query + resp := checkHTTPSendCache(bctx, req) + if resp == nil { + var err error + resp, err = executeHTTPRequest(bctx, req) + if err != nil { + return handleHTTPSendErr(bctx, err) + } + + // add result to cache + insertIntoHTTPSendCache(bctx, req, resp) + } + + return iter(ast.NewTerm(resp)) +} + +func init() { + createAllowedKeys() + initDefaults() + RegisterBuiltinFunc(ast.HTTPSend.Name, builtinHTTPSend) +} + +func handleHTTPSendErr(bctx BuiltinContext, err error) error { + // Return HTTP client timeout errors in a generic error message to avoid confusion about what happened. + // Do not do this if the builtin context was cancelled and is what caused the request to stop. + if urlErr, ok := err.(*url.Error); ok && urlErr.Timeout() && bctx.Context.Err() == nil { + err = fmt.Errorf("%s %s: request timed out", urlErr.Op, urlErr.URL) + } + return handleBuiltinErr(ast.HTTPSend.Name, bctx.Location, err) +} + +func initDefaults() { + timeoutDuration := os.Getenv(defaultHTTPRequestTimeoutEnv) + if timeoutDuration != "" { + var err error + defaultHTTPRequestTimeout, err = time.ParseDuration(timeoutDuration) + if err != nil { + // If it is set to something not valid don't let the process continue in a state + // that will almost definitely give unexpected results by having it set at 0 + // which means no timeout.. + // This environment variable isn't considered part of the public API. + // TODO(patrick-east): Remove the environment variable + panic(fmt.Sprintf("invalid value for HTTP_SEND_TIMEOUT: %s", err)) + } + } +} + +func validateHTTPRequestOperand(term *ast.Term, pos int) (ast.Object, error) { + + obj, err := builtins.ObjectOperand(term.Value, pos) + if err != nil { + return nil, err + } + + requestKeys := ast.NewSet(obj.Keys()...) + + invalidKeys := requestKeys.Diff(allowedKeys) + if invalidKeys.Len() != 0 { + return nil, builtins.NewOperandErr(pos, "invalid request parameters(s): %v", invalidKeys) + } + + missingKeys := requiredKeys.Diff(requestKeys) + if missingKeys.Len() != 0 { + return nil, builtins.NewOperandErr(pos, "missing required request parameters(s): %v", missingKeys) + } + + return obj, nil + +} + +// Adds custom headers to a new HTTP request. +func addHeaders(req *http.Request, headers map[string]interface{}) (bool, error) { + for k, v := range headers { + // Type assertion + header, ok := v.(string) + if !ok { + return false, fmt.Errorf("invalid type for headers value %q", v) + } + + // If the Host header is given, bump that up to + // the request. Otherwise, just collect it in the + // headers. + k := http.CanonicalHeaderKey(k) + switch k { + case "Host": + req.Host = header + default: + req.Header.Add(k, header) + } + } + + return true, nil +} + +func executeHTTPRequest(bctx BuiltinContext, obj ast.Object) (ast.Value, error) { + var url string + var method string + var tlsCaCertEnvVar []byte + var tlsCaCertFile string + var tlsClientKeyEnvVar []byte + var tlsClientCertEnvVar []byte + var tlsClientCertFile string + var tlsClientKeyFile string + var body *bytes.Buffer + var rawBody *bytes.Buffer + var enableRedirect bool + var forceJSONDecode bool + var tlsUseSystemCerts bool + var tlsConfig tls.Config + var clientCerts []tls.Certificate + var customHeaders map[string]interface{} + var tlsInsecureSkipVerify bool + var timeout = defaultHTTPRequestTimeout + + for _, val := range obj.Keys() { + key, err := ast.JSON(val.Value) + if err != nil { + return nil, err + } + key = key.(string) + + switch key { + case "method": + method = obj.Get(val).String() + method = strings.ToUpper(strings.Trim(method, "\"")) + case "url": + url = obj.Get(val).String() + url = strings.Trim(url, "\"") + case "enable_redirect": + enableRedirect, err = strconv.ParseBool(obj.Get(val).String()) + if err != nil { + return nil, err + } + case "force_json_decode": + forceJSONDecode, err = strconv.ParseBool(obj.Get(val).String()) + if err != nil { + return nil, err + } + case "body": + bodyVal := obj.Get(val).Value + bodyValInterface, err := ast.JSON(bodyVal) + if err != nil { + return nil, err + } + + bodyValBytes, err := json.Marshal(bodyValInterface) + if err != nil { + return nil, err + } + body = bytes.NewBuffer(bodyValBytes) + case "raw_body": + s, ok := obj.Get(val).Value.(ast.String) + if !ok { + return nil, fmt.Errorf("raw_body must be a string") + } + rawBody = bytes.NewBuffer([]byte(s)) + case "tls_use_system_certs": + tlsUseSystemCerts, err = strconv.ParseBool(obj.Get(val).String()) + if err != nil { + return nil, err + } + case "tls_ca_cert_file": + tlsCaCertFile = obj.Get(val).String() + tlsCaCertFile = strings.Trim(tlsCaCertFile, "\"") + case "tls_ca_cert_env_variable": + caCertEnv := obj.Get(val).String() + caCertEnv = strings.Trim(caCertEnv, "\"") + tlsCaCertEnvVar = []byte(os.Getenv(caCertEnv)) + case "tls_client_cert_env_variable": + clientCertEnv := obj.Get(val).String() + clientCertEnv = strings.Trim(clientCertEnv, "\"") + tlsClientCertEnvVar = []byte(os.Getenv(clientCertEnv)) + case "tls_client_key_env_variable": + clientKeyEnv := obj.Get(val).String() + clientKeyEnv = strings.Trim(clientKeyEnv, "\"") + tlsClientKeyEnvVar = []byte(os.Getenv(clientKeyEnv)) + case "tls_client_cert_file": + tlsClientCertFile = obj.Get(val).String() + tlsClientCertFile = strings.Trim(tlsClientCertFile, "\"") + case "tls_client_key_file": + tlsClientKeyFile = obj.Get(val).String() + tlsClientKeyFile = strings.Trim(tlsClientKeyFile, "\"") + case "headers": + headersVal := obj.Get(val).Value + headersValInterface, err := ast.JSON(headersVal) + if err != nil { + return nil, err + } + var ok bool + customHeaders, ok = headersValInterface.(map[string]interface{}) + if !ok { + return nil, fmt.Errorf("invalid type for headers key") + } + case "tls_insecure_skip_verify": + tlsInsecureSkipVerify, err = strconv.ParseBool(obj.Get(val).String()) + if err != nil { + return nil, err + } + case "timeout": + timeout, err = parseTimeout(obj.Get(val).Value) + if err != nil { + return nil, err + } + default: + return nil, fmt.Errorf("invalid parameter %q", key) + } + } + + client := &http.Client{ + Timeout: timeout, + } + + if tlsInsecureSkipVerify { + client.Transport = &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: tlsInsecureSkipVerify}, + } + } + if tlsClientCertFile != "" && tlsClientKeyFile != "" { + clientCertFromFile, err := tls.LoadX509KeyPair(tlsClientCertFile, tlsClientKeyFile) + if err != nil { + return nil, err + } + clientCerts = append(clientCerts, clientCertFromFile) + } + + if len(tlsClientCertEnvVar) > 0 && len(tlsClientKeyEnvVar) > 0 { + clientCertFromEnv, err := tls.X509KeyPair(tlsClientCertEnvVar, tlsClientKeyEnvVar) + if err != nil { + return nil, err + } + clientCerts = append(clientCerts, clientCertFromEnv) + } + + isTLS := false + if len(clientCerts) > 0 { + isTLS = true + tlsConfig.Certificates = append(tlsConfig.Certificates, clientCerts...) + } + + if tlsUseSystemCerts || len(tlsCaCertFile) > 0 || len(tlsCaCertEnvVar) > 0 { + isTLS = true + connRootCAs, err := createRootCAs(tlsCaCertFile, tlsCaCertEnvVar, tlsUseSystemCerts) + if err != nil { + return nil, err + } + tlsConfig.RootCAs = connRootCAs + } + + if isTLS { + client.Transport = &http.Transport{ + TLSClientConfig: &tlsConfig, + } + } + + // check if redirects are enabled + if !enableRedirect { + client.CheckRedirect = func(*http.Request, []*http.Request) error { + return http.ErrUseLastResponse + } + } + + if rawBody != nil { + body = rawBody + } else if body == nil { + body = bytes.NewBufferString("") + } + + // create the http request, use the builtin context's context to ensure + // the request is cancelled if evaluation is cancelled. + req, err := http.NewRequest(method, url, body) + if err != nil { + return nil, err + } + req = req.WithContext(bctx.Context) + + // Add custom headers + if len(customHeaders) != 0 { + if ok, err := addHeaders(req, customHeaders); !ok { + return nil, err + } + // Don't overwrite or append to one that was set in the custom headers + if _, hasUA := customHeaders["User-Agent"]; !hasUA { + req.Header.Add("User-Agent", version.UserAgent) + } + } + + // execute the http request + resp, err := client.Do(req) + if err != nil { + return nil, err + } + + defer resp.Body.Close() + + // format the http result + var resultBody interface{} + var resultRawBody []byte + + var buf bytes.Buffer + tee := io.TeeReader(resp.Body, &buf) + resultRawBody, err = ioutil.ReadAll(tee) + if err != nil { + return nil, err + } + + // If the response body cannot be JSON decoded, + // an error will not be returned. Instead the "body" field + // in the result will be null. + if isContentTypeJSON(resp.Header) || forceJSONDecode { + json.NewDecoder(&buf).Decode(&resultBody) + } + + result := make(map[string]interface{}) + result["status"] = resp.Status + result["status_code"] = resp.StatusCode + result["body"] = resultBody + result["raw_body"] = string(resultRawBody) + + resultObj, err := ast.InterfaceToValue(result) + if err != nil { + return nil, err + } + + return resultObj, nil +} + +func isContentTypeJSON(header http.Header) bool { + return strings.Contains(header.Get("Content-Type"), "application/json") +} + +// In the BuiltinContext cache we only store a single entry that points to +// our ValueMap which is the "real" http.send() cache. +func getHTTPSendCache(bctx BuiltinContext) *ast.ValueMap { + raw, ok := bctx.Cache.Get(httpSendBuiltinCacheKey) + if !ok { + // Initialize if it isn't there + cache := ast.NewValueMap() + bctx.Cache.Put(httpSendBuiltinCacheKey, cache) + return cache + } + + cache, ok := raw.(*ast.ValueMap) + if !ok { + return nil + } + return cache +} + +// checkHTTPSendCache checks for the given key's value in the cache +func checkHTTPSendCache(bctx BuiltinContext, key ast.Object) ast.Value { + requestCache := getHTTPSendCache(bctx) + if requestCache == nil { + return nil + } + + return requestCache.Get(key) +} + +func insertIntoHTTPSendCache(bctx BuiltinContext, key ast.Object, value ast.Value) { + requestCache := getHTTPSendCache(bctx) + if requestCache == nil { + // Should never happen.. if it does just skip caching the value + return + } + requestCache.Put(key, value) +} + +func createAllowedKeys() { + for _, element := range allowedKeyNames { + allowedKeys.Add(ast.StringTerm(element)) + } +} + +func parseTimeout(timeoutVal ast.Value) (time.Duration, error) { + var timeout time.Duration + switch t := timeoutVal.(type) { + case ast.Number: + timeoutInt, ok := t.Int64() + if !ok { + return timeout, fmt.Errorf("invalid timeout number value %v, must be int64", timeoutVal) + } + return time.Duration(timeoutInt), nil + case ast.String: + // Support strings without a unit, treat them the same as just a number value (ns) + var err error + timeoutInt, err := strconv.ParseInt(string(t), 10, 64) + if err == nil { + return time.Duration(timeoutInt), nil + } + + // Try parsing it as a duration (requires a supported units suffix) + timeout, err = time.ParseDuration(string(t)) + if err != nil { + return timeout, fmt.Errorf("invalid timeout value %v: %s", timeoutVal, err) + } + return timeout, nil + default: + return timeout, builtins.NewOperandErr(1, "'timeout' must be one of {string, number} but got %s", ast.TypeName(t)) + } +} diff --git a/vendor/github.com/open-policy-agent/opa/topdown/input.go b/vendor/github.com/open-policy-agent/opa/topdown/input.go new file mode 100644 index 000000000..a74ef199f --- /dev/null +++ b/vendor/github.com/open-policy-agent/opa/topdown/input.go @@ -0,0 +1,74 @@ +// Copyright 2016 The OPA Authors. All rights reserved. +// Use of this source code is governed by an Apache2 +// license that can be found in the LICENSE file. + +package topdown + +import ( + "fmt" + + "github.com/open-policy-agent/opa/ast" +) + +var errConflictingDoc = fmt.Errorf("conflicting documents") +var errBadPath = fmt.Errorf("bad document path") + +func mergeTermWithValues(exist *ast.Term, pairs [][2]*ast.Term) (*ast.Term, error) { + + var result *ast.Term + + if exist != nil { + result = exist.Copy() + } + + for _, pair := range pairs { + + if err := ast.IsValidImportPath(pair[0].Value); err != nil { + return nil, errBadPath + } + + target := pair[0].Value.(ast.Ref) + + if len(target) == 1 { + result = pair[1] + } else if result == nil { + result = ast.NewTerm(makeTree(target[1:], pair[1])) + } else { + node := result + done := false + for i := 1; i < len(target)-1 && !done; i++ { + if child := node.Get(target[i]); child == nil { + obj, ok := node.Value.(ast.Object) + if !ok { + return nil, errConflictingDoc + } + obj.Insert(target[i], ast.NewTerm(makeTree(target[i+1:], pair[1]))) + done = true + } else { + node = child + } + } + if !done { + obj, ok := node.Value.(ast.Object) + if !ok { + return nil, errConflictingDoc + } + obj.Insert(target[len(target)-1], pair[1]) + } + } + } + + return result, nil +} + +// makeTree returns an object that represents a document where the value v is +// the leaf and elements in k represent intermediate objects. +func makeTree(k ast.Ref, v *ast.Term) ast.Object { + var obj ast.Object + for i := len(k) - 1; i >= 1; i-- { + obj = ast.NewObject(ast.Item(k[i], v)) + v = &ast.Term{Value: obj} + } + obj = ast.NewObject(ast.Item(k[0], v)) + return obj +} diff --git a/vendor/github.com/open-policy-agent/opa/topdown/instrumentation.go b/vendor/github.com/open-policy-agent/opa/topdown/instrumentation.go new file mode 100644 index 000000000..fa871cf90 --- /dev/null +++ b/vendor/github.com/open-policy-agent/opa/topdown/instrumentation.go @@ -0,0 +1,59 @@ +// Copyright 2018 The OPA Authors. All rights reserved. +// Use of this source code is governed by an Apache2 +// license that can be found in the LICENSE file. + +package topdown + +import "github.com/open-policy-agent/opa/metrics" + +const ( + evalOpPlug = "eval_op_plug" + evalOpResolve = "eval_op_resolve" + evalOpRuleIndex = "eval_op_rule_index" + evalOpBuiltinCall = "eval_op_builtin_call" + evalOpVirtualCacheHit = "eval_op_virtual_cache_hit" + evalOpVirtualCacheMiss = "eval_op_virtual_cache_miss" + evalOpBaseCacheHit = "eval_op_base_cache_hit" + evalOpBaseCacheMiss = "eval_op_base_cache_miss" + partialOpSaveUnify = "partial_op_save_unify" + partialOpSaveSetContains = "partial_op_save_set_contains" + partialOpSaveSetContainsRec = "partial_op_save_set_contains_rec" + partialOpCopyPropagation = "partial_op_copy_propagation" +) + +// Instrumentation implements helper functions to instrument query evaluation +// to diagnose performance issues. Instrumentation may be expensive in some +// cases, so it is disabled by default. +type Instrumentation struct { + m metrics.Metrics +} + +// NewInstrumentation returns a new Instrumentation object. Performance +// diagnostics recorded on this Instrumentation object will stored in m. +func NewInstrumentation(m metrics.Metrics) *Instrumentation { + return &Instrumentation{ + m: m, + } +} + +func (instr *Instrumentation) startTimer(name string) { + if instr == nil { + return + } + instr.m.Timer(name).Start() +} + +func (instr *Instrumentation) stopTimer(name string) { + if instr == nil { + return + } + delta := instr.m.Timer(name).Stop() + instr.m.Histogram(name).Update(delta) +} + +func (instr *Instrumentation) counterIncr(name string) { + if instr == nil { + return + } + instr.m.Counter(name).Incr() +} diff --git a/vendor/github.com/open-policy-agent/opa/topdown/internal/jwx/LICENSE b/vendor/github.com/open-policy-agent/opa/topdown/internal/jwx/LICENSE new file mode 100644 index 000000000..6369f4fcc --- /dev/null +++ b/vendor/github.com/open-policy-agent/opa/topdown/internal/jwx/LICENSE @@ -0,0 +1,21 @@ +The MIT License (MIT) + +Copyright (c) 2015 lestrrat + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/vendor/github.com/open-policy-agent/opa/topdown/internal/jwx/buffer/buffer.go b/vendor/github.com/open-policy-agent/opa/topdown/internal/jwx/buffer/buffer.go new file mode 100644 index 000000000..ca4ac419b --- /dev/null +++ b/vendor/github.com/open-policy-agent/opa/topdown/internal/jwx/buffer/buffer.go @@ -0,0 +1,113 @@ +// Package buffer provides a very thin wrapper around []byte buffer called +// `Buffer`, to provide functionalities that are often used within the jwx +// related packages +package buffer + +import ( + "encoding/base64" + "encoding/binary" + "encoding/json" + + "github.com/pkg/errors" +) + +// Buffer wraps `[]byte` and provides functions that are often used in +// the jwx related packages. One notable difference is that while +// encoding/json marshalls `[]byte` using base64.StdEncoding, this +// module uses base64.RawURLEncoding as mandated by the spec +type Buffer []byte + +// FromUint creates a `Buffer` from an unsigned int +func FromUint(v uint64) Buffer { + data := make([]byte, 8) + binary.BigEndian.PutUint64(data, v) + + i := 0 + for ; i < len(data); i++ { + if data[i] != 0x0 { + break + } + } + return Buffer(data[i:]) +} + +// FromBase64 constructs a new Buffer from a base64 encoded data +func FromBase64(v []byte) (Buffer, error) { + b := Buffer{} + if err := b.Base64Decode(v); err != nil { + return Buffer(nil), errors.Wrap(err, "failed to decode from base64") + } + + return b, nil +} + +// FromNData constructs a new Buffer from a "n:data" format +// (I made that name up) +func FromNData(v []byte) (Buffer, error) { + size := binary.BigEndian.Uint32(v) + buf := make([]byte, int(size)) + copy(buf, v[4:4+size]) + return Buffer(buf), nil +} + +// Bytes returns the raw bytes that comprises the Buffer +func (b Buffer) Bytes() []byte { + return []byte(b) +} + +// NData returns Datalen || Data, where Datalen is a 32 bit counter for +// the length of the following data, and Data is the octets that comprise +// the buffer data +func (b Buffer) NData() []byte { + buf := make([]byte, 4+b.Len()) + binary.BigEndian.PutUint32(buf, uint32(b.Len())) + + copy(buf[4:], b.Bytes()) + return buf +} + +// Len returns the number of bytes that the Buffer holds +func (b Buffer) Len() int { + return len(b) +} + +// Base64Encode encodes the contents of the Buffer using base64.RawURLEncoding +func (b Buffer) Base64Encode() ([]byte, error) { + enc := base64.RawURLEncoding + out := make([]byte, enc.EncodedLen(len(b))) + enc.Encode(out, b) + return out, nil +} + +// Base64Decode decodes the contents of the Buffer using base64.RawURLEncoding +func (b *Buffer) Base64Decode(v []byte) error { + enc := base64.RawURLEncoding + out := make([]byte, enc.DecodedLen(len(v))) + n, err := enc.Decode(out, v) + if err != nil { + return errors.Wrap(err, "failed to decode from base64") + } + out = out[:n] + *b = Buffer(out) + return nil +} + +// MarshalJSON marshals the buffer into JSON format after encoding the buffer +// with base64.RawURLEncoding +func (b Buffer) MarshalJSON() ([]byte, error) { + v, err := b.Base64Encode() + if err != nil { + return nil, errors.Wrap(err, "failed to encode to base64") + } + return json.Marshal(string(v)) +} + +// UnmarshalJSON unmarshals from a JSON string into a Buffer, after decoding it +// with base64.RawURLEncoding +func (b *Buffer) UnmarshalJSON(data []byte) error { + var x string + if err := json.Unmarshal(data, &x); err != nil { + return errors.Wrap(err, "failed to unmarshal JSON") + } + return b.Base64Decode([]byte(x)) +} diff --git a/vendor/github.com/open-policy-agent/opa/topdown/internal/jwx/jwa/elliptic.go b/vendor/github.com/open-policy-agent/opa/topdown/internal/jwx/jwa/elliptic.go new file mode 100644 index 000000000..b7e35dc70 --- /dev/null +++ b/vendor/github.com/open-policy-agent/opa/topdown/internal/jwx/jwa/elliptic.go @@ -0,0 +1,11 @@ +package jwa + +// EllipticCurveAlgorithm represents the algorithms used for EC keys +type EllipticCurveAlgorithm string + +// Supported values for EllipticCurveAlgorithm +const ( + P256 EllipticCurveAlgorithm = "P-256" + P384 EllipticCurveAlgorithm = "P-384" + P521 EllipticCurveAlgorithm = "P-521" +) diff --git a/vendor/github.com/open-policy-agent/opa/topdown/internal/jwx/jwa/key_type.go b/vendor/github.com/open-policy-agent/opa/topdown/internal/jwx/jwa/key_type.go new file mode 100644 index 000000000..076bd39ed --- /dev/null +++ b/vendor/github.com/open-policy-agent/opa/topdown/internal/jwx/jwa/key_type.go @@ -0,0 +1,67 @@ +package jwa + +import ( + "strconv" + + "github.com/pkg/errors" +) + +// KeyType represents the key type ("kty") that are supported +type KeyType string + +var keyTypeAlg = map[string]struct{}{"EC": {}, "oct": {}, "RSA": {}} + +// Supported values for KeyType +const ( + EC KeyType = "EC" // Elliptic Curve + InvalidKeyType KeyType = "" // Invalid KeyType + OctetSeq KeyType = "oct" // Octet sequence (used to represent symmetric keys) + RSA KeyType = "RSA" // RSA +) + +// Accept is used when conversion from values given by +// outside sources (such as JSON payloads) is required +func (keyType *KeyType) Accept(value interface{}) error { + var tmp KeyType + switch x := value.(type) { + case string: + tmp = KeyType(x) + case KeyType: + tmp = x + default: + return errors.Errorf(`invalid type for jwa.KeyType: %T`, value) + } + _, ok := keyTypeAlg[tmp.String()] + if !ok { + return errors.Errorf("Unknown Key Type algorithm") + } + + *keyType = tmp + return nil +} + +// String returns the string representation of a KeyType +func (keyType KeyType) String() string { + return string(keyType) +} + +// UnmarshalJSON unmarshals and checks data as KeyType Algorithm +func (keyType *KeyType) UnmarshalJSON(data []byte) error { + var quote byte = '"' + var quoted string + if data[0] == quote { + var err error + quoted, err = strconv.Unquote(string(data)) + if err != nil { + return errors.Wrap(err, "Failed to process signature algorithm") + } + } else { + quoted = string(data) + } + _, ok := keyTypeAlg[quoted] + if !ok { + return errors.Errorf("Unknown signature algorithm") + } + *keyType = KeyType(quoted) + return nil +} diff --git a/vendor/github.com/open-policy-agent/opa/topdown/internal/jwx/jwa/parameters.go b/vendor/github.com/open-policy-agent/opa/topdown/internal/jwx/jwa/parameters.go new file mode 100644 index 000000000..63c5a6462 --- /dev/null +++ b/vendor/github.com/open-policy-agent/opa/topdown/internal/jwx/jwa/parameters.go @@ -0,0 +1,29 @@ +package jwa + +import ( + "crypto/elliptic" + + "github.com/open-policy-agent/opa/topdown/internal/jwx/buffer" +) + +// EllipticCurve provides a indirect type to standard elliptic curve such that we can +// use it for unmarshal +type EllipticCurve struct { + elliptic.Curve +} + +// AlgorithmParameters provides a single structure suitable to unmarshaling any JWK +type AlgorithmParameters struct { + N buffer.Buffer `json:"n,omitempty"` + E buffer.Buffer `json:"e,omitempty"` + D buffer.Buffer `json:"d,omitempty"` + P buffer.Buffer `json:"p,omitempty"` + Q buffer.Buffer `json:"q,omitempty"` + Dp buffer.Buffer `json:"dp,omitempty"` + Dq buffer.Buffer `json:"dq,omitempty"` + Qi buffer.Buffer `json:"qi,omitempty"` + Crv EllipticCurveAlgorithm `json:"crv,omitempty"` + X buffer.Buffer `json:"x,omitempty"` + Y buffer.Buffer `json:"y,omitempty"` + K buffer.Buffer `json:"k,omitempty"` +} diff --git a/vendor/github.com/open-policy-agent/opa/topdown/internal/jwx/jwa/signature.go b/vendor/github.com/open-policy-agent/opa/topdown/internal/jwx/jwa/signature.go new file mode 100644 index 000000000..a0988ecab --- /dev/null +++ b/vendor/github.com/open-policy-agent/opa/topdown/internal/jwx/jwa/signature.go @@ -0,0 +1,76 @@ +package jwa + +import ( + "strconv" + + "github.com/pkg/errors" +) + +// SignatureAlgorithm represents the various signature algorithms as described in https://tools.ietf.org/html/rfc7518#section-3.1 +type SignatureAlgorithm string + +var signatureAlg = map[string]struct{}{"ES256": {}, "ES384": {}, "ES512": {}, "HS256": {}, "HS384": {}, "HS512": {}, "PS256": {}, "PS384": {}, "PS512": {}, "RS256": {}, "RS384": {}, "RS512": {}, "none": {}} + +// Supported values for SignatureAlgorithm +const ( + ES256 SignatureAlgorithm = "ES256" // ECDSA using P-256 and SHA-256 + ES384 SignatureAlgorithm = "ES384" // ECDSA using P-384 and SHA-384 + ES512 SignatureAlgorithm = "ES512" // ECDSA using P-521 and SHA-512 + HS256 SignatureAlgorithm = "HS256" // HMAC using SHA-256 + HS384 SignatureAlgorithm = "HS384" // HMAC using SHA-384 + HS512 SignatureAlgorithm = "HS512" // HMAC using SHA-512 + NoSignature SignatureAlgorithm = "none" + PS256 SignatureAlgorithm = "PS256" // RSASSA-PSS using SHA256 and MGF1-SHA256 + PS384 SignatureAlgorithm = "PS384" // RSASSA-PSS using SHA384 and MGF1-SHA384 + PS512 SignatureAlgorithm = "PS512" // RSASSA-PSS using SHA512 and MGF1-SHA512 + RS256 SignatureAlgorithm = "RS256" // RSASSA-PKCS-v1.5 using SHA-256 + RS384 SignatureAlgorithm = "RS384" // RSASSA-PKCS-v1.5 using SHA-384 + RS512 SignatureAlgorithm = "RS512" // RSASSA-PKCS-v1.5 using SHA-512 + NoValue SignatureAlgorithm = "" // No value is different from none +) + +// Accept is used when conversion from values given by +// outside sources (such as JSON payloads) is required +func (signature *SignatureAlgorithm) Accept(value interface{}) error { + var tmp SignatureAlgorithm + switch x := value.(type) { + case string: + tmp = SignatureAlgorithm(x) + case SignatureAlgorithm: + tmp = x + default: + return errors.Errorf(`invalid type for jwa.SignatureAlgorithm: %T`, value) + } + _, ok := signatureAlg[tmp.String()] + if !ok { + return errors.Errorf("Unknown signature algorithm") + } + *signature = tmp + return nil +} + +// String returns the string representation of a SignatureAlgorithm +func (signature SignatureAlgorithm) String() string { + return string(signature) +} + +// UnmarshalJSON unmarshals and checks data as Signature Algorithm +func (signature *SignatureAlgorithm) UnmarshalJSON(data []byte) error { + var quote byte = '"' + var quoted string + if data[0] == quote { + var err error + quoted, err = strconv.Unquote(string(data)) + if err != nil { + return errors.Wrap(err, "Failed to process signature algorithm") + } + } else { + quoted = string(data) + } + _, ok := signatureAlg[quoted] + if !ok { + return errors.Errorf("Unknown signature algorithm") + } + *signature = SignatureAlgorithm(quoted) + return nil +} diff --git a/vendor/github.com/open-policy-agent/opa/topdown/internal/jwx/jwk/ecdsa.go b/vendor/github.com/open-policy-agent/opa/topdown/internal/jwx/jwk/ecdsa.go new file mode 100644 index 000000000..7bff2bf8e --- /dev/null +++ b/vendor/github.com/open-policy-agent/opa/topdown/internal/jwx/jwk/ecdsa.go @@ -0,0 +1,120 @@ +package jwk + +import ( + "crypto/ecdsa" + "crypto/elliptic" + "math/big" + + "github.com/pkg/errors" + + "github.com/open-policy-agent/opa/topdown/internal/jwx/jwa" +) + +func newECDSAPublicKey(key *ecdsa.PublicKey) (*ECDSAPublicKey, error) { + + var hdr StandardHeaders + err := hdr.Set(KeyTypeKey, jwa.EC) + if err != nil { + return nil, errors.Wrapf(err, "Failed to set Key Type") + } + + return &ECDSAPublicKey{ + StandardHeaders: &hdr, + key: key, + }, nil +} + +func newECDSAPrivateKey(key *ecdsa.PrivateKey) (*ECDSAPrivateKey, error) { + + var hdr StandardHeaders + err := hdr.Set(KeyTypeKey, jwa.EC) + if err != nil { + return nil, errors.Wrapf(err, "Failed to set Key Type") + } + + return &ECDSAPrivateKey{ + StandardHeaders: &hdr, + key: key, + }, nil +} + +// Materialize returns the EC-DSA public key represented by this JWK +func (k ECDSAPublicKey) Materialize() (interface{}, error) { + return k.key, nil +} + +// Materialize returns the EC-DSA private key represented by this JWK +func (k ECDSAPrivateKey) Materialize() (interface{}, error) { + return k.key, nil +} + +// GenerateKey creates a ECDSAPublicKey from JWK format +func (k *ECDSAPublicKey) GenerateKey(keyJSON *RawKeyJSON) error { + + var x, y big.Int + + if keyJSON.X == nil || keyJSON.Y == nil || keyJSON.Crv == "" { + return errors.Errorf("Missing mandatory key parameters X, Y or Crv") + } + + x.SetBytes(keyJSON.X.Bytes()) + y.SetBytes(keyJSON.Y.Bytes()) + + var curve elliptic.Curve + switch keyJSON.Crv { + case jwa.P256: + curve = elliptic.P256() + case jwa.P384: + curve = elliptic.P384() + case jwa.P521: + curve = elliptic.P521() + default: + return errors.Errorf(`invalid curve name %s`, keyJSON.Crv) + } + + *k = ECDSAPublicKey{ + StandardHeaders: &keyJSON.StandardHeaders, + key: &ecdsa.PublicKey{ + Curve: curve, + X: &x, + Y: &y, + }, + } + return nil +} + +// GenerateKey creates a ECDSAPrivateKey from JWK format +func (k *ECDSAPrivateKey) GenerateKey(keyJSON *RawKeyJSON) error { + + if keyJSON.D == nil { + return errors.Errorf("Missing mandatory key parameter D") + } + eCDSAPublicKey := &ECDSAPublicKey{} + err := eCDSAPublicKey.GenerateKey(keyJSON) + if err != nil { + return errors.Wrap(err, `failed to generate public key`) + } + dBytes := keyJSON.D.Bytes() + // The length of this octet string MUST be ceiling(log-base-2(n)/8) + // octets (where n is the order of the curve). This is because the private + // key d must be in the interval [1, n-1] so the bitlength of d should be + // no larger than the bitlength of n-1. The easiest way to find the octet + // length is to take bitlength(n-1), add 7 to force a carry, and shift this + // bit sequence right by 3, which is essentially dividing by 8 and adding + // 1 if there is any remainder. Thus, the private key value d should be + // output to (bitlength(n-1)+7)>>3 octets. + n := eCDSAPublicKey.key.Params().N + octetLength := (new(big.Int).Sub(n, big.NewInt(1)).BitLen() + 7) >> 3 + if octetLength-len(dBytes) != 0 { + return errors.Errorf("Failed to generate private key. Incorrect D value") + } + privateKey := &ecdsa.PrivateKey{ + PublicKey: *eCDSAPublicKey.key, + D: (&big.Int{}).SetBytes(keyJSON.D.Bytes()), + } + + k.key = privateKey + k.StandardHeaders = &keyJSON.StandardHeaders + + return nil +} diff --git a/vendor/github.com/open-policy-agent/opa/topdown/internal/jwx/jwk/headers.go b/vendor/github.com/open-policy-agent/opa/topdown/internal/jwx/jwk/headers.go new file mode 100644 index 000000000..8f310a4c1 --- /dev/null +++ b/vendor/github.com/open-policy-agent/opa/topdown/internal/jwx/jwk/headers.go @@ -0,0 +1,178 @@ +package jwk + +import ( + "github.com/pkg/errors" + + "github.com/open-policy-agent/opa/topdown/internal/jwx/jwa" +) + +// Convenience constants for common JWK parameters +const ( + AlgorithmKey = "alg" + KeyIDKey = "kid" + KeyOpsKey = "key_ops" + KeyTypeKey = "kty" + KeyUsageKey = "use" + PrivateParamsKey = "privateParams" +) + +// Headers provides a common interface to all future possible headers +type Headers interface { + Get(string) (interface{}, bool) + Set(string, interface{}) error + Walk(func(string, interface{}) error) error + GetAlgorithm() jwa.SignatureAlgorithm + GetKeyID() string + GetKeyOps() KeyOperationList + GetKeyType() jwa.KeyType + GetKeyUsage() string + GetPrivateParams() map[string]interface{} +} + +// StandardHeaders stores the common JWK parameters +type StandardHeaders struct { + Algorithm *jwa.SignatureAlgorithm `json:"alg,omitempty"` // https://tools.ietf.org/html/rfc7517#section-4.4 + KeyID string `json:"kid,omitempty"` // https://tools.ietf.org/html/rfc7515#section-4.1.4 + KeyOps KeyOperationList `json:"key_ops,omitempty"` // https://tools.ietf.org/html/rfc7517#section-4.3 + KeyType jwa.KeyType `json:"kty,omitempty"` // https://tools.ietf.org/html/rfc7517#section-4.1 + KeyUsage string `json:"use,omitempty"` // https://tools.ietf.org/html/rfc7517#section-4.2 + PrivateParams map[string]interface{} `json:"privateParams,omitempty"` // https://tools.ietf.org/html/rfc7515#section-4.1.4 +} + +// GetAlgorithm is a convenience function to retrieve the corresponding value stored in the StandardHeaders +func (h *StandardHeaders) GetAlgorithm() jwa.SignatureAlgorithm { + if v := h.Algorithm; v != nil { + return *v + } + return jwa.NoValue +} + +// GetKeyID is a convenience function to retrieve the corresponding value stored in the StandardHeaders +func (h *StandardHeaders) GetKeyID() string { + return h.KeyID +} + +// GetKeyOps is a convenience function to retrieve the corresponding value stored in the StandardHeaders +func (h *StandardHeaders) GetKeyOps() KeyOperationList { + return h.KeyOps +} + +// GetKeyType is a convenience function to retrieve the corresponding value stored in the StandardHeaders +func (h *StandardHeaders) GetKeyType() jwa.KeyType { + return h.KeyType +} + +// GetKeyUsage is a convenience function to retrieve the corresponding value stored in the StandardHeaders +func (h *StandardHeaders) GetKeyUsage() string { + return h.KeyUsage +} + +// GetPrivateParams is a convenience function to retrieve the corresponding value stored in the StandardHeaders +func (h *StandardHeaders) GetPrivateParams() map[string]interface{} { + return h.PrivateParams +} + +// Get is a general getter function for JWK StandardHeaders structure +func (h *StandardHeaders) Get(name string) (interface{}, bool) { + switch name { + case AlgorithmKey: + alg := h.GetAlgorithm() + if alg != jwa.NoValue { + return alg, true + } + return nil, false + case KeyIDKey: + v := h.KeyID + if v == "" { + return nil, false + } + return v, true + case KeyOpsKey: + v := h.KeyOps + if v == nil { + return nil, false + } + return v, true + case KeyTypeKey: + v := h.KeyType + if v == jwa.InvalidKeyType { + return nil, false + } + return v, true + case KeyUsageKey: + v := h.KeyUsage + if v == "" { + return nil, false + } + return v, true + case PrivateParamsKey: + v := h.PrivateParams + if len(v) == 0 { + return nil, false + } + return v, true + default: + return nil, false + } +} + +// Set is a general getter function for JWK StandardHeaders structure +func (h *StandardHeaders) Set(name string, value interface{}) error { + switch name { + case AlgorithmKey: + var acceptor jwa.SignatureAlgorithm + if err := acceptor.Accept(value); err != nil { + return errors.Wrapf(err, `invalid value for %s key`, AlgorithmKey) + } + h.Algorithm = &acceptor + return nil + case KeyIDKey: + if v, ok := value.(string); ok { + h.KeyID = v + return nil + } + return errors.Errorf("invalid value for %s key: %T", KeyIDKey, value) + case KeyOpsKey: + if err := h.KeyOps.Accept(value); err != nil { + return errors.Wrapf(err, "invalid value for %s key", KeyOpsKey) + } + return nil + case KeyTypeKey: + if err := h.KeyType.Accept(value); err != nil { + return errors.Wrapf(err, "invalid value for %s key", KeyTypeKey) + } + return nil + case KeyUsageKey: + if v, ok := value.(string); ok { + h.KeyUsage = v + return nil + } + return errors.Errorf("invalid value for %s key: %T", KeyUsageKey, value) + case PrivateParamsKey: + if v, ok := value.(map[string]interface{}); ok { + h.PrivateParams = v + return nil + } + return errors.Errorf("invalid value for %s key: %T", PrivateParamsKey, value) + default: + return errors.Errorf(`invalid key: %s`, name) + } +} + +// Walk iterates over all JWK standard headers fields while applying a function to its value. +func (h StandardHeaders) Walk(f func(string, interface{}) error) error { + for _, key := range []string{AlgorithmKey, KeyIDKey, KeyOpsKey, KeyTypeKey, KeyUsageKey, PrivateParamsKey} { + if v, ok := h.Get(key); ok { + if err := f(key, v); err != nil { + return errors.Wrapf(err, `walk function returned error for %s`, key) + } + } + } + + for k, v := range h.PrivateParams { + if err := f(k, v); err != nil { + return errors.Wrapf(err, `walk function returned error for %s`, k) + } + } + return nil +} diff --git a/vendor/github.com/open-policy-agent/opa/topdown/internal/jwx/jwk/interface.go b/vendor/github.com/open-policy-agent/opa/topdown/internal/jwx/jwk/interface.go new file mode 100644 index 000000000..f718bec67 --- /dev/null +++ b/vendor/github.com/open-policy-agent/opa/topdown/internal/jwx/jwk/interface.go @@ -0,0 +1,70 @@ +package jwk + +import ( + "crypto/ecdsa" + "crypto/rsa" + + "github.com/open-policy-agent/opa/topdown/internal/jwx/jwa" +) + +// Set is a convenience struct to allow generating and parsing +// JWK sets as opposed to single JWKs +type Set struct { + Keys []Key `json:"keys"` +} + +// Key defines the minimal interface for each of the +// key types. Their use and implementation differ significantly +// between each key types, so you should use type assertions +// to perform more specific tasks with each key +type Key interface { + Headers + + // Materialize creates the corresponding key. For example, + // RSA types would create *rsa.PublicKey or *rsa.PrivateKey, + // EC types would create *ecdsa.PublicKey or *ecdsa.PrivateKey, + // and OctetSeq types create a []byte key. + Materialize() (interface{}, error) + GenerateKey(*RawKeyJSON) error +} + +// RawKeyJSON is generic type that represents any kind JWK +type RawKeyJSON struct { + StandardHeaders + jwa.AlgorithmParameters +} + +// RawKeySetJSON is generic type that represents a JWK Set +type RawKeySetJSON struct { + Keys []RawKeyJSON `json:"keys"` +} + +// RSAPublicKey is a type of JWK generated from RSA public keys +type RSAPublicKey struct { + *StandardHeaders + key *rsa.PublicKey +} + +// RSAPrivateKey is a type of JWK generated from RSA private keys +type RSAPrivateKey struct { + *StandardHeaders + key *rsa.PrivateKey +} + +// SymmetricKey is a type of JWK generated from symmetric keys +type SymmetricKey struct { + *StandardHeaders + key []byte +} + +// ECDSAPublicKey is a type of JWK generated from ECDSA public keys +type ECDSAPublicKey struct { + *StandardHeaders + key *ecdsa.PublicKey +} + +// ECDSAPrivateKey is a type of JWK generated from ECDH-ES private keys +type ECDSAPrivateKey struct { + *StandardHeaders + key *ecdsa.PrivateKey +} diff --git a/vendor/github.com/open-policy-agent/opa/topdown/internal/jwx/jwk/jwk.go b/vendor/github.com/open-policy-agent/opa/topdown/internal/jwx/jwk/jwk.go new file mode 100644 index 000000000..18835cbb3 --- /dev/null +++ b/vendor/github.com/open-policy-agent/opa/topdown/internal/jwx/jwk/jwk.go @@ -0,0 +1,150 @@ +// Package jwk implements JWK as described in https://tools.ietf.org/html/rfc7517 +package jwk + +import ( + "crypto/ecdsa" + "crypto/rsa" + "encoding/json" + + "github.com/pkg/errors" + + "github.com/open-policy-agent/opa/topdown/internal/jwx/jwa" +) + +// GetPublicKey returns the public key based on the private key type. +// For rsa key types *rsa.PublicKey is returned; for ecdsa key types *ecdsa.PublicKey; +// for byte slice (raw) keys, the key itself is returned. If the corresponding +// public key cannot be deduced, an error is returned +func GetPublicKey(key interface{}) (interface{}, error) { + if key == nil { + return nil, errors.New(`jwk.New requires a non-nil key`) + } + + switch v := key.(type) { + // Mental note: although Public() is defined in both types, + // you can not coalesce the clauses for rsa.PrivateKey and + // ecdsa.PrivateKey, as then `v` becomes interface{} + // b/c the compiler cannot deduce the exact type. + case *rsa.PrivateKey: + return v.Public(), nil + case *ecdsa.PrivateKey: + return v.Public(), nil + case []byte: + return v, nil + default: + return nil, errors.Errorf(`invalid key type %T`, key) + } +} + +// GetKeyTypeFromKey creates a jwk.Key from the given key. +func GetKeyTypeFromKey(key interface{}) jwa.KeyType { + + switch key.(type) { + case *rsa.PrivateKey, *rsa.PublicKey: + return jwa.RSA + case *ecdsa.PrivateKey, *ecdsa.PublicKey: + return jwa.EC + case []byte: + return jwa.OctetSeq + default: + return jwa.InvalidKeyType + } +} + +// New creates a jwk.Key from the given key. +func New(key interface{}) (Key, error) { + if key == nil { + return nil, errors.New(`jwk.New requires a non-nil key`) + } + + switch v := key.(type) { + case *rsa.PrivateKey: + return newRSAPrivateKey(v) + case *rsa.PublicKey: + return newRSAPublicKey(v) + case *ecdsa.PrivateKey: + return newECDSAPrivateKey(v) + case *ecdsa.PublicKey: + return newECDSAPublicKey(v) + case []byte: + return newSymmetricKey(v) + default: + return nil, errors.Errorf(`invalid key type %T`, key) + } +} + +func parse(jwkSrc string) (*Set, error) { + + var jwkKeySet Set + var jwkKey Key + rawKeySetJSON := &RawKeySetJSON{} + err := json.Unmarshal([]byte(jwkSrc), rawKeySetJSON) + if err != nil { + return nil, errors.Wrap(err, "Failed to unmarshal JWK Set") + } + if len(rawKeySetJSON.Keys) == 0 { + + // It might be a single key + rawKeyJSON := &RawKeyJSON{} + err := json.Unmarshal([]byte(jwkSrc), rawKeyJSON) + if err != nil { + return nil, errors.Wrap(err, "Failed to unmarshal JWK") + } + jwkKey, err = rawKeyJSON.GenerateKey() + if err != nil { + return nil, errors.Wrap(err, "Failed to generate key") + } + // Add to set + jwkKeySet.Keys = append(jwkKeySet.Keys, jwkKey) + } else { + for i := range rawKeySetJSON.Keys { + rawKeyJSON := rawKeySetJSON.Keys[i] + jwkKey, err = rawKeyJSON.GenerateKey() + if err != nil { + return nil, errors.Wrap(err, "Failed to generate key: %s") + } + jwkKeySet.Keys = append(jwkKeySet.Keys, jwkKey) + } + } + return &jwkKeySet, nil +} + +// ParseBytes parses JWK from the incoming byte buffer. +func ParseBytes(buf []byte) (*Set, error) { + return parse(string(buf[:])) +} + +// ParseString parses JWK from the incoming string. +func ParseString(s string) (*Set, error) { + return parse(s) +} + +// GenerateKey creates an internal representation of a key from a raw JWK JSON +func (r *RawKeyJSON) GenerateKey() (Key, error) { + + var key Key + + switch r.KeyType { + case jwa.RSA: + if r.D != nil { + key = &RSAPrivateKey{} + } else { + key = &RSAPublicKey{} + } + case jwa.EC: + if r.D != nil { + key = &ECDSAPrivateKey{} + } else { + key = &ECDSAPublicKey{} + } + case jwa.OctetSeq: + key = &SymmetricKey{} + default: + return nil, errors.Errorf(`Unrecognized key type`) + } + err := key.GenerateKey(r) + if err != nil { + return nil, errors.Wrap(err, "Failed to generate key from JWK") + } + return key, nil +} diff --git a/vendor/github.com/open-policy-agent/opa/topdown/internal/jwx/jwk/key_ops.go b/vendor/github.com/open-policy-agent/opa/topdown/internal/jwx/jwk/key_ops.go new file mode 100644 index 000000000..f9c7a4639 --- /dev/null +++ b/vendor/github.com/open-policy-agent/opa/topdown/internal/jwx/jwk/key_ops.go @@ -0,0 +1,68 @@ +package jwk + +import ( + "encoding/json" + "fmt" + + "github.com/pkg/errors" +) + +// KeyUsageType is used to denote what this key should be used for +type KeyUsageType string + +const ( + // ForSignature is the value used in the headers to indicate that + // this key should be used for signatures + ForSignature KeyUsageType = "sig" + // ForEncryption is the value used in the headers to indicate that + // this key should be used for encryptiong + ForEncryption KeyUsageType = "enc" +) + +// KeyOperation is used to denote the allowed operations for a Key +type KeyOperation string + +// KeyOperationList represents an slice of KeyOperation +type KeyOperationList []KeyOperation + +var keyOps = map[string]struct{}{"sign": {}, "verify": {}, "encrypt": {}, "decrypt": {}, "wrapKey": {}, "unwrapKey": {}, "deriveKey": {}, "deriveBits": {}} + +// KeyOperation constants +const ( + KeyOpSign KeyOperation = "sign" // (compute digital signature or MAC) + KeyOpVerify = "verify" // (verify digital signature or MAC) + KeyOpEncrypt = "encrypt" // (encrypt content) + KeyOpDecrypt = "decrypt" // (decrypt content and validate decryption, if applicable) + KeyOpWrapKey = "wrapKey" // (encrypt key) + KeyOpUnwrapKey = "unwrapKey" // (decrypt key and validate decryption, if applicable) + KeyOpDeriveKey = "deriveKey" // (derive key) + KeyOpDeriveBits = "deriveBits" // (derive bits not to be used as a key) +) + +// Accept determines if Key Operation is valid +func (keyOperationList *KeyOperationList) Accept(v interface{}) error { + switch x := v.(type) { + case KeyOperationList: + *keyOperationList = x + return nil + default: + return errors.Errorf(`invalid value %T`, v) + } +} + +// UnmarshalJSON unmarshals and checks data as KeyType Algorithm +func (keyOperationList *KeyOperationList) UnmarshalJSON(data []byte) error { + var tempKeyOperationList []string + err := json.Unmarshal(data, &tempKeyOperationList) + if err != nil { + return fmt.Errorf("invalid key operation") + } + for _, value := range tempKeyOperationList { + _, ok := keyOps[value] + if !ok { + return fmt.Errorf("unknown key operation") + } + *keyOperationList = append(*keyOperationList, KeyOperation(value)) + } + return nil +} diff --git a/vendor/github.com/open-policy-agent/opa/topdown/internal/jwx/jwk/rsa.go b/vendor/github.com/open-policy-agent/opa/topdown/internal/jwx/jwk/rsa.go new file mode 100644 index 000000000..e15e907d5 --- /dev/null +++ b/vendor/github.com/open-policy-agent/opa/topdown/internal/jwx/jwk/rsa.go @@ -0,0 +1,103 @@ +package jwk + +import ( + "crypto/rsa" + "math/big" + + "github.com/pkg/errors" + + "github.com/open-policy-agent/opa/topdown/internal/jwx/jwa" +) + +func newRSAPublicKey(key *rsa.PublicKey) (*RSAPublicKey, error) { + + var hdr StandardHeaders + err := hdr.Set(KeyTypeKey, jwa.RSA) + if err != nil { + return nil, errors.Wrapf(err, "Failed to set Key Type") + } + return &RSAPublicKey{ + StandardHeaders: &hdr, + key: key, + }, nil +} + +func newRSAPrivateKey(key *rsa.PrivateKey) (*RSAPrivateKey, error) { + + var hdr StandardHeaders + err := hdr.Set(KeyTypeKey, jwa.RSA) + if err != nil { + return nil, errors.Wrapf(err, "Failed to set Key Type") + } + return &RSAPrivateKey{ + StandardHeaders: &hdr, + key: key, + }, nil +} + +// Materialize returns the standard RSA Public Key representation stored in the internal representation +func (k *RSAPublicKey) Materialize() (interface{}, error) { + if k.key == nil { + return nil, errors.New(`key has no rsa.PublicKey associated with it`) + } + return k.key, nil +} + +// Materialize returns the standard RSA Private Key representation stored in the internal representation +func (k *RSAPrivateKey) Materialize() (interface{}, error) { + if k.key == nil { + return nil, errors.New(`key has no rsa.PrivateKey associated with it`) + } + return k.key, nil +} + +// GenerateKey creates a RSAPublicKey from a RawKeyJSON +func (k *RSAPublicKey) GenerateKey(keyJSON *RawKeyJSON) error { + + if keyJSON.N == nil || keyJSON.E == nil { + return errors.Errorf("Missing mandatory key parameters N or E") + } + rsaPublicKey := &rsa.PublicKey{ + N: (&big.Int{}).SetBytes(keyJSON.N.Bytes()), + E: int((&big.Int{}).SetBytes(keyJSON.E.Bytes()).Int64()), + } + k.key = rsaPublicKey + k.StandardHeaders = &keyJSON.StandardHeaders + return nil +} + +// GenerateKey creates a RSAPublicKey from a RawKeyJSON +func (k *RSAPrivateKey) GenerateKey(keyJSON *RawKeyJSON) error { + + rsaPublicKey := &RSAPublicKey{} + err := rsaPublicKey.GenerateKey(keyJSON) + if err != nil { + return errors.Wrap(err, "failed to generate public key") + } + + if keyJSON.D == nil || keyJSON.P == nil || keyJSON.Q == nil { + return errors.Errorf("Missing mandatory key parameters D, P or Q") + } + privateKey := &rsa.PrivateKey{ + PublicKey: *rsaPublicKey.key, + D: (&big.Int{}).SetBytes(keyJSON.D.Bytes()), + Primes: []*big.Int{ + (&big.Int{}).SetBytes(keyJSON.P.Bytes()), + (&big.Int{}).SetBytes(keyJSON.Q.Bytes()), + }, + } + + if keyJSON.Dp.Len() > 0 { + privateKey.Precomputed.Dp = (&big.Int{}).SetBytes(keyJSON.Dp.Bytes()) + } + if keyJSON.Dq.Len() > 0 { + privateKey.Precomputed.Dq = (&big.Int{}).SetBytes(keyJSON.Dq.Bytes()) + } + if keyJSON.Qi.Len() > 0 { + privateKey.Precomputed.Qinv = (&big.Int{}).SetBytes(keyJSON.Qi.Bytes()) + } + + k.key = privateKey + k.StandardHeaders = &keyJSON.StandardHeaders + return nil +} diff --git a/vendor/github.com/open-policy-agent/opa/topdown/internal/jwx/jwk/symmetric.go b/vendor/github.com/open-policy-agent/opa/topdown/internal/jwx/jwk/symmetric.go new file mode 100644 index 000000000..6d1da1e40 --- /dev/null +++ b/vendor/github.com/open-policy-agent/opa/topdown/internal/jwx/jwk/symmetric.go @@ -0,0 +1,41 @@ +package jwk + +import ( + "github.com/pkg/errors" + + "github.com/open-policy-agent/opa/topdown/internal/jwx/jwa" +) + +func newSymmetricKey(key []byte) (*SymmetricKey, error) { + var hdr StandardHeaders + + err := hdr.Set(KeyTypeKey, jwa.OctetSeq) + if err != nil { + return nil, errors.Wrapf(err, "Failed to set Key Type") + } + return &SymmetricKey{ + StandardHeaders: &hdr, + key: key, + }, nil +} + +// Materialize returns the octets for this symmetric key. +// Since this is a symmetric key, this just calls Octets +func (s SymmetricKey) Materialize() (interface{}, error) { + return s.Octets(), nil +} + +// Octets returns the octets in the key +func (s SymmetricKey) Octets() []byte { + return s.key +} + +// GenerateKey creates a Symmetric key from a RawKeyJSON +func (s *SymmetricKey) GenerateKey(keyJSON *RawKeyJSON) error { + + *s = SymmetricKey{ + StandardHeaders: &keyJSON.StandardHeaders, + key: keyJSON.K, + } + return nil +} diff --git a/vendor/github.com/open-policy-agent/opa/topdown/internal/jwx/jws/headers.go b/vendor/github.com/open-policy-agent/opa/topdown/internal/jwx/jws/headers.go new file mode 100644 index 000000000..fd6ffbe0e --- /dev/null +++ b/vendor/github.com/open-policy-agent/opa/topdown/internal/jwx/jws/headers.go @@ -0,0 +1,154 @@ +package jws + +import ( + "github.com/pkg/errors" + + "github.com/open-policy-agent/opa/topdown/internal/jwx/jwa" +) + +// Constants for JWS Common parameters +const ( + AlgorithmKey = "alg" + ContentTypeKey = "cty" + CriticalKey = "crit" + JWKKey = "jwk" + JWKSetURLKey = "jku" + KeyIDKey = "kid" + PrivateParamsKey = "privateParams" + TypeKey = "typ" +) + +// Headers provides a common interface for common header parameters +type Headers interface { + Get(string) (interface{}, bool) + Set(string, interface{}) error + GetAlgorithm() jwa.SignatureAlgorithm +} + +// StandardHeaders contains JWS common parameters. +type StandardHeaders struct { + Algorithm jwa.SignatureAlgorithm `json:"alg,omitempty"` // https://tools.ietf.org/html/rfc7515#section-4.1.1 + ContentType string `json:"cty,omitempty"` // https://tools.ietf.org/html/rfc7515#section-4.1.10 + Critical []string `json:"crit,omitempty"` // https://tools.ietf.org/html/rfc7515#section-4.1.11 + JWK string `json:"jwk,omitempty"` // https://tools.ietf.org/html/rfc7515#section-4.1.3 + JWKSetURL string `json:"jku,omitempty"` // https://tools.ietf.org/html/rfc7515#section-4.1.2 + KeyID string `json:"kid,omitempty"` // https://tools.ietf.org/html/rfc7515#section-4.1.4 + PrivateParams map[string]interface{} `json:"privateParams,omitempty"` // https://tools.ietf.org/html/rfc7515#section-4.1.9 + Type string `json:"typ,omitempty"` // https://tools.ietf.org/html/rfc7515#section-4.1.9 +} + +// GetAlgorithm returns algorithm +func (h *StandardHeaders) GetAlgorithm() jwa.SignatureAlgorithm { + return h.Algorithm +} + +// Get is a general getter function for StandardHeaders structure +func (h *StandardHeaders) Get(name string) (interface{}, bool) { + switch name { + case AlgorithmKey: + v := h.Algorithm + if v == "" { + return nil, false + } + return v, true + case ContentTypeKey: + v := h.ContentType + if v == "" { + return nil, false + } + return v, true + case CriticalKey: + v := h.Critical + if len(v) == 0 { + return nil, false + } + return v, true + case JWKKey: + v := h.JWK + if v == "" { + return nil, false + } + return v, true + case JWKSetURLKey: + v := h.JWKSetURL + if v == "" { + return nil, false + } + return v, true + case KeyIDKey: + v := h.KeyID + if v == "" { + return nil, false + } + return v, true + case PrivateParamsKey: + v := h.PrivateParams + if len(v) == 0 { + return nil, false + } + return v, true + case TypeKey: + v := h.Type + if v == "" { + return nil, false + } + return v, true + default: + return nil, false + } +} + +// Set is a general setter function for StandardHeaders structure +func (h *StandardHeaders) Set(name string, value interface{}) error { + switch name { + case AlgorithmKey: + if err := h.Algorithm.Accept(value); err != nil { + return errors.Wrapf(err, `invalid value for %s key`, AlgorithmKey) + } + return nil + case ContentTypeKey: + if v, ok := value.(string); ok { + h.ContentType = v + return nil + } + return errors.Errorf(`invalid value for %s key: %T`, ContentTypeKey, value) + case CriticalKey: + if v, ok := value.([]string); ok { + h.Critical = v + return nil + } + return errors.Errorf(`invalid value for %s key: %T`, CriticalKey, value) + case JWKKey: + if v, ok := value.(string); ok { + h.JWK = v + return nil + } + return errors.Errorf(`invalid value for %s key: %T`, JWKKey, value) + case JWKSetURLKey: + if v, ok := value.(string); ok { + h.JWKSetURL = v + return nil + } + return errors.Errorf(`invalid value for %s key: %T`, JWKSetURLKey, value) + case KeyIDKey: + if v, ok := value.(string); ok { + h.KeyID = v + return nil + } + return errors.Errorf(`invalid value for %s key: %T`, KeyIDKey, value) + case PrivateParamsKey: + if v, ok := value.(map[string]interface{}); ok { + h.PrivateParams = v + return nil + } + return errors.Errorf(`invalid value for %s key: %T`, PrivateParamsKey, value) + case TypeKey: + if v, ok := value.(string); ok { + h.Type = v + return nil + } + return errors.Errorf(`invalid value for %s key: %T`, TypeKey, value) + default: + return errors.Errorf(`invalid key: %s`, name) + } +} diff --git a/vendor/github.com/open-policy-agent/opa/topdown/internal/jwx/jws/interface.go b/vendor/github.com/open-policy-agent/opa/topdown/internal/jwx/jws/interface.go new file mode 100644 index 000000000..e647c8ac9 --- /dev/null +++ b/vendor/github.com/open-policy-agent/opa/topdown/internal/jwx/jws/interface.go @@ -0,0 +1,22 @@ +package jws + +// Message represents a full JWS encoded message. Flattened serialization +// is not supported as a struct, but rather it's represented as a +// Message struct with only one `Signature` element. +// +// Do not expect to use the Message object to verify or construct a +// signed payloads with. You should only use this when you want to actually +// want to programmatically view the contents for the full JWS Payload. +// +// To sign and verify, use the appropriate `SignWithOption()` nad `Verify()` functions +type Message struct { + Payload []byte `json:"payload"` + Signatures []*Signature `json:"signatures,omitempty"` +} + +// Signature represents the headers and signature of a JWS message +type Signature struct { + Headers Headers `json:"header,omitempty"` // Unprotected Headers + Protected Headers `json:"Protected,omitempty"` // Protected Headers + Signature []byte `json:"signature,omitempty"` // GetSignature +} diff --git a/vendor/github.com/open-policy-agent/opa/topdown/internal/jwx/jws/jws.go b/vendor/github.com/open-policy-agent/opa/topdown/internal/jwx/jws/jws.go new file mode 100644 index 000000000..34e18a499 --- /dev/null +++ b/vendor/github.com/open-policy-agent/opa/topdown/internal/jwx/jws/jws.go @@ -0,0 +1,210 @@ +// Package jws implements the digital Signature on JSON based data +// structures as described in https://tools.ietf.org/html/rfc7515 +// +// If you do not care about the details, the only things that you +// would need to use are the following functions: +// +// jws.SignWithOption(Payload, algorithm, key) +// jws.Verify(encodedjws, algorithm, key) +// +// To sign, simply use `jws.SignWithOption`. `Payload` is a []byte buffer that +// contains whatever data you want to sign. `alg` is one of the +// jwa.SignatureAlgorithm constants from package jwa. For RSA and +// ECDSA family of algorithms, you will need to prepare a private key. +// For HMAC family, you just need a []byte value. The `jws.SignWithOption` +// function will return the encoded JWS message on success. +// +// To verify, use `jws.Verify`. It will parse the `encodedjws` buffer +// and verify the result using `algorithm` and `key`. Upon successful +// verification, the original Payload is returned, so you can work on it. +package jws + +import ( + "bytes" + "encoding/base64" + "encoding/json" + "strings" + + "github.com/open-policy-agent/opa/topdown/internal/jwx/jwa" + "github.com/open-policy-agent/opa/topdown/internal/jwx/jwk" + "github.com/open-policy-agent/opa/topdown/internal/jwx/jws/sign" + "github.com/open-policy-agent/opa/topdown/internal/jwx/jws/verify" + + "github.com/pkg/errors" +) + +// SignLiteral generates a Signature for the given Payload and Headers, and serializes +// it in compact serialization format. In this format you may NOT use +// multiple signers. +// +func SignLiteral(payload []byte, alg jwa.SignatureAlgorithm, key interface{}, hdrBuf []byte) ([]byte, error) { + encodedHdr := base64.RawURLEncoding.EncodeToString(hdrBuf) + encodedPayload := base64.RawURLEncoding.EncodeToString(payload) + signingInput := strings.Join( + []string{ + encodedHdr, + encodedPayload, + }, ".", + ) + signer, err := sign.New(alg) + if err != nil { + return nil, errors.Wrap(err, `failed to create signer`) + } + signature, err := signer.Sign([]byte(signingInput), key) + if err != nil { + return nil, errors.Wrap(err, `failed to sign Payload`) + } + encodedSignature := base64.RawURLEncoding.EncodeToString(signature) + compactSerialization := strings.Join( + []string{ + signingInput, + encodedSignature, + }, ".", + ) + return []byte(compactSerialization), nil +} + +// SignWithOption generates a Signature for the given Payload, and serializes +// it in compact serialization format. In this format you may NOT use +// multiple signers. +// +// If you would like to pass custom Headers, use the WithHeaders option. +func SignWithOption(payload []byte, alg jwa.SignatureAlgorithm, key interface{}) ([]byte, error) { + var headers Headers = &StandardHeaders{} + + err := headers.Set(AlgorithmKey, alg) + if err != nil { + return nil, errors.Wrap(err, "Failed to set alg value") + } + + hdrBuf, err := json.Marshal(headers) + if err != nil { + return nil, errors.Wrap(err, `failed to marshal Headers`) + } + return SignLiteral(payload, alg, key, hdrBuf) +} + +// Verify checks if the given JWS message is verifiable using `alg` and `key`. +// If the verification is successful, `err` is nil, and the content of the +// Payload that was signed is returned. If you need more fine-grained +// control of the verification process, manually call `Parse`, generate a +// verifier, and call `Verify` on the parsed JWS message object. +func Verify(buf []byte, alg jwa.SignatureAlgorithm, key interface{}) (ret []byte, err error) { + + verifier, err := verify.New(alg) + if err != nil { + return nil, errors.Wrap(err, "failed to create verifier") + } + + buf = bytes.TrimSpace(buf) + if len(buf) == 0 { + return nil, errors.New(`attempt to verify empty buffer`) + } + + parts, err := SplitCompact(string(buf[:])) + if err != nil { + return nil, errors.Wrap(err, `failed extract from compact serialization format`) + } + + signingInput := strings.Join( + []string{ + parts[0], + parts[1], + }, ".", + ) + + decodedSignature, err := base64.RawURLEncoding.DecodeString(parts[2]) + if err != nil { + return nil, errors.Wrap(err, "Failed to decode signature") + } + if err := verifier.Verify([]byte(signingInput), decodedSignature, key); err != nil { + return nil, errors.Wrap(err, "Failed to verify message") + } + + if decodedPayload, err := base64.RawURLEncoding.DecodeString(parts[1]); err == nil { + return decodedPayload, nil + } + return nil, errors.Wrap(err, "Failed to decode Payload") +} + +// VerifyWithJWK verifies the JWS message using the specified JWK +func VerifyWithJWK(buf []byte, key jwk.Key) (payload []byte, err error) { + + keyVal, err := key.Materialize() + if err != nil { + return nil, errors.Wrap(err, "Failed to materialize key") + } + return Verify(buf, key.GetAlgorithm(), keyVal) +} + +// VerifyWithJWKSet verifies the JWS message using JWK key set. +// By default it will only pick up keys that have the "use" key +// set to either "sig" or "enc", but you can override it by +// providing a keyaccept function. +func VerifyWithJWKSet(buf []byte, keyset *jwk.Set) (payload []byte, err error) { + + for _, key := range keyset.Keys { + payload, err := VerifyWithJWK(buf, key) + if err == nil { + return payload, nil + } + } + return nil, errors.New("failed to verify with any of the keys") +} + +// ParseByte parses a JWS value serialized via compact serialization and provided as []byte. +func ParseByte(jwsCompact []byte) (m *Message, err error) { + return parseCompact(string(jwsCompact[:])) +} + +// ParseString parses a JWS value serialized via compact serialization and provided as string. +func ParseString(s string) (*Message, error) { + return parseCompact(s) +} + +// SplitCompact splits a JWT and returns its three parts +// separately: Protected Headers, Payload and Signature. +func SplitCompact(jwsCompact string) ([]string, error) { + + parts := strings.Split(jwsCompact, ".") + if len(parts) < 3 { + return nil, errors.New("Failed to split compact serialization") + } + return parts, nil +} + +// parseCompact parses a JWS value serialized via compact serialization. +func parseCompact(str string) (m *Message, err error) { + + var decodedHeader, decodedPayload, decodedSignature []byte + parts, err := SplitCompact(str) + if err != nil { + return nil, errors.Wrap(err, `invalid compact serialization format`) + } + + if decodedHeader, err = base64.RawURLEncoding.DecodeString(parts[0]); err != nil { + return nil, errors.Wrap(err, `failed to decode Headers`) + } + var hdr StandardHeaders + if err := json.Unmarshal(decodedHeader, &hdr); err != nil { + return nil, errors.Wrap(err, `failed to parse JOSE Headers`) + } + + if decodedPayload, err = base64.RawURLEncoding.DecodeString(parts[1]); err != nil { + return nil, errors.Wrap(err, `failed to decode Payload`) + } + + if len(parts) > 2 { + if decodedSignature, err = base64.RawURLEncoding.DecodeString(parts[2]); err != nil { + return nil, errors.Wrap(err, `failed to decode Signature`) + } + } + + var msg Message + msg.Payload = decodedPayload + msg.Signatures = append(msg.Signatures, &Signature{ + Protected: &hdr, + Signature: decodedSignature, + }) + return &msg, nil +} diff --git a/vendor/github.com/open-policy-agent/opa/topdown/internal/jwx/jws/message.go b/vendor/github.com/open-policy-agent/opa/topdown/internal/jwx/jws/message.go new file mode 100644 index 000000000..1366a3d7b --- /dev/null +++ b/vendor/github.com/open-policy-agent/opa/topdown/internal/jwx/jws/message.go @@ -0,0 +1,26 @@ +package jws + +// PublicHeaders returns the public headers in a JWS +func (s Signature) PublicHeaders() Headers { + return s.Headers +} + +// ProtectedHeaders returns the protected headers in a JWS +func (s Signature) ProtectedHeaders() Headers { + return s.Protected +} + +// GetSignature returns the signature in a JWS +func (s Signature) GetSignature() []byte { + return s.Signature +} + +// GetPayload returns the payload in a JWS +func (m Message) GetPayload() []byte { + return m.Payload +} + +// GetSignatures returns the all signatures in a JWS +func (m Message) GetSignatures() []*Signature { + return m.Signatures +} diff --git a/vendor/github.com/open-policy-agent/opa/topdown/internal/jwx/jws/sign/ecdsa.go b/vendor/github.com/open-policy-agent/opa/topdown/internal/jwx/jws/sign/ecdsa.go new file mode 100644 index 000000000..02fc9f022 --- /dev/null +++ b/vendor/github.com/open-policy-agent/opa/topdown/internal/jwx/jws/sign/ecdsa.go @@ -0,0 +1,84 @@ +package sign + +import ( + "crypto" + "crypto/ecdsa" + "crypto/rand" + + "github.com/open-policy-agent/opa/topdown/internal/jwx/jwa" + + "github.com/pkg/errors" +) + +var ecdsaSignFuncs = map[jwa.SignatureAlgorithm]ecdsaSignFunc{} + +func init() { + algs := map[jwa.SignatureAlgorithm]crypto.Hash{ + jwa.ES256: crypto.SHA256, + jwa.ES384: crypto.SHA384, + jwa.ES512: crypto.SHA512, + } + + for alg, h := range algs { + ecdsaSignFuncs[alg] = makeECDSASignFunc(h) + } +} + +func makeECDSASignFunc(hash crypto.Hash) ecdsaSignFunc { + return ecdsaSignFunc(func(payload []byte, key *ecdsa.PrivateKey) ([]byte, error) { + curveBits := key.Curve.Params().BitSize + keyBytes := curveBits / 8 + // Curve bits do not need to be a multiple of 8. + if curveBits%8 > 0 { + keyBytes++ + } + h := hash.New() + h.Write(payload) + r, s, err := ecdsa.Sign(rand.Reader, key, h.Sum(nil)) + if err != nil { + return nil, errors.Wrap(err, "failed to sign payload using ecdsa") + } + + rBytes := r.Bytes() + rBytesPadded := make([]byte, keyBytes) + copy(rBytesPadded[keyBytes-len(rBytes):], rBytes) + + sBytes := s.Bytes() + sBytesPadded := make([]byte, keyBytes) + copy(sBytesPadded[keyBytes-len(sBytes):], sBytes) + + out := append(rBytesPadded, sBytesPadded...) + return out, nil + }) +} + +func newECDSA(alg jwa.SignatureAlgorithm) (*ECDSASigner, error) { + signfn, ok := ecdsaSignFuncs[alg] + if !ok { + return nil, errors.Errorf(`unsupported algorithm while trying to create ECDSA signer: %s`, alg) + } + + return &ECDSASigner{ + alg: alg, + sign: signfn, + }, nil +} + +// Algorithm returns the signer algorithm +func (s ECDSASigner) Algorithm() jwa.SignatureAlgorithm { + return s.alg +} + +// Sign signs payload with a ECDSA private key +func (s ECDSASigner) Sign(payload []byte, key interface{}) ([]byte, error) { + if key == nil { + return nil, errors.New(`missing private key while signing payload`) + } + + privateKey, ok := key.(*ecdsa.PrivateKey) + if !ok { + return nil, errors.Errorf(`invalid key type %T. *ecdsa.PrivateKey is required`, key) + } + + return s.sign(payload, privateKey) +} diff --git a/vendor/github.com/open-policy-agent/opa/topdown/internal/jwx/jws/sign/hmac.go b/vendor/github.com/open-policy-agent/opa/topdown/internal/jwx/jws/sign/hmac.go new file mode 100644 index 000000000..f86283efb --- /dev/null +++ b/vendor/github.com/open-policy-agent/opa/topdown/internal/jwx/jws/sign/hmac.go @@ -0,0 +1,66 @@ +package sign + +import ( + "crypto/hmac" + "crypto/sha256" + "crypto/sha512" + "hash" + + "github.com/open-policy-agent/opa/topdown/internal/jwx/jwa" + + "github.com/pkg/errors" +) + +var hmacSignFuncs = map[jwa.SignatureAlgorithm]hmacSignFunc{} + +func init() { + algs := map[jwa.SignatureAlgorithm]func() hash.Hash{ + jwa.HS256: sha256.New, + jwa.HS384: sha512.New384, + jwa.HS512: sha512.New, + } + + for alg, h := range algs { + hmacSignFuncs[alg] = makeHMACSignFunc(h) + + } +} + +func newHMAC(alg jwa.SignatureAlgorithm) (*HMACSigner, error) { + signer, ok := hmacSignFuncs[alg] + if !ok { + return nil, errors.Errorf(`unsupported algorithm while trying to create HMAC signer: %s`, alg) + } + + return &HMACSigner{ + alg: alg, + sign: signer, + }, nil +} + +func makeHMACSignFunc(hfunc func() hash.Hash) hmacSignFunc { + return hmacSignFunc(func(payload []byte, key []byte) ([]byte, error) { + h := hmac.New(hfunc, key) + h.Write(payload) + return h.Sum(nil), nil + }) +} + +// Algorithm returns the signer algorithm +func (s HMACSigner) Algorithm() jwa.SignatureAlgorithm { + return s.alg +} + +// Sign signs payload with a Symmetric key +func (s HMACSigner) Sign(payload []byte, key interface{}) ([]byte, error) { + hmackey, ok := key.([]byte) + if !ok { + return nil, errors.Errorf(`invalid key type %T. []byte is required`, key) + } + + if len(hmackey) == 0 { + return nil, errors.New(`missing key while signing payload`) + } + + return s.sign(payload, hmackey) +} diff --git a/vendor/github.com/open-policy-agent/opa/topdown/internal/jwx/jws/sign/interface.go b/vendor/github.com/open-policy-agent/opa/topdown/internal/jwx/jws/sign/interface.go new file mode 100644 index 000000000..c79fd3e93 --- /dev/null +++ b/vendor/github.com/open-policy-agent/opa/topdown/internal/jwx/jws/sign/interface.go @@ -0,0 +1,45 @@ +package sign + +import ( + "crypto/ecdsa" + "crypto/rsa" + + "github.com/open-policy-agent/opa/topdown/internal/jwx/jwa" +) + +// Signer provides a common interface for supported alg signing methods +type Signer interface { + // Sign creates a signature for the given `payload`. + // `key` is the key used for signing the payload, and is usually + // the private key type associated with the signature method. For example, + // for `jwa.RSXXX` and `jwa.PSXXX` types, you need to pass the + // `*"crypto/rsa".PrivateKey` type. + // Check the documentation for each signer for details + Sign(payload []byte, key interface{}) ([]byte, error) + + Algorithm() jwa.SignatureAlgorithm +} + +type rsaSignFunc func([]byte, *rsa.PrivateKey) ([]byte, error) + +// RSASigner uses crypto/rsa to sign the payloads. +type RSASigner struct { + alg jwa.SignatureAlgorithm + sign rsaSignFunc +} + +type ecdsaSignFunc func([]byte, *ecdsa.PrivateKey) ([]byte, error) + +// ECDSASigner uses crypto/ecdsa to sign the payloads. +type ECDSASigner struct { + alg jwa.SignatureAlgorithm + sign ecdsaSignFunc +} + +type hmacSignFunc func([]byte, []byte) ([]byte, error) + +// HMACSigner uses crypto/hmac to sign the payloads. +type HMACSigner struct { + alg jwa.SignatureAlgorithm + sign hmacSignFunc +} diff --git a/vendor/github.com/open-policy-agent/opa/topdown/internal/jwx/jws/sign/rsa.go b/vendor/github.com/open-policy-agent/opa/topdown/internal/jwx/jws/sign/rsa.go new file mode 100644 index 000000000..d9cc13af9 --- /dev/null +++ b/vendor/github.com/open-policy-agent/opa/topdown/internal/jwx/jws/sign/rsa.go @@ -0,0 +1,97 @@ +package sign + +import ( + "crypto" + "crypto/rand" + "crypto/rsa" + + "github.com/open-policy-agent/opa/topdown/internal/jwx/jwa" + + "github.com/pkg/errors" +) + +var rsaSignFuncs = map[jwa.SignatureAlgorithm]rsaSignFunc{} + +func init() { + algs := map[jwa.SignatureAlgorithm]struct { + Hash crypto.Hash + SignFunc func(crypto.Hash) rsaSignFunc + }{ + jwa.RS256: { + Hash: crypto.SHA256, + SignFunc: makeSignPKCS1v15, + }, + jwa.RS384: { + Hash: crypto.SHA384, + SignFunc: makeSignPKCS1v15, + }, + jwa.RS512: { + Hash: crypto.SHA512, + SignFunc: makeSignPKCS1v15, + }, + jwa.PS256: { + Hash: crypto.SHA256, + SignFunc: makeSignPSS, + }, + jwa.PS384: { + Hash: crypto.SHA384, + SignFunc: makeSignPSS, + }, + jwa.PS512: { + Hash: crypto.SHA512, + SignFunc: makeSignPSS, + }, + } + + for alg, item := range algs { + rsaSignFuncs[alg] = item.SignFunc(item.Hash) + } +} + +func makeSignPKCS1v15(hash crypto.Hash) rsaSignFunc { + return rsaSignFunc(func(payload []byte, key *rsa.PrivateKey) ([]byte, error) { + h := hash.New() + h.Write(payload) + return rsa.SignPKCS1v15(rand.Reader, key, hash, h.Sum(nil)) + }) +} + +func makeSignPSS(hash crypto.Hash) rsaSignFunc { + return rsaSignFunc(func(payload []byte, key *rsa.PrivateKey) ([]byte, error) { + h := hash.New() + h.Write(payload) + return rsa.SignPSS(rand.Reader, key, hash, h.Sum(nil), &rsa.PSSOptions{ + SaltLength: rsa.PSSSaltLengthAuto, + }) + }) +} + +func newRSA(alg jwa.SignatureAlgorithm) (*RSASigner, error) { + signfn, ok := rsaSignFuncs[alg] + if !ok { + return nil, errors.Errorf(`unsupported algorithm while trying to create RSA signer: %s`, alg) + } + return &RSASigner{ + alg: alg, + sign: signfn, + }, nil +} + +// Algorithm returns the signer algorithm +func (s RSASigner) Algorithm() jwa.SignatureAlgorithm { + return s.alg +} + +// Sign creates a signature using crypto/rsa. key must be a non-nil instance of +// `*"crypto/rsa".PrivateKey`. +func (s RSASigner) Sign(payload []byte, key interface{}) ([]byte, error) { + if key == nil { + return nil, errors.New(`missing private key while signing payload`) + } + rsakey, ok := key.(*rsa.PrivateKey) + if !ok { + return nil, errors.Errorf(`invalid key type %T. *rsa.PrivateKey is required`, key) + } + + return s.sign(payload, rsakey) +} diff --git a/vendor/github.com/open-policy-agent/opa/topdown/internal/jwx/jws/sign/sign.go b/vendor/github.com/open-policy-agent/opa/topdown/internal/jwx/jws/sign/sign.go new file mode 100644 index 000000000..fd4b0645f --- /dev/null +++ b/vendor/github.com/open-policy-agent/opa/topdown/internal/jwx/jws/sign/sign.go @@ -0,0 +1,21 @@ +package sign + +import ( + "github.com/pkg/errors" + + "github.com/open-policy-agent/opa/topdown/internal/jwx/jwa" +) + +// New creates a signer that signs payloads using the given signature algorithm. +func New(alg jwa.SignatureAlgorithm) (Signer, error) { + switch alg { + case jwa.RS256, jwa.RS384, jwa.RS512, jwa.PS256, jwa.PS384, jwa.PS512: + return newRSA(alg) + case jwa.ES256, jwa.ES384, jwa.ES512: + return newECDSA(alg) + case jwa.HS256, jwa.HS384, jwa.HS512: + return newHMAC(alg) + default: + return nil, errors.Errorf(`unsupported signature algorithm %s`, alg) + } +} diff --git a/vendor/github.com/open-policy-agent/opa/topdown/internal/jwx/jws/verify/ecdsa.go b/vendor/github.com/open-policy-agent/opa/topdown/internal/jwx/jws/verify/ecdsa.go new file mode 100644 index 000000000..5adccda30 --- /dev/null +++ b/vendor/github.com/open-policy-agent/opa/topdown/internal/jwx/jws/verify/ecdsa.go @@ -0,0 +1,67 @@ +package verify + +import ( + "crypto" + "crypto/ecdsa" + "math/big" + + "github.com/pkg/errors" + + "github.com/open-policy-agent/opa/topdown/internal/jwx/jwa" +) + +var ecdsaVerifyFuncs = map[jwa.SignatureAlgorithm]ecdsaVerifyFunc{} + +func init() { + algs := map[jwa.SignatureAlgorithm]crypto.Hash{ + jwa.ES256: crypto.SHA256, + jwa.ES384: crypto.SHA384, + jwa.ES512: crypto.SHA512, + } + + for alg, h := range algs { + ecdsaVerifyFuncs[alg] = makeECDSAVerifyFunc(h) + } +} + +func makeECDSAVerifyFunc(hash crypto.Hash) ecdsaVerifyFunc { + return ecdsaVerifyFunc(func(payload []byte, signature []byte, key *ecdsa.PublicKey) error { + + r, s := &big.Int{}, &big.Int{} + n := len(signature) / 2 + r.SetBytes(signature[:n]) + s.SetBytes(signature[n:]) + + h := hash.New() + h.Write(payload) + + if !ecdsa.Verify(key, h.Sum(nil), r, s) { + return errors.New(`failed to verify signature using ecdsa`) + } + return nil + }) +} + +func newECDSA(alg jwa.SignatureAlgorithm) (*ECDSAVerifier, error) { + verifyfn, ok := ecdsaVerifyFuncs[alg] + if !ok { + return nil, errors.Errorf(`unsupported algorithm while trying to create ECDSA verifier: %s`, alg) + } + + return &ECDSAVerifier{ + verify: verifyfn, + }, nil +} + +// Verify checks whether the signature for a given input and key is correct +func (v ECDSAVerifier) Verify(payload []byte, signature []byte, key interface{}) error { + if key == nil { + return errors.New(`missing public key while verifying payload`) + } + ecdsakey, ok := key.(*ecdsa.PublicKey) + if !ok { + return errors.Errorf(`invalid key type %T. *ecdsa.PublicKey is required`, key) + } + + return v.verify(payload, signature, ecdsakey) +} diff --git a/vendor/github.com/open-policy-agent/opa/topdown/internal/jwx/jws/verify/hmac.go b/vendor/github.com/open-policy-agent/opa/topdown/internal/jwx/jws/verify/hmac.go new file mode 100644 index 000000000..e0b5e1981 --- /dev/null +++ b/vendor/github.com/open-policy-agent/opa/topdown/internal/jwx/jws/verify/hmac.go @@ -0,0 +1,33 @@ +package verify + +import ( + "crypto/hmac" + + "github.com/pkg/errors" + + "github.com/open-policy-agent/opa/topdown/internal/jwx/jwa" + "github.com/open-policy-agent/opa/topdown/internal/jwx/jws/sign" +) + +func newHMAC(alg jwa.SignatureAlgorithm) (*HMACVerifier, error) { + + s, err := sign.New(alg) + if err != nil { + return nil, errors.Wrap(err, `failed to generate HMAC signer`) + } + return &HMACVerifier{signer: s}, nil +} + +// Verify checks whether the signature for a given input and key is correct +func (v HMACVerifier) Verify(signingInput, signature []byte, key interface{}) (err error) { + + expected, err := v.signer.Sign(signingInput, key) + if err != nil { + return errors.Wrap(err, `failed to generated signature`) + } + + if !hmac.Equal(signature, expected) { + return errors.New(`failed to match hmac signature`) + } + return nil +} diff --git a/vendor/github.com/open-policy-agent/opa/topdown/internal/jwx/jws/verify/interface.go b/vendor/github.com/open-policy-agent/opa/topdown/internal/jwx/jws/verify/interface.go new file mode 100644 index 000000000..b72b7232a --- /dev/null +++ b/vendor/github.com/open-policy-agent/opa/topdown/internal/jwx/jws/verify/interface.go @@ -0,0 +1,39 @@ +package verify + +import ( + "crypto/ecdsa" + "crypto/rsa" + + "github.com/open-policy-agent/opa/topdown/internal/jwx/jws/sign" +) + +// Verifier provides a common interface for supported alg verification methods +type Verifier interface { + // Verify checks whether the payload and signature are valid for + // the given key. + // `key` is the key used for verifying the payload, and is usually + // the public key associated with the signature method. For example, + // for `jwa.RSXXX` and `jwa.PSXXX` types, you need to pass the + // `*"crypto/rsa".PublicKey` type. + // Check the documentation for each verifier for details + Verify(payload []byte, signature []byte, key interface{}) error +} + +type rsaVerifyFunc func([]byte, []byte, *rsa.PublicKey) error + +// RSAVerifier implements the Verifier interface +type RSAVerifier struct { + verify rsaVerifyFunc +} + +type ecdsaVerifyFunc func([]byte, []byte, *ecdsa.PublicKey) error + +// ECDSAVerifier implements the Verifier interface +type ECDSAVerifier struct { + verify ecdsaVerifyFunc +} + +// HMACVerifier implements the Verifier interface +type HMACVerifier struct { + signer sign.Signer +} diff --git a/vendor/github.com/open-policy-agent/opa/topdown/internal/jwx/jws/verify/rsa.go b/vendor/github.com/open-policy-agent/opa/topdown/internal/jwx/jws/verify/rsa.go new file mode 100644 index 000000000..26f341d12 --- /dev/null +++ b/vendor/github.com/open-policy-agent/opa/topdown/internal/jwx/jws/verify/rsa.go @@ -0,0 +1,88 @@ +package verify + +import ( + "crypto" + "crypto/rsa" + + "github.com/open-policy-agent/opa/topdown/internal/jwx/jwa" + + "github.com/pkg/errors" +) + +var rsaVerifyFuncs = map[jwa.SignatureAlgorithm]rsaVerifyFunc{} + +func init() { + algs := map[jwa.SignatureAlgorithm]struct { + Hash crypto.Hash + VerifyFunc func(crypto.Hash) rsaVerifyFunc + }{ + jwa.RS256: { + Hash: crypto.SHA256, + VerifyFunc: makeVerifyPKCS1v15, + }, + jwa.RS384: { + Hash: crypto.SHA384, + VerifyFunc: makeVerifyPKCS1v15, + }, + jwa.RS512: { + Hash: crypto.SHA512, + VerifyFunc: makeVerifyPKCS1v15, + }, + jwa.PS256: { + Hash: crypto.SHA256, + VerifyFunc: makeVerifyPSS, + }, + jwa.PS384: { + Hash: crypto.SHA384, + VerifyFunc: makeVerifyPSS, + }, + jwa.PS512: { + Hash: crypto.SHA512, + VerifyFunc: makeVerifyPSS, + }, + } + + for alg, item := range algs { + rsaVerifyFuncs[alg] = item.VerifyFunc(item.Hash) + } +} + +func makeVerifyPKCS1v15(hash crypto.Hash) rsaVerifyFunc { + return rsaVerifyFunc(func(payload, signature []byte, key *rsa.PublicKey) error { + h := hash.New() + h.Write(payload) + return rsa.VerifyPKCS1v15(key, hash, h.Sum(nil), signature) + }) +} + +func makeVerifyPSS(hash crypto.Hash) rsaVerifyFunc { + return rsaVerifyFunc(func(payload, signature []byte, key *rsa.PublicKey) error { + h := hash.New() + h.Write(payload) + return rsa.VerifyPSS(key, hash, h.Sum(nil), signature, nil) + }) +} + +func newRSA(alg jwa.SignatureAlgorithm) (*RSAVerifier, error) { + verifyfn, ok := rsaVerifyFuncs[alg] + if !ok { + return nil, errors.Errorf(`unsupported algorithm while trying to create RSA verifier: %s`, alg) + } + + return &RSAVerifier{ + verify: verifyfn, + }, nil +} + +// Verify checks if a JWS is valid. +func (v RSAVerifier) Verify(payload, signature []byte, key interface{}) error { + if key == nil { + return errors.New(`missing public key while verifying payload`) + } + rsaKey, ok := key.(*rsa.PublicKey) + if !ok { + return errors.Errorf(`invalid key type %T. *rsa.PublicKey is required`, key) + } + + return v.verify(payload, signature, rsaKey) +} diff --git a/vendor/github.com/open-policy-agent/opa/topdown/internal/jwx/jws/verify/verify.go b/vendor/github.com/open-policy-agent/opa/topdown/internal/jwx/jws/verify/verify.go new file mode 100644 index 000000000..d484cda0b --- /dev/null +++ b/vendor/github.com/open-policy-agent/opa/topdown/internal/jwx/jws/verify/verify.go @@ -0,0 +1,22 @@ +package verify + +import ( + "github.com/pkg/errors" + + "github.com/open-policy-agent/opa/topdown/internal/jwx/jwa" +) + +// New creates a new JWS verifier using the specified algorithm +// and the public key +func New(alg jwa.SignatureAlgorithm) (Verifier, error) { + switch alg { + case jwa.RS256, jwa.RS384, jwa.RS512, jwa.PS256, jwa.PS384, jwa.PS512: + return newRSA(alg) + case jwa.ES256, jwa.ES384, jwa.ES512: + return newECDSA(alg) + case jwa.HS256, jwa.HS384, jwa.HS512: + return newHMAC(alg) + default: + return nil, errors.Errorf(`unsupported signature algorithm: %s`, alg) + } +} diff --git a/vendor/github.com/open-policy-agent/opa/topdown/json.go b/vendor/github.com/open-policy-agent/opa/topdown/json.go new file mode 100644 index 000000000..1253015c5 --- /dev/null +++ b/vendor/github.com/open-policy-agent/opa/topdown/json.go @@ -0,0 +1,235 @@ +// Copyright 2019 The OPA Authors. All rights reserved. +// Use of this source code is governed by an Apache2 +// license that can be found in the LICENSE file. + +package topdown + +import ( + "fmt" + "strconv" + "strings" + + "github.com/open-policy-agent/opa/ast" + "github.com/open-policy-agent/opa/topdown/builtins" +) + +func builtinJSONRemove(_ BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error { + + // Expect an object and a string or array/set of strings + _, err := builtins.ObjectOperand(operands[0].Value, 1) + if err != nil { + return err + } + + // Build a list of json pointers to remove + paths, err := getJSONPaths(operands[1].Value) + if err != nil { + return err + } + + newObj, err := jsonRemove(operands[0], ast.NewTerm(pathsToObject(paths))) + if err != nil { + return err + } + + if newObj == nil { + return nil + } + + return iter(newObj) +} + +// jsonRemove returns a new term that is the result of walking +// through a and omitting removing any values that are in b but +// have ast.Null values (ie leaf nodes for b). +func jsonRemove(a *ast.Term, b *ast.Term) (*ast.Term, error) { + if b == nil { + // The paths diverged, return a + return a, nil + } + + var bObj ast.Object + switch bValue := b.Value.(type) { + case ast.Object: + bObj = bValue + case ast.Null: + // Means we hit a leaf node on "b", dont add the value for a + return nil, nil + default: + // The paths diverged, return a + return a, nil + } + + switch aValue := a.Value.(type) { + case ast.String, ast.Number, ast.Boolean, ast.Null: + return a, nil + case ast.Object: + newObj := ast.NewObject() + err := aValue.Iter(func(k *ast.Term, v *ast.Term) error { + // recurse and add the diff of sub objects as needed + diffValue, err := jsonRemove(v, bObj.Get(k)) + if err != nil || diffValue == nil { + return err + } + newObj.Insert(k, diffValue) + return nil + }) + if err != nil { + return nil, err + } + return ast.NewTerm(newObj), nil + case ast.Set: + newSet := ast.NewSet() + err := aValue.Iter(func(v *ast.Term) error { + // recurse and add the diff of sub objects as needed + diffValue, err := jsonRemove(v, bObj.Get(v)) + if err != nil || diffValue == nil { + return err + } + newSet.Add(diffValue) + return nil + }) + if err != nil { + return nil, err + } + return ast.NewTerm(newSet), nil + case ast.Array: + // When indexes are removed we shift left to close empty spots in the array + // as per the JSON patch spec. + var newArray ast.Array + for i, v := range aValue { + // recurse and add the diff of sub objects as needed + // Note: Keys in b will be strings for the index, eg path /a/1/b => {"a": {"1": {"b": null}}} + diffValue, err := jsonRemove(v, bObj.Get(ast.StringTerm(strconv.Itoa(i)))) + if err != nil { + return nil, err + } + if diffValue != nil { + newArray = append(newArray, diffValue) + } + } + return ast.NewTerm(newArray), nil + default: + return nil, fmt.Errorf("invalid value type %T", a) + } +} + +func builtinJSONFilter(_ BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error { + + // Ensure we have the right parameters, expect an object and a string or array/set of strings + obj, err := builtins.ObjectOperand(operands[0].Value, 1) + if err != nil { + return err + } + + // Build a list of filter strings + filters, err := getJSONPaths(operands[1].Value) + if err != nil { + return err + } + + // Actually do the filtering + filterObj := pathsToObject(filters) + r, err := obj.Filter(filterObj) + if err != nil { + return err + } + + return iter(ast.NewTerm(r)) +} + +func getJSONPaths(operand ast.Value) ([]ast.Ref, error) { + var paths []ast.Ref + + switch v := operand.(type) { + case ast.Array: + for _, f := range v { + filter, err := parsePath(f) + if err != nil { + return nil, err + } + paths = append(paths, filter) + } + case ast.Set: + err := v.Iter(func(f *ast.Term) error { + filter, err := parsePath(f) + if err != nil { + return err + } + paths = append(paths, filter) + return nil + }) + if err != nil { + return nil, err + } + default: + return nil, builtins.NewOperandTypeErr(2, v, "set", "array") + } + + return paths, nil +} + +func parsePath(path *ast.Term) (ast.Ref, error) { + // paths can either be a `/` separated json path or + // an array or set of values + var pathSegments ast.Ref + switch p := path.Value.(type) { + case ast.String: + parts := strings.Split(strings.Trim(string(p), "/"), "/") + for _, part := range parts { + part = strings.ReplaceAll(strings.ReplaceAll(part, "~1", "/"), "~0", "~") + pathSegments = append(pathSegments, ast.StringTerm(part)) + } + case ast.Array: + for _, term := range p { + pathSegments = append(pathSegments, term) + } + default: + return nil, builtins.NewOperandErr(2, "must be one of {set, array} containing string paths or array of path segments but got %v", ast.TypeName(p)) + } + + return pathSegments, nil +} + +func pathsToObject(paths []ast.Ref) ast.Object { + + root := ast.NewObject() + + for _, path := range paths { + node := root + var done bool + + for i := 0; i < len(path)-1 && !done; i++ { + + k := path[i] + child := node.Get(k) + + if child == nil { + obj := ast.NewObject() + node.Insert(k, ast.NewTerm(obj)) + node = obj + continue + } + + switch v := child.Value.(type) { + case ast.Null: + done = true + case ast.Object: + node = v + default: + panic("unreachable") + } + } + + if !done { + node.Insert(path[len(path)-1], ast.NullTerm()) + } + } + + return root +} + +func init() { + RegisterBuiltinFunc(ast.JSONFilter.Name, builtinJSONFilter) + RegisterBuiltinFunc(ast.JSONRemove.Name, builtinJSONRemove) +} diff --git a/vendor/github.com/open-policy-agent/opa/topdown/object.go b/vendor/github.com/open-policy-agent/opa/topdown/object.go new file mode 100644 index 000000000..5fb3ee847 --- /dev/null +++ b/vendor/github.com/open-policy-agent/opa/topdown/object.go @@ -0,0 +1,139 @@ +// Copyright 2020 The OPA Authors. All rights reserved. +// Use of this source code is governed by an Apache2 +// license that can be found in the LICENSE file. + +package topdown + +import ( + "github.com/open-policy-agent/opa/ast" + "github.com/open-policy-agent/opa/topdown/builtins" + "github.com/open-policy-agent/opa/types" +) + +func builtinObjectUnion(_ BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error { + objA, err := builtins.ObjectOperand(operands[0].Value, 1) + if err != nil { + return err + } + + objB, err := builtins.ObjectOperand(operands[1].Value, 2) + if err != nil { + return err + } + + r := mergeWithOverwrite(objA, objB) + + return iter(ast.NewTerm(r)) +} + +func builtinObjectRemove(_ BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error { + // Expect an object and an array/set/object of keys + obj, err := builtins.ObjectOperand(operands[0].Value, 1) + if err != nil { + return err + } + + // Build a set of keys to remove + keysToRemove, err := getObjectKeysParam(operands[1].Value) + if err != nil { + return err + } + r := ast.NewObject() + obj.Foreach(func(key *ast.Term, value *ast.Term) { + if !keysToRemove.Contains(key) { + r.Insert(key, value) + } + }) + + return iter(ast.NewTerm(r)) +} + +func builtinObjectFilter(_ BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error { + // Expect an object and an array/set/object of keys + obj, err := builtins.ObjectOperand(operands[0].Value, 1) + if err != nil { + return err + } + + // Build a new object from the supplied filter keys + keys, err := getObjectKeysParam(operands[1].Value) + if err != nil { + return err + } + + filterObj := ast.NewObject() + keys.Foreach(func(key *ast.Term) { + filterObj.Insert(key, ast.NullTerm()) + }) + + // Actually do the filtering + r, err := obj.Filter(filterObj) + if err != nil { + return err + } + + return iter(ast.NewTerm(r)) +} + +func builtinObjectGet(_ BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error { + object, err := builtins.ObjectOperand(operands[0].Value, 1) + if err != nil { + return err + } + + if ret := object.Get(operands[1]); ret != nil { + return iter(ret) + } + + return iter(operands[2]) +} + +// getObjectKeysParam returns a set of key values +// from a supplied ast array, object, set value +func getObjectKeysParam(arrayOrSet ast.Value) (ast.Set, error) { + keys := ast.NewSet() + + switch v := arrayOrSet.(type) { + case ast.Array: + for _, f := range v { + keys.Add(f) + } + case ast.Set: + _ = v.Iter(func(f *ast.Term) error { + keys.Add(f) + return nil + }) + case ast.Object: + _ = v.Iter(func(k *ast.Term, _ *ast.Term) error { + keys.Add(k) + return nil + }) + default: + return nil, builtins.NewOperandTypeErr(2, arrayOrSet, ast.TypeName(types.Object{}), ast.TypeName(types.S), ast.TypeName(types.Array{})) + } + + return keys, nil +} + +func mergeWithOverwrite(objA, objB ast.Object) ast.Object { + merged, _ := objA.MergeWith(objB, func(v1, v2 *ast.Term) (*ast.Term, bool) { + originalValueObj, ok2 := v1.Value.(ast.Object) + updateValueObj, ok1 := v2.Value.(ast.Object) + if !ok1 || !ok2 { + // If we can't merge, stick with the right-hand value + return v2, false + } + + // Recursively update the existing value + merged := mergeWithOverwrite(originalValueObj, updateValueObj) + return ast.NewTerm(merged), false + }) + return merged +} + +func init() { + RegisterBuiltinFunc(ast.ObjectUnion.Name, builtinObjectUnion) + RegisterBuiltinFunc(ast.ObjectRemove.Name, builtinObjectRemove) + RegisterBuiltinFunc(ast.ObjectFilter.Name, builtinObjectFilter) + RegisterBuiltinFunc(ast.ObjectGet.Name, builtinObjectGet) +} diff --git a/vendor/github.com/open-policy-agent/opa/topdown/parse.go b/vendor/github.com/open-policy-agent/opa/topdown/parse.go new file mode 100644 index 000000000..dc6e6fc51 --- /dev/null +++ b/vendor/github.com/open-policy-agent/opa/topdown/parse.go @@ -0,0 +1,47 @@ +// Copyright 2018 The OPA Authors. All rights reserved. +// Use of this source code is governed by an Apache2 +// license that can be found in the LICENSE file. + +package topdown + +import ( + "bytes" + "encoding/json" + + "github.com/open-policy-agent/opa/ast" + "github.com/open-policy-agent/opa/topdown/builtins" +) + +func builtinRegoParseModule(a, b ast.Value) (ast.Value, error) { + + filename, err := builtins.StringOperand(a, 1) + if err != nil { + return nil, err + } + + input, err := builtins.StringOperand(b, 1) + if err != nil { + return nil, err + } + + module, err := ast.ParseModule(string(filename), string(input)) + if err != nil { + return nil, err + } + + var buf bytes.Buffer + if err := json.NewEncoder(&buf).Encode(module); err != nil { + return nil, err + } + + term, err := ast.ParseTerm(buf.String()) + if err != nil { + return nil, err + } + + return term.Value, nil +} + +func init() { + RegisterFunctionalBuiltin2(ast.RegoParseModule.Name, builtinRegoParseModule) +} diff --git a/vendor/github.com/open-policy-agent/opa/topdown/parse_bytes.go b/vendor/github.com/open-policy-agent/opa/topdown/parse_bytes.go new file mode 100644 index 000000000..817edd5ee --- /dev/null +++ b/vendor/github.com/open-policy-agent/opa/topdown/parse_bytes.go @@ -0,0 +1,153 @@ +// Copyright 2016 The OPA Authors. All rights reserved. +// Use of this source code is governed by an Apache2 +// license that can be found in the LICENSE file. + +package topdown + +import ( + "fmt" + "math/big" + "strconv" + "strings" + + "github.com/open-policy-agent/opa/ast" + "github.com/open-policy-agent/opa/topdown/builtins" +) + +const ( + none int64 = 1 + kb = 1000 + ki = 1024 + mb = kb * 1000 + mi = ki * 1024 + gb = mb * 1000 + gi = mi * 1024 + tb = gb * 1000 + ti = gi * 1024 +) + +// The rune values for 0..9 as well as the period symbol (for parsing floats) +var numRunes = []rune("0123456789.") + +func parseNumBytesError(msg string) error { + return fmt.Errorf("%s error: %s", ast.UnitsParseBytes.Name, msg) +} + +func errUnitNotRecognized(unit string) error { + return parseNumBytesError(fmt.Sprintf("byte unit %s not recognized", unit)) +} + +var ( + errNoAmount = parseNumBytesError("no byte amount provided") + errIntConv = parseNumBytesError("could not parse byte amount to integer") + errIncludesSpaces = parseNumBytesError("spaces not allowed in resource strings") +) + +func builtinNumBytes(a ast.Value) (ast.Value, error) { + var m int64 + + raw, err := builtins.StringOperand(a, 1) + if err != nil { + return nil, err + } + + s := formatString(raw) + + if strings.Contains(s, " ") { + return nil, errIncludesSpaces + } + + numStr, unitStr := extractNumAndUnit(s) + + if numStr == "" { + return nil, errNoAmount + } + + switch unitStr { + case "": + m = none + case "kb": + m = kb + case "kib": + m = ki + case "mb": + m = mb + case "mib": + m = mi + case "gb": + m = gb + case "gib": + m = gi + case "tb": + m = tb + case "tib": + m = ti + default: + return nil, errUnitNotRecognized(unitStr) + } + + num, err := strconv.ParseInt(numStr, 10, 64) + if err != nil { + return nil, errIntConv + } + + total := num * m + + return builtins.IntToNumber(big.NewInt(total)), nil +} + +// Makes the string lower case and removes spaces and quotation marks +func formatString(s ast.String) string { + str := string(s) + lower := strings.ToLower(str) + return strings.Replace(lower, "\"", "", -1) +} + +// Splits the string into a number string à la "10" or "10.2" and a unit string à la "gb" or "MiB" or "foo". Either +// can be an empty string (error handling is provided elsewhere). +func extractNumAndUnit(s string) (string, string) { + isNum := func(r rune) (isNum bool) { + for _, nr := range numRunes { + if nr == r { + return true + } + } + + return false + } + + // Returns the index of the first rune that's not a number (or 0 if there are only numbers) + getFirstNonNumIdx := func(s string) int { + for idx, r := range s { + if !isNum(r) { + return idx + } + } + + return 0 + } + + firstRuneIsNum := func(s string) bool { + return isNum(rune(s[0])) + } + + firstNonNumIdx := getFirstNonNumIdx(s) + + // The string contains only a number + numOnly := firstNonNumIdx == 0 && firstRuneIsNum(s) + + // The string contains only a unit + unitOnly := firstNonNumIdx == 0 && !firstRuneIsNum(s) + + if numOnly { + return s, "" + } else if unitOnly { + return "", s + } else { + return s[0:firstNonNumIdx], s[firstNonNumIdx:] + } +} + +func init() { + RegisterFunctionalBuiltin1(ast.UnitsParseBytes.Name, builtinNumBytes) +} diff --git a/vendor/github.com/open-policy-agent/opa/topdown/query.go b/vendor/github.com/open-policy-agent/opa/topdown/query.go new file mode 100644 index 000000000..95fab6b52 --- /dev/null +++ b/vendor/github.com/open-policy-agent/opa/topdown/query.go @@ -0,0 +1,317 @@ +package topdown + +import ( + "context" + "sort" + + "github.com/open-policy-agent/opa/ast" + "github.com/open-policy-agent/opa/metrics" + "github.com/open-policy-agent/opa/storage" + "github.com/open-policy-agent/opa/topdown/builtins" + "github.com/open-policy-agent/opa/topdown/copypropagation" +) + +// QueryResultSet represents a collection of results returned by a query. +type QueryResultSet []QueryResult + +// QueryResult represents a single result returned by a query. The result +// contains bindings for all variables that appear in the query. +type QueryResult map[ast.Var]*ast.Term + +// Query provides a configurable interface for performing query evaluation. +type Query struct { + cancel Cancel + query ast.Body + queryCompiler ast.QueryCompiler + compiler *ast.Compiler + store storage.Store + txn storage.Transaction + input *ast.Term + tracers []Tracer + unknowns []*ast.Term + partialNamespace string + metrics metrics.Metrics + instr *Instrumentation + disableInlining []ast.Ref + genvarprefix string + runtime *ast.Term + builtins map[string]*Builtin + indexing bool +} + +// Builtin represents a built-in function that queries can call. +type Builtin struct { + Decl *ast.Builtin + Func BuiltinFunc +} + +// NewQuery returns a new Query object that can be run. +func NewQuery(query ast.Body) *Query { + return &Query{ + query: query, + genvarprefix: ast.WildcardPrefix, + indexing: true, + } +} + +// WithQueryCompiler sets the queryCompiler used for the query. +func (q *Query) WithQueryCompiler(queryCompiler ast.QueryCompiler) *Query { + q.queryCompiler = queryCompiler + return q +} + +// WithCompiler sets the compiler to use for the query. +func (q *Query) WithCompiler(compiler *ast.Compiler) *Query { + q.compiler = compiler + return q +} + +// WithStore sets the store to use for the query. +func (q *Query) WithStore(store storage.Store) *Query { + q.store = store + return q +} + +// WithTransaction sets the transaction to use for the query. All queries +// should be performed over a consistent snapshot of the storage layer. +func (q *Query) WithTransaction(txn storage.Transaction) *Query { + q.txn = txn + return q +} + +// WithCancel sets the cancellation object to use for the query. Set this if +// you need to abort queries based on a deadline. This is optional. +func (q *Query) WithCancel(cancel Cancel) *Query { + q.cancel = cancel + return q +} + +// WithInput sets the input object to use for the query. References rooted at +// input will be evaluated against this value. This is optional. +func (q *Query) WithInput(input *ast.Term) *Query { + q.input = input + return q +} + +// WithTracer adds a query tracer to use during evaluation. This is optional. +func (q *Query) WithTracer(tracer Tracer) *Query { + q.tracers = append(q.tracers, tracer) + return q +} + +// WithMetrics sets the metrics collection to add evaluation metrics to. This +// is optional. +func (q *Query) WithMetrics(m metrics.Metrics) *Query { + q.metrics = m + return q +} + +// WithInstrumentation sets the instrumentation configuration to enable on the +// evaluation process. By default, instrumentation is turned off. +func (q *Query) WithInstrumentation(instr *Instrumentation) *Query { + q.instr = instr + return q +} + +// WithUnknowns sets the initial set of variables or references to treat as +// unknown during query evaluation. This is required for partial evaluation. +func (q *Query) WithUnknowns(terms []*ast.Term) *Query { + q.unknowns = terms + return q +} + +// WithPartialNamespace sets the namespace to use for supporting rules +// generated as part of the partial evaluation process. The ns value must be a +// valid package path component. +func (q *Query) WithPartialNamespace(ns string) *Query { + q.partialNamespace = ns + return q +} + +// WithDisableInlining adds a set of paths to the query that should be excluded from +// inlining. Inlining during partial evaluation can be expensive in some cases +// (e.g., when a cross-product is computed.) Disabling inlining avoids expensive +// computation at the cost of generating support rules. +func (q *Query) WithDisableInlining(paths []ast.Ref) *Query { + q.disableInlining = paths + return q +} + +// WithRuntime sets the runtime data to execute the query with. The runtime data +// can be returned by the `opa.runtime` built-in function. +func (q *Query) WithRuntime(runtime *ast.Term) *Query { + q.runtime = runtime + return q +} + +// WithBuiltins adds a set of built-in functions that can be called by the +// query. +func (q *Query) WithBuiltins(builtins map[string]*Builtin) *Query { + q.builtins = builtins + return q +} + +// WithIndexing will enable or disable using rule indexing for the evaluation +// of the query. The default is enabled. +func (q *Query) WithIndexing(enabled bool) *Query { + q.indexing = enabled + return q +} + +// PartialRun executes partial evaluation on the query with respect to unknown +// values. Partial evaluation attempts to evaluate as much of the query as +// possible without requiring values for the unknowns set on the query. The +// result of partial evaluation is a new set of queries that can be evaluated +// once the unknown value is known. In addition to new queries, partial +// evaluation may produce additional support modules that should be used in +// conjunction with the partially evaluated queries. +func (q *Query) PartialRun(ctx context.Context) (partials []ast.Body, support []*ast.Module, err error) { + if q.partialNamespace == "" { + q.partialNamespace = "partial" // lazily initialize partial namespace + } + f := &queryIDFactory{} + b := newBindings(0, q.instr) + e := &eval{ + ctx: ctx, + cancel: q.cancel, + query: q.query, + queryCompiler: q.queryCompiler, + queryIDFact: f, + queryID: f.Next(), + bindings: b, + compiler: q.compiler, + store: q.store, + baseCache: newBaseCache(), + targetStack: newRefStack(), + txn: q.txn, + input: q.input, + tracers: q.tracers, + instr: q.instr, + builtins: q.builtins, + builtinCache: builtins.Cache{}, + virtualCache: newVirtualCache(), + saveSet: newSaveSet(q.unknowns, b, q.instr), + saveStack: newSaveStack(), + saveSupport: newSaveSupport(), + saveNamespace: ast.StringTerm(q.partialNamespace), + genvarprefix: q.genvarprefix, + runtime: q.runtime, + indexing: q.indexing, + } + + if len(q.disableInlining) > 0 { + e.disableInlining = [][]ast.Ref{q.disableInlining} + } + + e.caller = e + q.startTimer(metrics.RegoPartialEval) + defer q.stopTimer(metrics.RegoPartialEval) + + livevars := ast.NewVarSet() + + ast.WalkVars(q.query, func(x ast.Var) bool { + if !x.IsGenerated() { + livevars.Add(x) + } + return false + }) + + p := copypropagation.New(livevars) + + err = e.Run(func(e *eval) error { + + // Build output from saved expressions. + body := ast.NewBody() + + for _, elem := range e.saveStack.Stack[len(e.saveStack.Stack)-1] { + body.Append(elem.Plug(e.bindings)) + } + + // Include bindings as exprs so that when caller evals the result, they + // can obtain values for the vars in their query. + bindingExprs := []*ast.Expr{} + e.bindings.Iter(e.bindings, func(a, b *ast.Term) error { + bindingExprs = append(bindingExprs, ast.Equality.Expr(a, b)) + return nil + }) + + // Sort binding expressions so that results are deterministic. + sort.Slice(bindingExprs, func(i, j int) bool { + return bindingExprs[i].Compare(bindingExprs[j]) < 0 + }) + + for i := range bindingExprs { + body.Append(bindingExprs[i]) + } + + partials = append(partials, applyCopyPropagation(p, e.instr, body)) + return nil + }) + + support = e.saveSupport.List() + + return partials, support, err +} + +// Run is a wrapper around Iter that accumulates query results and returns them +// in one shot. +func (q *Query) Run(ctx context.Context) (QueryResultSet, error) { + qrs := QueryResultSet{} + return qrs, q.Iter(ctx, func(qr QueryResult) error { + qrs = append(qrs, qr) + return nil + }) +} + +// Iter executes the query and invokes the iter function with query results +// produced by evaluating the query. +func (q *Query) Iter(ctx context.Context, iter func(QueryResult) error) error { + f := &queryIDFactory{} + e := &eval{ + ctx: ctx, + cancel: q.cancel, + query: q.query, + queryCompiler: q.queryCompiler, + queryIDFact: f, + queryID: f.Next(), + bindings: newBindings(0, q.instr), + compiler: q.compiler, + store: q.store, + baseCache: newBaseCache(), + targetStack: newRefStack(), + txn: q.txn, + input: q.input, + tracers: q.tracers, + instr: q.instr, + builtins: q.builtins, + builtinCache: builtins.Cache{}, + virtualCache: newVirtualCache(), + genvarprefix: q.genvarprefix, + runtime: q.runtime, + indexing: q.indexing, + } + e.caller = e + q.startTimer(metrics.RegoQueryEval) + err := e.Run(func(e *eval) error { + qr := QueryResult{} + e.bindings.Iter(nil, func(k, v *ast.Term) error { + qr[k.Value.(ast.Var)] = v + return nil + }) + return iter(qr) + }) + q.stopTimer(metrics.RegoQueryEval) + return err +} + +func (q *Query) startTimer(name string) { + if q.metrics != nil { + q.metrics.Timer(name).Start() + } +} + +func (q *Query) stopTimer(name string) { + if q.metrics != nil { + q.metrics.Timer(name).Stop() + } +} diff --git a/vendor/github.com/open-policy-agent/opa/topdown/regex.go b/vendor/github.com/open-policy-agent/opa/topdown/regex.go new file mode 100644 index 000000000..7005cad2a --- /dev/null +++ b/vendor/github.com/open-policy-agent/opa/topdown/regex.go @@ -0,0 +1,201 @@ +// Copyright 2016 The OPA Authors. All rights reserved. +// Use of this source code is governed by an Apache2 +// license that can be found in the LICENSE file. + +package topdown + +import ( + "fmt" + "regexp" + "sync" + + "github.com/yashtewari/glob-intersection" + + "github.com/open-policy-agent/opa/ast" + "github.com/open-policy-agent/opa/topdown/builtins" +) + +var regexpCacheLock = sync.Mutex{} +var regexpCache map[string]*regexp.Regexp + +func builtinRegexMatch(a, b ast.Value) (ast.Value, error) { + s1, err := builtins.StringOperand(a, 1) + if err != nil { + return nil, err + } + s2, err := builtins.StringOperand(b, 2) + if err != nil { + return nil, err + } + re, err := getRegexp(string(s1)) + if err != nil { + return nil, err + } + return ast.Boolean(re.Match([]byte(s2))), nil +} + +func builtinRegexMatchTemplate(a, b, c, d ast.Value) (ast.Value, error) { + pattern, err := builtins.StringOperand(a, 1) + if err != nil { + return nil, err + } + match, err := builtins.StringOperand(b, 2) + if err != nil { + return nil, err + } + start, err := builtins.StringOperand(c, 3) + if err != nil { + return nil, err + } + end, err := builtins.StringOperand(d, 4) + if err != nil { + return nil, err + } + if len(start) != 1 { + return nil, fmt.Errorf("start delimiter has to be exactly one character long but is %d long", len(start)) + } + if len(end) != 1 { + return nil, fmt.Errorf("end delimiter has to be exactly one character long but is %d long", len(start)) + } + re, err := getRegexpTemplate(string(pattern), string(start)[0], string(end)[0]) + if err != nil { + return nil, err + } + return ast.Boolean(re.MatchString(string(match))), nil +} + +func builtinRegexSplit(a, b ast.Value) (ast.Value, error) { + s1, err := builtins.StringOperand(a, 1) + if err != nil { + return nil, err + } + s2, err := builtins.StringOperand(b, 2) + if err != nil { + return nil, err + } + re, err := getRegexp(string(s1)) + if err != nil { + return nil, err + } + + elems := re.Split(string(s2), -1) + arr := make(ast.Array, len(elems)) + for i := range arr { + arr[i] = ast.StringTerm(elems[i]) + } + return arr, nil +} + +func getRegexp(pat string) (*regexp.Regexp, error) { + regexpCacheLock.Lock() + defer regexpCacheLock.Unlock() + re, ok := regexpCache[pat] + if !ok { + var err error + re, err = regexp.Compile(string(pat)) + if err != nil { + return nil, err + } + regexpCache[pat] = re + } + return re, nil +} + +func getRegexpTemplate(pat string, delimStart, delimEnd byte) (*regexp.Regexp, error) { + regexpCacheLock.Lock() + defer regexpCacheLock.Unlock() + re, ok := regexpCache[pat] + if !ok { + var err error + re, err = compileRegexTemplate(string(pat), delimStart, delimEnd) + if err != nil { + return nil, err + } + regexpCache[pat] = re + } + return re, nil +} + +func builtinGlobsMatch(a, b ast.Value) (ast.Value, error) { + s1, err := builtins.StringOperand(a, 1) + if err != nil { + return nil, err + } + s2, err := builtins.StringOperand(b, 2) + if err != nil { + return nil, err + } + ne, err := gintersect.NonEmpty(string(s1), string(s2)) + if err != nil { + return nil, err + } + return ast.Boolean(ne), nil +} + +func builtinRegexFind(a, b, c ast.Value) (ast.Value, error) { + s1, err := builtins.StringOperand(a, 1) + if err != nil { + return nil, err + } + s2, err := builtins.StringOperand(b, 2) + if err != nil { + return nil, err + } + n, err := builtins.IntOperand(c, 3) + if err != nil { + return nil, err + } + re, err := getRegexp(string(s1)) + if err != nil { + return nil, err + } + + elems := re.FindAllString(string(s2), n) + arr := make(ast.Array, len(elems)) + for i := range arr { + arr[i] = ast.StringTerm(elems[i]) + } + return arr, nil +} + +func builtinRegexFindAllStringSubmatch(a, b, c ast.Value) (ast.Value, error) { + s1, err := builtins.StringOperand(a, 1) + if err != nil { + return nil, err + } + s2, err := builtins.StringOperand(b, 2) + if err != nil { + return nil, err + } + n, err := builtins.IntOperand(c, 3) + if err != nil { + return nil, err + } + + re, err := getRegexp(string(s1)) + if err != nil { + return nil, err + } + matches := re.FindAllStringSubmatch(string(s2), n) + + outer := make(ast.Array, len(matches)) + for i := range outer { + inner := make(ast.Array, len(matches[i])) + for j := range inner { + inner[j] = ast.StringTerm(matches[i][j]) + } + outer[i] = ast.ArrayTerm(inner...) + } + + return outer, nil +} + +func init() { + regexpCache = map[string]*regexp.Regexp{} + RegisterFunctionalBuiltin2(ast.RegexMatch.Name, builtinRegexMatch) + RegisterFunctionalBuiltin2(ast.RegexSplit.Name, builtinRegexSplit) + RegisterFunctionalBuiltin2(ast.GlobsMatch.Name, builtinGlobsMatch) + RegisterFunctionalBuiltin4(ast.RegexTemplateMatch.Name, builtinRegexMatchTemplate) + RegisterFunctionalBuiltin3(ast.RegexFind.Name, builtinRegexFind) + RegisterFunctionalBuiltin3(ast.RegexFindAllStringSubmatch.Name, builtinRegexFindAllStringSubmatch) +} diff --git a/vendor/github.com/open-policy-agent/opa/topdown/regex_template.go b/vendor/github.com/open-policy-agent/opa/topdown/regex_template.go new file mode 100644 index 000000000..39f92346f --- /dev/null +++ b/vendor/github.com/open-policy-agent/opa/topdown/regex_template.go @@ -0,0 +1,122 @@ +package topdown + +// Copyright 2012 The Gorilla Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license as follows: + +// Copyright (c) 2012 Rodrigo Moraes. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +// This file was forked from https://github.com/gorilla/mux/commit/eac83ba2c004bb75 + +import ( + "bytes" + "fmt" + "regexp" +) + +// delimiterIndices returns the first level delimiter indices from a string. +// It returns an error in case of unbalanced delimiters. +func delimiterIndices(s string, delimiterStart, delimiterEnd byte) ([]int, error) { + var level, idx int + idxs := make([]int, 0) + for i := 0; i < len(s); i++ { + switch s[i] { + case delimiterStart: + if level++; level == 1 { + idx = i + } + case delimiterEnd: + if level--; level == 0 { + idxs = append(idxs, idx, i+1) + } else if level < 0 { + return nil, fmt.Errorf(`unbalanced braces in %q`, s) + } + } + } + + if level != 0 { + return nil, fmt.Errorf(`unbalanced braces in %q`, s) + } + + return idxs, nil +} + +// compileRegexTemplate parses a template and returns a Regexp. +// +// You can define your own delimiters. It is e.g. common to use curly braces {} but I recommend using characters +// which have no special meaning in Regex, e.g.: <, > +// +// reg, err := compiler.CompileRegex("foo:bar.baz:<[0-9]{2,10}>", '<', '>') +// // if err != nil ... +// reg.MatchString("foo:bar.baz:123") +func compileRegexTemplate(tpl string, delimiterStart, delimiterEnd byte) (*regexp.Regexp, error) { + // Check if it is well-formed. + idxs, errBraces := delimiterIndices(tpl, delimiterStart, delimiterEnd) + if errBraces != nil { + return nil, errBraces + } + varsR := make([]*regexp.Regexp, len(idxs)/2) + pattern := bytes.NewBufferString("") + + // WriteByte's error value is always nil for bytes.Buffer, no need to check it. + pattern.WriteByte('^') + + var end int + var err error + for i := 0; i < len(idxs); i += 2 { + // Set all values we are interested in. + raw := tpl[end:idxs[i]] + end = idxs[i+1] + patt := tpl[idxs[i]+1 : end-1] + // Build the regexp pattern. + varIdx := i / 2 + fmt.Fprintf(pattern, "%s(%s)", regexp.QuoteMeta(raw), patt) + varsR[varIdx], err = regexp.Compile(fmt.Sprintf("^%s$", patt)) + if err != nil { + return nil, err + } + } + + // Add the remaining. + raw := tpl[end:] + + // WriteString's error value is always nil for bytes.Buffer, no need to check it. + pattern.WriteString(regexp.QuoteMeta(raw)) + + // WriteByte's error value is always nil for bytes.Buffer, no need to check it. + pattern.WriteByte('$') + + // Compile full regexp. + reg, errCompile := regexp.Compile(pattern.String()) + if errCompile != nil { + return nil, errCompile + } + + return reg, nil +} diff --git a/vendor/github.com/open-policy-agent/opa/topdown/runtime.go b/vendor/github.com/open-policy-agent/opa/topdown/runtime.go new file mode 100644 index 000000000..67e183d01 --- /dev/null +++ b/vendor/github.com/open-policy-agent/opa/topdown/runtime.go @@ -0,0 +1,20 @@ +// Copyright 2018 The OPA Authors. All rights reserved. +// Use of this source code is governed by an Apache2 +// license that can be found in the LICENSE file. + +package topdown + +import "github.com/open-policy-agent/opa/ast" + +func builtinOPARuntime(bctx BuiltinContext, _ []*ast.Term, iter func(*ast.Term) error) error { + + if bctx.Runtime == nil { + return iter(ast.ObjectTerm()) + } + + return iter(bctx.Runtime) +} + +func init() { + RegisterBuiltinFunc(ast.OPARuntime.Name, builtinOPARuntime) +} diff --git a/vendor/github.com/open-policy-agent/opa/topdown/save.go b/vendor/github.com/open-policy-agent/opa/topdown/save.go new file mode 100644 index 000000000..4037530a8 --- /dev/null +++ b/vendor/github.com/open-policy-agent/opa/topdown/save.go @@ -0,0 +1,382 @@ +package topdown + +import ( + "container/list" + "fmt" + "strings" + + "github.com/open-policy-agent/opa/ast" +) + +// saveSet contains a stack of terms that are considered 'unknown' during +// partial evaluation. Only var and ref terms (rooted at one of the root +// documents) can be added to the save set. Vars added to the save set are +// namespaced by the binding list they are added with. This means the save set +// can be shared across queries. +type saveSet struct { + instr *Instrumentation + l *list.List +} + +func newSaveSet(ts []*ast.Term, b *bindings, instr *Instrumentation) *saveSet { + ss := &saveSet{ + l: list.New(), + instr: instr, + } + ss.Push(ts, b) + return ss +} + +func (ss *saveSet) Push(ts []*ast.Term, b *bindings) { + ss.l.PushBack(newSaveSetElem(ts, b)) +} + +func (ss *saveSet) Pop() { + ss.l.Remove(ss.l.Back()) +} + +// Contains returns true if the term t is contained in the save set. Non-var and +// non-ref terms are never contained. Ref terms are contained if they share a +// prefix with a ref that was added (in either direction). +func (ss *saveSet) Contains(t *ast.Term, b *bindings) bool { + if ss != nil { + ss.instr.startTimer(partialOpSaveSetContains) + ret := ss.contains(t, b) + ss.instr.stopTimer(partialOpSaveSetContains) + return ret + } + return false +} + +func (ss *saveSet) contains(t *ast.Term, b *bindings) bool { + for el := ss.l.Back(); el != nil; el = el.Prev() { + if el.Value.(*saveSetElem).Contains(t, b) { + return true + } + } + return false +} + +// ContainsRecursive retruns true if the term t is or contains a term that is +// contained in the save set. This function will close over the binding list +// when it encounters vars. +func (ss *saveSet) ContainsRecursive(t *ast.Term, b *bindings) bool { + if ss != nil { + ss.instr.startTimer(partialOpSaveSetContainsRec) + ret := ss.containsrec(t, b) + ss.instr.stopTimer(partialOpSaveSetContainsRec) + return ret + } + return false +} + +func (ss *saveSet) containsrec(t *ast.Term, b *bindings) bool { + var found bool + ast.WalkTerms(t, func(x *ast.Term) bool { + if _, ok := x.Value.(ast.Var); ok { + x1, b1 := b.apply(x) + if x1 != x || b1 != b { + if ss.containsrec(x1, b1) { + found = true + } + } else if ss.contains(x1, b1) { + found = true + } + } + return found + }) + return found +} + +func (ss *saveSet) Vars(caller *bindings) ast.VarSet { + result := ast.NewVarSet() + for x := ss.l.Front(); x != nil; x = x.Next() { + elem := x.Value.(*saveSetElem) + for _, v := range elem.vars { + if v, ok := elem.b.PlugNamespaced(v, caller).Value.(ast.Var); ok { + result.Add(v) + } + } + } + return result +} + +func (ss *saveSet) String() string { + var buf []string + + for x := ss.l.Front(); x != nil; x = x.Next() { + buf = append(buf, x.Value.(*saveSetElem).String()) + } + + return "(" + strings.Join(buf, " ") + ")" +} + +type saveSetElem struct { + refs []ast.Ref + vars []*ast.Term + b *bindings +} + +func newSaveSetElem(ts []*ast.Term, b *bindings) *saveSetElem { + + var refs []ast.Ref + var vars []*ast.Term + + for _, t := range ts { + switch v := t.Value.(type) { + case ast.Var: + vars = append(vars, t) + case ast.Ref: + refs = append(refs, v) + default: + panic("illegal value") + } + } + + return &saveSetElem{ + b: b, + vars: vars, + refs: refs, + } +} + +func (sse *saveSetElem) Contains(t *ast.Term, b *bindings) bool { + switch other := t.Value.(type) { + case ast.Var: + return sse.containsVar(t, b) + case ast.Ref: + for _, ref := range sse.refs { + if ref.HasPrefix(other) || other.HasPrefix(ref) { + return true + } + } + return sse.containsVar(other[0], b) + } + return false +} + +func (sse *saveSetElem) String() string { + return fmt.Sprintf("(refs: %v, vars: %v, b: %v)", sse.refs, sse.vars, sse.b) +} + +func (sse *saveSetElem) containsVar(t *ast.Term, b *bindings) bool { + if b == sse.b { + for _, v := range sse.vars { + if v.Equal(t) { + return true + } + } + } + return false +} + +// saveStack contains a stack of queries that represent the result of partial +// evaluation. When partial evaluation completes, the top of the stack +// represents a complete, partially evaluated query that can be saved and +// evaluated later. +// +// The result is stored in a stack so that partial evaluation of a query can be +// paused and then resumed in cases where different queries make up the result +// of partial evaluation, such as when a rule with a default clause is +// partially evaluated. In this case, the partially evaluated rule will be +// output in the support module. +type saveStack struct { + Stack []saveStackQuery +} + +func newSaveStack() *saveStack { + return &saveStack{ + Stack: []saveStackQuery{ + {}, + }, + } +} + +func (s *saveStack) PushQuery(query saveStackQuery) { + s.Stack = append(s.Stack, query) +} + +func (s *saveStack) PopQuery() saveStackQuery { + last := s.Stack[len(s.Stack)-1] + s.Stack = s.Stack[:len(s.Stack)-1] + return last +} + +func (s *saveStack) Peek() saveStackQuery { + return s.Stack[len(s.Stack)-1] +} + +func (s *saveStack) Push(expr *ast.Expr, b1 *bindings, b2 *bindings) { + idx := len(s.Stack) - 1 + s.Stack[idx] = append(s.Stack[idx], saveStackElem{expr, b1, b2}) +} + +func (s *saveStack) Pop() { + idx := len(s.Stack) - 1 + query := s.Stack[idx] + s.Stack[idx] = query[:len(query)-1] +} + +type saveStackQuery []saveStackElem + +func (s saveStackQuery) Plug(b *bindings) ast.Body { + if len(s) == 0 { + return ast.NewBody(ast.NewExpr(ast.BooleanTerm(true))) + } + result := make(ast.Body, len(s)) + for i := range s { + expr := s[i].Plug(b) + result.Set(expr, i) + } + return result +} + +type saveStackElem struct { + Expr *ast.Expr + B1 *bindings + B2 *bindings +} + +func (e saveStackElem) Plug(caller *bindings) *ast.Expr { + if e.B1 == nil && e.B2 == nil { + return e.Expr + } + expr := e.Expr.Copy() + switch terms := expr.Terms.(type) { + case []*ast.Term: + if expr.IsEquality() { + terms[1] = e.B1.PlugNamespaced(terms[1], caller) + terms[2] = e.B2.PlugNamespaced(terms[2], caller) + } else { + for i := 1; i < len(terms); i++ { + terms[i] = e.B1.PlugNamespaced(terms[i], caller) + } + } + case *ast.Term: + expr.Terms = e.B1.PlugNamespaced(terms, caller) + } + for i := range expr.With { + expr.With[i].Value = e.B1.PlugNamespaced(expr.With[i].Value, caller) + } + return expr +} + +// saveSupport contains additional partially evaluated policies that are part +// of the output of partial evaluation. +// +// The support structure is accumulated as partial evaluation runs and then +// considered complete once partial evaluation finishes (but not before). This +// differs from partially evaluated queries which are considered complete as +// soon as each one finishes. +type saveSupport struct { + modules map[string]*ast.Module +} + +func newSaveSupport() *saveSupport { + return &saveSupport{ + modules: map[string]*ast.Module{}, + } +} + +func (s *saveSupport) List() []*ast.Module { + result := []*ast.Module{} + for _, module := range s.modules { + result = append(result, module) + } + return result +} + +func (s *saveSupport) Exists(path ast.Ref) bool { + k := path[:len(path)-1].String() + module, ok := s.modules[k] + if !ok { + return false + } + name := ast.Var(path[len(path)-1].Value.(ast.String)) + for _, rule := range module.Rules { + if rule.Head.Name.Equal(name) { + return true + } + } + return false +} + +func (s *saveSupport) Insert(path ast.Ref, rule *ast.Rule) { + pkg := path[:len(path)-1] + k := pkg.String() + module, ok := s.modules[k] + if !ok { + module = &ast.Module{ + Package: &ast.Package{ + Path: pkg, + }, + } + s.modules[k] = module + } + rule.Module = module + module.Rules = append(module.Rules, rule) +} + +// saveRequired returns true if the statement x will result in some expressions +// being saved. This check allows the evaluator to evaluate statements +// completely during partial evaluation as long as they do not depend on any +// kind of unknown value or statements that would generate saves. +func saveRequired(c *ast.Compiler, ss *saveSet, b *bindings, x interface{}, rec bool) bool { + + var found bool + + vis := ast.NewGenericVisitor(func(node interface{}) bool { + if found { + return found + } + switch node := node.(type) { + case *ast.Expr: + found = len(node.With) > 0 || ignoreExprDuringPartial(node) + case *ast.Term: + switch v := node.Value.(type) { + case ast.Var: + // Variables only need to be tested in the node from call site + // because once traversal recurses into a rule existing unknown + // variables are out-of-scope. + if !rec && ss.ContainsRecursive(node, b) { + found = true + } + case ast.Ref: + if ss.Contains(node, b) { + found = true + } else { + for _, rule := range c.GetRulesDynamic(v) { + if saveRequired(c, ss, b, rule, true) { + found = true + break + } + } + } + } + } + return found + }) + + vis.Walk(x) + + return found +} + +func ignoreExprDuringPartial(expr *ast.Expr) bool { + if !expr.IsCall() { + return false + } + + bi, ok := ast.BuiltinMap[expr.Operator().String()] + + return ok && ignoreDuringPartial(bi) +} + +func ignoreDuringPartial(bi *ast.Builtin) bool { + for _, ignore := range ast.IgnoreDuringPartialEval { + if bi == ignore { + return true + } + } + return false +} diff --git a/vendor/github.com/open-policy-agent/opa/topdown/sets.go b/vendor/github.com/open-policy-agent/opa/topdown/sets.go new file mode 100644 index 000000000..a9c5ad86c --- /dev/null +++ b/vendor/github.com/open-policy-agent/opa/topdown/sets.go @@ -0,0 +1,84 @@ +// Copyright 2016 The OPA Authors. All rights reserved. +// Use of this source code is governed by an Apache2 +// license that can be found in the LICENSE file. + +package topdown + +import ( + "github.com/open-policy-agent/opa/ast" + "github.com/open-policy-agent/opa/topdown/builtins" +) + +// Deprecated in v0.4.2 in favour of minus/infix "-" operation. +func builtinSetDiff(a, b ast.Value) (ast.Value, error) { + + s1, err := builtins.SetOperand(a, 1) + if err != nil { + return nil, err + } + + s2, err := builtins.SetOperand(b, 2) + if err != nil { + return nil, err + } + + return s1.Diff(s2), nil +} + +// builtinSetIntersection returns the intersection of the given input sets +func builtinSetIntersection(a ast.Value) (ast.Value, error) { + + inputSet, err := builtins.SetOperand(a, 1) + if err != nil { + return nil, err + } + + // empty input set + if inputSet.Len() == 0 { + return ast.NewSet(), nil + } + + var result ast.Set + + err = inputSet.Iter(func(x *ast.Term) error { + n, err := builtins.SetOperand(x.Value, 1) + if err != nil { + return err + } + + if result == nil { + result = n + } else { + result = result.Intersect(n) + } + return nil + }) + return result, err +} + +// builtinSetUnion returns the union of the given input sets +func builtinSetUnion(a ast.Value) (ast.Value, error) { + + inputSet, err := builtins.SetOperand(a, 1) + if err != nil { + return nil, err + } + + result := ast.NewSet() + + err = inputSet.Iter(func(x *ast.Term) error { + n, err := builtins.SetOperand(x.Value, 1) + if err != nil { + return err + } + result = result.Union(n) + return nil + }) + return result, err +} + +func init() { + RegisterFunctionalBuiltin2(ast.SetDiff.Name, builtinSetDiff) + RegisterFunctionalBuiltin1(ast.Intersection.Name, builtinSetIntersection) + RegisterFunctionalBuiltin1(ast.Union.Name, builtinSetUnion) +} diff --git a/vendor/github.com/open-policy-agent/opa/topdown/strings.go b/vendor/github.com/open-policy-agent/opa/topdown/strings.go new file mode 100644 index 000000000..15e592763 --- /dev/null +++ b/vendor/github.com/open-policy-agent/opa/topdown/strings.go @@ -0,0 +1,393 @@ +// Copyright 2016 The OPA Authors. All rights reserved. +// Use of this source code is governed by an Apache2 +// license that can be found in the LICENSE file. + +package topdown + +import ( + "errors" + "fmt" + "strings" + + "github.com/open-policy-agent/opa/ast" + "github.com/open-policy-agent/opa/topdown/builtins" +) + +func builtinFormatInt(a, b ast.Value) (ast.Value, error) { + + input, err := builtins.NumberOperand(a, 1) + if err != nil { + return nil, err + } + + base, err := builtins.NumberOperand(b, 2) + if err != nil { + return nil, err + } + + var format string + switch base { + case ast.Number("2"): + format = "%b" + case ast.Number("8"): + format = "%o" + case ast.Number("10"): + format = "%d" + case ast.Number("16"): + format = "%x" + default: + return nil, builtins.NewOperandEnumErr(2, "2", "8", "10", "16") + } + + f := builtins.NumberToFloat(input) + i, _ := f.Int(nil) + + return ast.String(fmt.Sprintf(format, i)), nil +} + +func builtinConcat(a, b ast.Value) (ast.Value, error) { + + join, err := builtins.StringOperand(a, 1) + if err != nil { + return nil, err + } + + strs := []string{} + + switch b := b.(type) { + case ast.Array: + for i := range b { + s, ok := b[i].Value.(ast.String) + if !ok { + return nil, builtins.NewOperandElementErr(2, b, b[i].Value, "string") + } + strs = append(strs, string(s)) + } + case ast.Set: + err := b.Iter(func(x *ast.Term) error { + s, ok := x.Value.(ast.String) + if !ok { + return builtins.NewOperandElementErr(2, b, x.Value, "string") + } + strs = append(strs, string(s)) + return nil + }) + if err != nil { + return nil, err + } + default: + return nil, builtins.NewOperandTypeErr(2, b, "set", "array") + } + + return ast.String(strings.Join(strs, string(join))), nil +} + +func builtinIndexOf(a, b ast.Value) (ast.Value, error) { + base, err := builtins.StringOperand(a, 1) + if err != nil { + return nil, err + } + + search, err := builtins.StringOperand(b, 2) + if err != nil { + return nil, err + } + + index := strings.Index(string(base), string(search)) + return ast.IntNumberTerm(index).Value, nil +} + +func builtinSubstring(a, b, c ast.Value) (ast.Value, error) { + + base, err := builtins.StringOperand(a, 1) + if err != nil { + return nil, err + } + + startIndex, err := builtins.IntOperand(b, 2) + if err != nil { + return nil, err + } else if startIndex >= len(base) { + return ast.String(""), nil + } else if startIndex < 0 { + return nil, fmt.Errorf("negative offset") + } + + length, err := builtins.IntOperand(c, 3) + if err != nil { + return nil, err + } + + var s ast.String + if length < 0 { + s = ast.String(base[startIndex:]) + } else { + upto := startIndex + length + if len(base) < upto { + upto = len(base) + } + s = ast.String(base[startIndex:upto]) + } + + return s, nil +} + +func builtinContains(a, b ast.Value) (ast.Value, error) { + s, err := builtins.StringOperand(a, 1) + if err != nil { + return nil, err + } + + substr, err := builtins.StringOperand(b, 2) + if err != nil { + return nil, err + } + + return ast.Boolean(strings.Contains(string(s), string(substr))), nil +} + +func builtinStartsWith(a, b ast.Value) (ast.Value, error) { + s, err := builtins.StringOperand(a, 1) + if err != nil { + return nil, err + } + + prefix, err := builtins.StringOperand(b, 2) + if err != nil { + return nil, err + } + + return ast.Boolean(strings.HasPrefix(string(s), string(prefix))), nil +} + +func builtinEndsWith(a, b ast.Value) (ast.Value, error) { + s, err := builtins.StringOperand(a, 1) + if err != nil { + return nil, err + } + + suffix, err := builtins.StringOperand(b, 2) + if err != nil { + return nil, err + } + + return ast.Boolean(strings.HasSuffix(string(s), string(suffix))), nil +} + +func builtinLower(a ast.Value) (ast.Value, error) { + s, err := builtins.StringOperand(a, 1) + if err != nil { + return nil, err + } + + return ast.String(strings.ToLower(string(s))), nil +} + +func builtinUpper(a ast.Value) (ast.Value, error) { + s, err := builtins.StringOperand(a, 1) + if err != nil { + return nil, err + } + + return ast.String(strings.ToUpper(string(s))), nil +} + +func builtinSplit(a, b ast.Value) (ast.Value, error) { + s, err := builtins.StringOperand(a, 1) + if err != nil { + return nil, err + } + d, err := builtins.StringOperand(b, 2) + if err != nil { + return nil, err + } + elems := strings.Split(string(s), string(d)) + arr := make(ast.Array, len(elems)) + for i := range arr { + arr[i] = ast.StringTerm(elems[i]) + } + return arr, nil +} + +func builtinReplace(a, b, c ast.Value) (ast.Value, error) { + s, err := builtins.StringOperand(a, 1) + if err != nil { + return nil, err + } + + old, err := builtins.StringOperand(b, 2) + if err != nil { + return nil, err + } + + new, err := builtins.StringOperand(c, 3) + if err != nil { + return nil, err + } + + return ast.String(strings.Replace(string(s), string(old), string(new), -1)), nil +} + +func builtinReplaceN(a, b ast.Value) (ast.Value, error) { + asJSON, err := ast.JSON(a) + if err != nil { + return nil, err + } + oldnewObj, ok := asJSON.(map[string]interface{}) + if !ok { + return nil, builtins.NewOperandTypeErr(1, a, "object") + } + + s, err := builtins.StringOperand(b, 2) + if err != nil { + return nil, err + } + + var oldnewArr []string + for k, v := range oldnewObj { + strVal, ok := v.(string) + if !ok { + return nil, errors.New("non-string value found in pattern object") + } + oldnewArr = append(oldnewArr, k, strVal) + } + + r := strings.NewReplacer(oldnewArr...) + replaced := r.Replace(string(s)) + + return ast.String(replaced), nil +} + +func builtinTrim(a, b ast.Value) (ast.Value, error) { + s, err := builtins.StringOperand(a, 1) + if err != nil { + return nil, err + } + + c, err := builtins.StringOperand(b, 2) + if err != nil { + return nil, err + } + + return ast.String(strings.Trim(string(s), string(c))), nil +} + +func builtinTrimLeft(a, b ast.Value) (ast.Value, error) { + s, err := builtins.StringOperand(a, 1) + if err != nil { + return nil, err + } + + c, err := builtins.StringOperand(b, 2) + if err != nil { + return nil, err + } + + return ast.String(strings.TrimLeft(string(s), string(c))), nil +} + +func builtinTrimPrefix(a, b ast.Value) (ast.Value, error) { + s, err := builtins.StringOperand(a, 1) + if err != nil { + return nil, err + } + + pre, err := builtins.StringOperand(b, 2) + if err != nil { + return nil, err + } + + return ast.String(strings.TrimPrefix(string(s), string(pre))), nil +} + +func builtinTrimRight(a, b ast.Value) (ast.Value, error) { + s, err := builtins.StringOperand(a, 1) + if err != nil { + return nil, err + } + + c, err := builtins.StringOperand(b, 2) + if err != nil { + return nil, err + } + + return ast.String(strings.TrimRight(string(s), string(c))), nil +} + +func builtinTrimSuffix(a, b ast.Value) (ast.Value, error) { + s, err := builtins.StringOperand(a, 1) + if err != nil { + return nil, err + } + + suf, err := builtins.StringOperand(b, 2) + if err != nil { + return nil, err + } + + return ast.String(strings.TrimSuffix(string(s), string(suf))), nil +} + +func builtinTrimSpace(a ast.Value) (ast.Value, error) { + s, err := builtins.StringOperand(a, 1) + if err != nil { + return nil, err + } + + return ast.String(strings.TrimSpace(string(s))), nil +} + +func builtinSprintf(a, b ast.Value) (ast.Value, error) { + s, err := builtins.StringOperand(a, 1) + if err != nil { + return nil, err + } + + astArr, ok := b.(ast.Array) + if !ok { + return nil, builtins.NewOperandTypeErr(2, b, "array") + } + + args := make([]interface{}, len(astArr)) + + for i := range astArr { + switch v := astArr[i].Value.(type) { + case ast.Number: + if n, ok := v.Int(); ok { + args[i] = n + } else if f, ok := v.Float64(); ok { + args[i] = f + } else { + args[i] = v.String() + } + case ast.String: + args[i] = string(v) + default: + args[i] = astArr[i].String() + } + } + + return ast.String(fmt.Sprintf(string(s), args...)), nil +} + +func init() { + RegisterFunctionalBuiltin2(ast.FormatInt.Name, builtinFormatInt) + RegisterFunctionalBuiltin2(ast.Concat.Name, builtinConcat) + RegisterFunctionalBuiltin2(ast.IndexOf.Name, builtinIndexOf) + RegisterFunctionalBuiltin3(ast.Substring.Name, builtinSubstring) + RegisterFunctionalBuiltin2(ast.Contains.Name, builtinContains) + RegisterFunctionalBuiltin2(ast.StartsWith.Name, builtinStartsWith) + RegisterFunctionalBuiltin2(ast.EndsWith.Name, builtinEndsWith) + RegisterFunctionalBuiltin1(ast.Upper.Name, builtinUpper) + RegisterFunctionalBuiltin1(ast.Lower.Name, builtinLower) + RegisterFunctionalBuiltin2(ast.Split.Name, builtinSplit) + RegisterFunctionalBuiltin3(ast.Replace.Name, builtinReplace) + RegisterFunctionalBuiltin2(ast.ReplaceN.Name, builtinReplaceN) + RegisterFunctionalBuiltin2(ast.Trim.Name, builtinTrim) + RegisterFunctionalBuiltin2(ast.TrimLeft.Name, builtinTrimLeft) + RegisterFunctionalBuiltin2(ast.TrimPrefix.Name, builtinTrimPrefix) + RegisterFunctionalBuiltin2(ast.TrimRight.Name, builtinTrimRight) + RegisterFunctionalBuiltin2(ast.TrimSuffix.Name, builtinTrimSuffix) + RegisterFunctionalBuiltin1(ast.TrimSpace.Name, builtinTrimSpace) + RegisterFunctionalBuiltin2(ast.Sprintf.Name, builtinSprintf) +} diff --git a/vendor/github.com/open-policy-agent/opa/topdown/time.go b/vendor/github.com/open-policy-agent/opa/topdown/time.go new file mode 100644 index 000000000..33de89dd2 --- /dev/null +++ b/vendor/github.com/open-policy-agent/opa/topdown/time.go @@ -0,0 +1,203 @@ +// Copyright 2017 The OPA Authors. All rights reserved. +// Use of this source code is governed by an Apache2 +// license that can be found in the LICENSE file. + +package topdown + +import ( + "encoding/json" + "fmt" + "math/big" + "strconv" + "sync" + "time" + + "github.com/open-policy-agent/opa/ast" + "github.com/open-policy-agent/opa/topdown/builtins" +) + +type nowKeyID string + +var nowKey = nowKeyID("time.now_ns") +var tzCache map[string]*time.Location +var tzCacheMutex *sync.Mutex + +func builtinTimeNowNanos(bctx BuiltinContext, _ []*ast.Term, iter func(*ast.Term) error) error { + + exist, ok := bctx.Cache.Get(nowKey) + var now *ast.Term + + if !ok { + curr := time.Now() + now = ast.NewTerm(ast.Number(int64ToJSONNumber(curr.UnixNano()))) + bctx.Cache.Put(nowKey, now) + } else { + now = exist.(*ast.Term) + } + + return iter(now) +} + +func builtinTimeParseNanos(a, b ast.Value) (ast.Value, error) { + + format, err := builtins.StringOperand(a, 1) + if err != nil { + return nil, err + } + + value, err := builtins.StringOperand(b, 2) + if err != nil { + return nil, err + } + + result, err := time.Parse(string(format), string(value)) + if err != nil { + return nil, err + } + + return ast.Number(int64ToJSONNumber(result.UnixNano())), nil +} + +func builtinTimeParseRFC3339Nanos(a ast.Value) (ast.Value, error) { + + value, err := builtins.StringOperand(a, 1) + if err != nil { + return nil, err + } + + result, err := time.Parse(time.RFC3339, string(value)) + if err != nil { + return nil, err + } + + return ast.Number(int64ToJSONNumber(result.UnixNano())), nil +} +func builtinParseDurationNanos(a ast.Value) (ast.Value, error) { + + duration, err := builtins.StringOperand(a, 1) + if err != nil { + return nil, err + } + value, err := time.ParseDuration(string(duration)) + if err != nil { + return nil, err + } + return ast.Number(int64ToJSONNumber(int64(value))), nil +} + +func builtinDate(a ast.Value) (ast.Value, error) { + t, err := tzTime(a) + if err != nil { + return nil, err + } + year, month, day := t.Date() + result := ast.Array{ast.IntNumberTerm(year), ast.IntNumberTerm(int(month)), ast.IntNumberTerm(day)} + return result, nil +} + +func builtinClock(a ast.Value) (ast.Value, error) { + t, err := tzTime(a) + if err != nil { + return nil, err + } + hour, minute, second := t.Clock() + result := ast.Array{ast.IntNumberTerm(hour), ast.IntNumberTerm(minute), ast.IntNumberTerm(second)} + return result, nil +} + +func builtinWeekday(a ast.Value) (ast.Value, error) { + t, err := tzTime(a) + if err != nil { + return nil, err + } + weekday := t.Weekday().String() + return ast.String(weekday), nil +} + +func tzTime(a ast.Value) (t time.Time, err error) { + var nVal ast.Value + loc := time.UTC + + switch va := a.(type) { + case ast.Array: + + if len(va) == 0 { + return time.Time{}, builtins.NewOperandTypeErr(1, a, "either number (ns) or [number (ns), string (tz)]") + } + + nVal, err = builtins.NumberOperand(va[0].Value, 1) + if err != nil { + return time.Time{}, err + } + + if len(va) > 1 { + tzVal, err := builtins.StringOperand(va[1].Value, 1) + if err != nil { + return time.Time{}, err + } + + tzName := string(tzVal) + + switch tzName { + case "", "UTC": + // loc is already UTC + + case "Local": + loc = time.Local + + default: + var ok bool + + tzCacheMutex.Lock() + loc, ok = tzCache[tzName] + + if !ok { + loc, err = time.LoadLocation(tzName) + if err != nil { + tzCacheMutex.Unlock() + return time.Time{}, err + } + tzCache[tzName] = loc + } + tzCacheMutex.Unlock() + } + } + + case ast.Number: + nVal = a + + default: + return time.Time{}, builtins.NewOperandTypeErr(1, a, "either number (ns) or [number (ns), string (tz)]") + } + + value, err := builtins.NumberOperand(nVal, 1) + if err != nil { + return time.Time{}, err + } + + f := builtins.NumberToFloat(value) + i64, acc := f.Int64() + if acc != big.Exact { + return time.Time{}, fmt.Errorf("timestamp too big") + } + + t = time.Unix(0, i64).In(loc) + + return t, nil +} + +func int64ToJSONNumber(i int64) json.Number { + return json.Number(strconv.FormatInt(i, 10)) +} + +func init() { + RegisterBuiltinFunc(ast.NowNanos.Name, builtinTimeNowNanos) + RegisterFunctionalBuiltin1(ast.ParseRFC3339Nanos.Name, builtinTimeParseRFC3339Nanos) + RegisterFunctionalBuiltin2(ast.ParseNanos.Name, builtinTimeParseNanos) + RegisterFunctionalBuiltin1(ast.ParseDurationNanos.Name, builtinParseDurationNanos) + RegisterFunctionalBuiltin1(ast.Date.Name, builtinDate) + RegisterFunctionalBuiltin1(ast.Clock.Name, builtinClock) + RegisterFunctionalBuiltin1(ast.Weekday.Name, builtinWeekday) + tzCacheMutex = &sync.Mutex{} + tzCache = make(map[string]*time.Location) +} diff --git a/vendor/github.com/open-policy-agent/opa/topdown/tokens.go b/vendor/github.com/open-policy-agent/opa/topdown/tokens.go new file mode 100644 index 000000000..376c5cbfa --- /dev/null +++ b/vendor/github.com/open-policy-agent/opa/topdown/tokens.go @@ -0,0 +1,967 @@ +// Copyright 2017 The OPA Authors. All rights reserved. +// Use of this source code is governed by an Apache2 +// license that can be found in the LICENSE file. + +package topdown + +import ( + "crypto" + "crypto/ecdsa" + "crypto/hmac" + "crypto/rsa" + "crypto/sha256" + "crypto/x509" + "encoding/hex" + "encoding/json" + "encoding/pem" + "fmt" + "math/big" + "strconv" + "strings" + "time" + + "github.com/pkg/errors" + + "github.com/open-policy-agent/opa/ast" + "github.com/open-policy-agent/opa/topdown/builtins" + "github.com/open-policy-agent/opa/topdown/internal/jwx/jwk" + "github.com/open-policy-agent/opa/topdown/internal/jwx/jws" +) + +var ( + jwtEncKey = ast.StringTerm("enc") + jwtCtyKey = ast.StringTerm("cty") + jwtAlgKey = ast.StringTerm("alg") + jwtIssKey = ast.StringTerm("iss") + jwtExpKey = ast.StringTerm("exp") + jwtNbfKey = ast.StringTerm("nbf") + jwtAudKey = ast.StringTerm("aud") +) + +// JSONWebToken represent the 3 parts (header, payload & signature) of +// a JWT in Base64. +type JSONWebToken struct { + header string + payload string + signature string + decodedHeader ast.Object +} + +// decodeHeader populates the decodedHeader field. +func (token *JSONWebToken) decodeHeader() (err error) { + var h ast.Value + if h, err = builtinBase64UrlDecode(ast.String(token.header)); err != nil { + return fmt.Errorf("JWT header had invalid encoding: %v", err) + } + if token.decodedHeader, err = validateJWTHeader(string(h.(ast.String))); err != nil { + return err + } + return +} + +// Implements JWT decoding/validation based on RFC 7519 Section 7.2: +// https://tools.ietf.org/html/rfc7519#section-7.2 +// It does no data validation, it merely checks that the given string +// represents a structurally valid JWT. It supports JWTs using JWS compact +// serialization. +func builtinJWTDecode(a ast.Value) (ast.Value, error) { + token, err := decodeJWT(a) + if err != nil { + return nil, err + } + + if err = token.decodeHeader(); err != nil { + return nil, err + } + + p, err := builtinBase64UrlDecode(ast.String(token.payload)) + if err != nil { + return nil, fmt.Errorf("JWT payload had invalid encoding: %v", err) + } + + if cty := token.decodedHeader.Get(jwtCtyKey); cty != nil { + ctyVal := string(cty.Value.(ast.String)) + // It is possible for the contents of a token to be another + // token as a result of nested signing or encryption. To handle + // the case where we are given a token such as this, we check + // the content type and recurse on the payload if the content + // is "JWT". + // When the payload is itself another encoded JWT, then its + // contents are quoted (behavior of https://jwt.io/). To fix + // this, remove leading and trailing quotes. + if ctyVal == "JWT" { + p, err = builtinTrim(p, ast.String(`"'`)) + if err != nil { + panic("not reached") + } + return builtinJWTDecode(p) + } + } + + payload, err := extractJSONObject(string(p.(ast.String))) + if err != nil { + return nil, err + } + + s, err := builtinBase64UrlDecode(ast.String(token.signature)) + if err != nil { + return nil, fmt.Errorf("JWT signature had invalid encoding: %v", err) + } + sign := hex.EncodeToString([]byte(s.(ast.String))) + + arr := make(ast.Array, 3) + arr[0] = ast.NewTerm(token.decodedHeader) + arr[1] = ast.NewTerm(payload) + arr[2] = ast.StringTerm(sign) + + return arr, nil +} + +// Implements RS256 JWT signature verification +func builtinJWTVerifyRS256(a ast.Value, b ast.Value) (ast.Value, error) { + return builtinJWTVerifyRSA(a, b, func(publicKey *rsa.PublicKey, digest []byte, signature []byte) error { + return rsa.VerifyPKCS1v15( + publicKey, + crypto.SHA256, + digest, + signature) + }) +} + +// Implements PS256 JWT signature verification +func builtinJWTVerifyPS256(a ast.Value, b ast.Value) (ast.Value, error) { + return builtinJWTVerifyRSA(a, b, func(publicKey *rsa.PublicKey, digest []byte, signature []byte) error { + return rsa.VerifyPSS( + publicKey, + crypto.SHA256, + digest, + signature, + nil) + }) +} + +// Implements RSA JWT signature verification. +func builtinJWTVerifyRSA(a ast.Value, b ast.Value, verify func(publicKey *rsa.PublicKey, digest []byte, signature []byte) error) (ast.Value, error) { + return builtinJWTVerify(a, b, func(publicKey interface{}, digest []byte, signature []byte) error { + publicKeyRsa, ok := publicKey.(*rsa.PublicKey) + if !ok { + return fmt.Errorf("incorrect public key type") + } + return verify(publicKeyRsa, digest, signature) + }) +} + +// Implements ES256 JWT signature verification. +func builtinJWTVerifyES256(a ast.Value, b ast.Value) (ast.Value, error) { + return builtinJWTVerify(a, b, func(publicKey interface{}, digest []byte, signature []byte) error { + publicKeyEcdsa, ok := publicKey.(*ecdsa.PublicKey) + if !ok { + return fmt.Errorf("incorrect public key type") + } + r, s := &big.Int{}, &big.Int{} + n := len(signature) / 2 + r.SetBytes(signature[:n]) + s.SetBytes(signature[n:]) + if ecdsa.Verify(publicKeyEcdsa, digest, r, s) { + return nil + } + return fmt.Errorf("ECDSA signature verification error") + }) +} + +// getKeyFromCertOrJWK returns the public key found in a X.509 certificate or JWK key(s). +// A valid PEM block is never valid JSON (and vice versa), hence can try parsing both. +func getKeyFromCertOrJWK(certificate string) ([]interface{}, error) { + if block, rest := pem.Decode([]byte(certificate)); block != nil { + if len(rest) > 0 { + return nil, fmt.Errorf("extra data after a PEM certificate block") + } + + if block.Type == "CERTIFICATE" { + cert, err := x509.ParseCertificate(block.Bytes) + if err != nil { + return nil, errors.Wrap(err, "failed to parse a PEM certificate") + } + + return []interface{}{cert.PublicKey}, nil + } + + if block.Type == "PUBLIC KEY" { + key, err := x509.ParsePKIXPublicKey(block.Bytes) + if err != nil { + return nil, errors.Wrap(err, "failed to parse a PEM public key") + } + + return []interface{}{key}, nil + } + + return nil, fmt.Errorf("failed to extract a Key from the PEM certificate") + } + + jwks, err := jwk.ParseString(certificate) + if err != nil { + return nil, errors.Wrap(err, "failed to parse a JWK key (set)") + } + + var keys []interface{} + for _, k := range jwks.Keys { + key, err := k.Materialize() + if err != nil { + return nil, err + } + keys = append(keys, key) + } + + return keys, nil +} + +// Implements JWT signature verification. +func builtinJWTVerify(a ast.Value, b ast.Value, verify func(publicKey interface{}, digest []byte, signature []byte) error) (ast.Value, error) { + token, err := decodeJWT(a) + if err != nil { + return nil, err + } + + s, err := builtins.StringOperand(b, 2) + if err != nil { + return nil, err + } + + keys, err := getKeyFromCertOrJWK(string(s)) + if err != nil { + return nil, err + } + + signature, err := token.decodeSignature() + if err != nil { + return nil, err + } + + // Validate the JWT signature + for _, key := range keys { + err = verify(key, + getInputSHA([]byte(token.header+"."+token.payload)), + []byte(signature)) + + if err == nil { + return ast.Boolean(true), nil + } + } + + // None of the keys worked, return false + return ast.Boolean(false), nil +} + +// Implements HS256 (secret) JWT signature verification +func builtinJWTVerifyHS256(a ast.Value, b ast.Value) (ast.Value, error) { + // Decode the JSON Web Token + token, err := decodeJWT(a) + if err != nil { + return nil, err + } + + // Process Secret input + astSecret, err := builtins.StringOperand(b, 2) + if err != nil { + return nil, err + } + secret := string(astSecret) + + mac := hmac.New(sha256.New, []byte(secret)) + _, err = mac.Write([]byte(token.header + "." + token.payload)) + if err != nil { + return nil, err + } + + signature, err := token.decodeSignature() + if err != nil { + return nil, err + } + + return ast.Boolean(hmac.Equal([]byte(signature), mac.Sum(nil))), nil +} + +// -- Full JWT verification and decoding -- + +// Verification constraints. See tokens_test.go for unit tests. + +// tokenConstraints holds decoded JWT verification constraints. +type tokenConstraints struct { + // The set of asymmetric keys we can verify with. + keys []interface{} + + // The single symmetric key we will verify with. + secret string + + // The algorithm that must be used to verify. + // If "", any algorithm is acceptable. + alg string + + // The required issuer. + // If "", any issuer is acceptable. + iss string + + // The required audience. + // If "", no audience is acceptable. + aud string + + // The time to validate against, or -1 if no constraint set. + // (If unset, the current time will be used.) + time int64 +} + +// tokenConstraintHandler is the handler type for JWT verification constraints. +type tokenConstraintHandler func(value ast.Value, parameters *tokenConstraints) (err error) + +// tokenConstraintTypes maps known JWT verification constraints to handlers. +var tokenConstraintTypes = map[string]tokenConstraintHandler{ + "cert": tokenConstraintCert, + "secret": func(value ast.Value, constraints *tokenConstraints) (err error) { + return tokenConstraintString("secret", value, &constraints.secret) + }, + "alg": func(value ast.Value, constraints *tokenConstraints) (err error) { + return tokenConstraintString("alg", value, &constraints.alg) + }, + "iss": func(value ast.Value, constraints *tokenConstraints) (err error) { + return tokenConstraintString("iss", value, &constraints.iss) + }, + "aud": func(value ast.Value, constraints *tokenConstraints) (err error) { + return tokenConstraintString("aud", value, &constraints.aud) + }, + "time": tokenConstraintTime, +} + +// tokenConstraintCert handles the `cert` constraint. +func tokenConstraintCert(value ast.Value, constraints *tokenConstraints) (err error) { + var s ast.String + var ok bool + if s, ok = value.(ast.String); !ok { + return fmt.Errorf("cert constraint: must be a string") + } + + constraints.keys, err = getKeyFromCertOrJWK(string(s)) + return +} + +// tokenConstraintTime handles the `time` constraint. +func tokenConstraintTime(value ast.Value, constraints *tokenConstraints) (err error) { + var time ast.Number + var ok bool + if time, ok = value.(ast.Number); !ok { + err = fmt.Errorf("token time constraint: must be a number") + return + } + var timeFloat float64 + if timeFloat, err = strconv.ParseFloat(string(time), 64); err != nil { + err = fmt.Errorf("token time constraint: %v", err) + return + } + if timeFloat < 0 { + err = fmt.Errorf("token time constraint: must not be negative") + return + } + constraints.time = int64(timeFloat) + return +} + +// tokenConstraintString handles string constraints. +func tokenConstraintString(name string, value ast.Value, where *string) (err error) { + var av ast.String + var ok bool + if av, ok = value.(ast.String); !ok { + err = fmt.Errorf("%s constraint: must be a string", name) + return + } + *where = string(av) + return +} + +// parseTokenConstraints parses the constraints argument. +func parseTokenConstraints(a ast.Value) (constraints tokenConstraints, err error) { + constraints.time = -1 + var o ast.Object + var ok bool + if o, ok = a.(ast.Object); !ok { + err = fmt.Errorf("token constraints must be object") + return + } + if err = o.Iter(func(k *ast.Term, v *ast.Term) (err error) { + var handler tokenConstraintHandler + var ok bool + name := string(k.Value.(ast.String)) + if handler, ok = tokenConstraintTypes[name]; ok { + if err = handler(v.Value, &constraints); err != nil { + return + } + } else { + // Anything unknown is rejected. + err = fmt.Errorf("unknown token validation constraint: %s", name) + return + } + return + }); err != nil { + return + } + return +} + +// validate validates the constraints argument. +func (constraints *tokenConstraints) validate() (err error) { + keys := 0 + if constraints.keys != nil { + keys++ + } + if constraints.secret != "" { + keys++ + } + if keys > 1 { + err = fmt.Errorf("duplicate key constraints") + return + } + if keys < 1 { + err = fmt.Errorf("no key constraint") + return + } + return +} + +// verify verifies a JWT using the constraints and the algorithm from the header +func (constraints *tokenConstraints) verify(kid, alg, header, payload, signature string) error { + // Construct the payload + plaintext := []byte(header) + plaintext = append(plaintext, []byte(".")...) + plaintext = append(plaintext, payload...) + // Look up the algorithm + var ok bool + var a tokenAlgorithm + a, ok = tokenAlgorithms[alg] + if !ok { + return fmt.Errorf("unknown JWS algorithm: %s", alg) + } + // If we're configured with asymmetric key(s) then only trust that + if constraints.keys != nil { + verified := false + for _, key := range constraints.keys { + err := a.verify(key, a.hash, plaintext, []byte(signature)) + if err == nil { + verified = true + break + } + } + if !verified { + return errSignatureNotVerified + } + return nil + } + if constraints.secret != "" { + return a.verify([]byte(constraints.secret), a.hash, plaintext, []byte(signature)) + } + // (*tokenConstraints)validate() should prevent this happening + return errors.New("unexpectedly found no keys to trust") +} + +// validAudience checks the audience of the JWT. +// It returns true if it meets the constraints and false otherwise. +func (constraints *tokenConstraints) validAudience(aud ast.Value) (valid bool) { + var ok bool + var s ast.String + if s, ok = aud.(ast.String); ok { + return string(s) == constraints.aud + } + var a ast.Array + if a, ok = aud.(ast.Array); ok { + for _, t := range a { + if s, ok = t.Value.(ast.String); ok { + if string(s) == constraints.aud { + return true + } + } else { + // Ill-formed aud claim + return false + } + } + } + return false +} + +// JWT algorithms + +type tokenVerifyFunction func(key interface{}, hash crypto.Hash, payload []byte, signature []byte) (err error) +type tokenVerifyAsymmetricFunction func(key interface{}, hash crypto.Hash, digest []byte, signature []byte) (err error) + +// jwtAlgorithm describes a JWS 'alg' value +type tokenAlgorithm struct { + hash crypto.Hash + verify tokenVerifyFunction +} + +// tokenAlgorithms is the known JWT algorithms +var tokenAlgorithms = map[string]tokenAlgorithm{ + "RS256": {crypto.SHA256, verifyAsymmetric(verifyRSAPKCS)}, + "RS384": {crypto.SHA384, verifyAsymmetric(verifyRSAPKCS)}, + "RS512": {crypto.SHA512, verifyAsymmetric(verifyRSAPKCS)}, + "PS256": {crypto.SHA256, verifyAsymmetric(verifyRSAPSS)}, + "PS384": {crypto.SHA384, verifyAsymmetric(verifyRSAPSS)}, + "PS512": {crypto.SHA512, verifyAsymmetric(verifyRSAPSS)}, + "ES256": {crypto.SHA256, verifyAsymmetric(verifyECDSA)}, + "ES384": {crypto.SHA384, verifyAsymmetric(verifyECDSA)}, + "ES512": {crypto.SHA512, verifyAsymmetric(verifyECDSA)}, + "HS256": {crypto.SHA256, verifyHMAC}, + "HS384": {crypto.SHA384, verifyHMAC}, + "HS512": {crypto.SHA512, verifyHMAC}, +} + +// errSignatureNotVerified is returned when a signature cannot be verified. +var errSignatureNotVerified = errors.New("signature not verified") + +func verifyHMAC(key interface{}, hash crypto.Hash, payload []byte, signature []byte) (err error) { + macKey, ok := key.([]byte) + if !ok { + return fmt.Errorf("incorrect symmetric key type") + } + mac := hmac.New(hash.New, macKey) + if _, err = mac.Write([]byte(payload)); err != nil { + return + } + if !hmac.Equal(signature, mac.Sum([]byte{})) { + err = errSignatureNotVerified + } + return +} + +func verifyAsymmetric(verify tokenVerifyAsymmetricFunction) tokenVerifyFunction { + return func(key interface{}, hash crypto.Hash, payload []byte, signature []byte) (err error) { + h := hash.New() + h.Write(payload) + return verify(key, hash, h.Sum([]byte{}), signature) + } +} + +func verifyRSAPKCS(key interface{}, hash crypto.Hash, digest []byte, signature []byte) (err error) { + publicKeyRsa, ok := key.(*rsa.PublicKey) + if !ok { + return fmt.Errorf("incorrect public key type") + } + if err = rsa.VerifyPKCS1v15(publicKeyRsa, hash, digest, signature); err != nil { + err = errSignatureNotVerified + } + return +} + +func verifyRSAPSS(key interface{}, hash crypto.Hash, digest []byte, signature []byte) (err error) { + publicKeyRsa, ok := key.(*rsa.PublicKey) + if !ok { + return fmt.Errorf("incorrect public key type") + } + if err = rsa.VerifyPSS(publicKeyRsa, hash, digest, signature, nil); err != nil { + err = errSignatureNotVerified + } + return +} + +func verifyECDSA(key interface{}, hash crypto.Hash, digest []byte, signature []byte) (err error) { + publicKeyEcdsa, ok := key.(*ecdsa.PublicKey) + if !ok { + return fmt.Errorf("incorrect public key type") + } + r, s := &big.Int{}, &big.Int{} + n := len(signature) / 2 + r.SetBytes(signature[:n]) + s.SetBytes(signature[n:]) + if ecdsa.Verify(publicKeyEcdsa, digest, r, s) { + return nil + } + return errSignatureNotVerified +} + +// JWT header parsing and parameters. See tokens_test.go for unit tests. + +// tokenHeaderType represents a recognized JWT header field +// tokenHeader is a parsed JWT header +type tokenHeader struct { + alg string + kid string + typ string + cty string + crit map[string]bool + unknown []string +} + +// tokenHeaderHandler handles a JWT header parameters +type tokenHeaderHandler func(header *tokenHeader, value ast.Value) (err error) + +// tokenHeaderTypes maps known JWT header parameters to handlers +var tokenHeaderTypes = map[string]tokenHeaderHandler{ + "alg": func(header *tokenHeader, value ast.Value) (err error) { + return tokenHeaderString("alg", &header.alg, value) + }, + "kid": func(header *tokenHeader, value ast.Value) (err error) { + return tokenHeaderString("kid", &header.kid, value) + }, + "typ": func(header *tokenHeader, value ast.Value) (err error) { + return tokenHeaderString("typ", &header.typ, value) + }, + "cty": func(header *tokenHeader, value ast.Value) (err error) { + return tokenHeaderString("cty", &header.cty, value) + }, + "crit": tokenHeaderCrit, +} + +// tokenHeaderCrit handles the 'crit' header parameter +func tokenHeaderCrit(header *tokenHeader, value ast.Value) (err error) { + var ok bool + var v ast.Array + if v, ok = value.(ast.Array); !ok { + err = fmt.Errorf("crit: must be a list") + return + } + header.crit = map[string]bool{} + for _, t := range v { + var tv ast.String + if tv, ok = t.Value.(ast.String); !ok { + err = fmt.Errorf("crit: must be a list of strings") + return + } + header.crit[string(tv)] = true + } + if len(header.crit) == 0 { + err = fmt.Errorf("crit: must be a nonempty list") // 'MUST NOT' use the empty list + return + } + return +} + +// tokenHeaderString handles string-format JWT header parameters +func tokenHeaderString(name string, where *string, value ast.Value) (err error) { + var ok bool + var v ast.String + if v, ok = value.(ast.String); !ok { + err = fmt.Errorf("%s: must be a string", name) + return + } + *where = string(v) + return +} + +// parseTokenHeader parses the JWT header. +func parseTokenHeader(token *JSONWebToken) (header tokenHeader, err error) { + header.unknown = []string{} + if err = token.decodedHeader.Iter(func(k *ast.Term, v *ast.Term) (err error) { + ks := string(k.Value.(ast.String)) + var ok bool + var handler tokenHeaderHandler + if handler, ok = tokenHeaderTypes[ks]; ok { + if err = handler(&header, v.Value); err != nil { + return + } + } else { + header.unknown = append(header.unknown, ks) + } + return + }); err != nil { + return + } + return +} + +// validTokenHeader returns true if the JOSE header is valid, otherwise false. +func (header *tokenHeader) valid() bool { + // RFC7515 s4.1.1 alg MUST be present + if header.alg == "" { + return false + } + // RFC7515 4.1.11 JWS is invalid if there is a critical parameter that we did not recognize + for _, u := range header.unknown { + if header.crit[u] { + return false + } + } + return true +} + +func commonBuiltinJWTEncodeSign(inputHeaders, jwsPayload, jwkSrc string) (v ast.Value, err error) { + + keys, err := jwk.ParseString(jwkSrc) + if err != nil { + return nil, err + } + key, err := keys.Keys[0].Materialize() + if err != nil { + return nil, err + } + if jwk.GetKeyTypeFromKey(key) != keys.Keys[0].GetKeyType() { + return nil, fmt.Errorf("JWK derived key type and keyType parameter do not match") + } + + standardHeaders := &jws.StandardHeaders{} + jwsHeaders := []byte(inputHeaders) + err = json.Unmarshal(jwsHeaders, standardHeaders) + if err != nil { + return nil, err + } + alg := standardHeaders.GetAlgorithm() + + if (standardHeaders.Type == "" || standardHeaders.Type == "JWT") && !json.Valid([]byte(jwsPayload)) { + return nil, fmt.Errorf("type is JWT but payload is not JSON") + } + + // process payload and sign + var jwsCompact []byte + jwsCompact, err = jws.SignLiteral([]byte(jwsPayload), alg, key, jwsHeaders) + if err != nil { + return nil, err + } + return ast.String(jwsCompact[:]), nil + +} + +func builtinJWTEncodeSign(a ast.Value, b ast.Value, c ast.Value) (v ast.Value, err error) { + + jwkSrc := c.String() + + inputHeaders := a.String() + + jwsPayload := b.String() + + return commonBuiltinJWTEncodeSign(inputHeaders, jwsPayload, jwkSrc) + +} + +func builtinJWTEncodeSignRaw(a ast.Value, b ast.Value, c ast.Value) (v ast.Value, err error) { + + jwkSrc, err := builtins.StringOperand(c, 1) + if err != nil { + return nil, err + } + inputHeaders, err := builtins.StringOperand(a, 1) + if err != nil { + return nil, err + } + jwsPayload, err := builtins.StringOperand(b, 1) + if err != nil { + return nil, err + } + return commonBuiltinJWTEncodeSign(string(inputHeaders), string(jwsPayload), string(jwkSrc)) +} + +// Implements full JWT decoding, validation and verification. +func builtinJWTDecodeVerify(a ast.Value, b ast.Value) (v ast.Value, err error) { + // io.jwt.decode_verify(string, constraints, [valid, header, payload]) + // + // If valid is true then the signature verifies and all constraints are met. + // If valid is false then either the signature did not verify or some constrain + // was not met. + // + // Decoding errors etc are returned as errors. + arr := make(ast.Array, 3) + arr[0] = ast.BooleanTerm(false) // by default, not verified + arr[1] = ast.NewTerm(ast.NewObject()) + arr[2] = ast.NewTerm(ast.NewObject()) + var constraints tokenConstraints + if constraints, err = parseTokenConstraints(b); err != nil { + return + } + if err = constraints.validate(); err != nil { + return + } + var token *JSONWebToken + var p ast.Value + for { + // RFC7519 7.2 #1-2 split into parts + if token, err = decodeJWT(a); err != nil { + return + } + // RFC7519 7.2 #3, #4, #6 + if err = token.decodeHeader(); err != nil { + return + } + // RFC7159 7.2 #5 (and RFC7159 5.2 #5) validate header fields + var header tokenHeader + if header, err = parseTokenHeader(token); err != nil { + return + } + if !header.valid() { + return arr, nil + } + // Check constraints that impact signature verification. + if constraints.alg != "" && constraints.alg != header.alg { + return arr, nil + } + // RFC7159 7.2 #7 verify the signature + var signature string + if signature, err = token.decodeSignature(); err != nil { + return + } + if err = constraints.verify(header.kid, header.alg, token.header, token.payload, signature); err != nil { + if err == errSignatureNotVerified { + return arr, nil + } + return + } + // RFC7159 7.2 #9-10 decode the payload + if p, err = builtinBase64UrlDecode(ast.String(token.payload)); err != nil { + return nil, fmt.Errorf("JWT payload had invalid encoding: %v", err) + } + // RFC7159 7.2 #8 and 5.2 cty + if strings.ToUpper(header.cty) == "JWT" { + // Nested JWT, go round again + a = p + continue + } else { + // Non-nested JWT (or we've reached the bottom of the nesting). + break + } + } + var payload ast.Object + if payload, err = extractJSONObject(string(p.(ast.String))); err != nil { + return + } + // Check registered claim names against constraints or environment + // RFC7159 4.1.1 iss + if constraints.iss != "" { + if iss := payload.Get(jwtIssKey); iss != nil { + issVal := string(iss.Value.(ast.String)) + if constraints.iss != issVal { + return arr, nil + } + } + } + // RFC7159 4.1.3 aud + if aud := payload.Get(jwtAudKey); aud != nil { + if !constraints.validAudience(aud.Value) { + return arr, nil + } + } else { + if constraints.aud != "" { + return arr, nil + } + } + // RFC7159 4.1.4 exp + if exp := payload.Get(jwtExpKey); exp != nil { + var expVal int64 + if expVal, err = strconv.ParseInt(string(exp.Value.(ast.Number)), 10, 64); err != nil { + err = fmt.Errorf("parsing 'exp' JWT claim: %v", err) + return + } + if constraints.time < 0 { + constraints.time = time.Now().UnixNano() + } + // constraints.time is in nanoseconds but expVal is in seconds + if constraints.time/1000000000 >= expVal { + return arr, nil + } + } + // RFC7159 4.1.5 nbf + if nbf := payload.Get(jwtNbfKey); nbf != nil { + var nbfVal int64 + if nbfVal, err = strconv.ParseInt(string(nbf.Value.(ast.Number)), 10, 64); err != nil { + err = fmt.Errorf("parsing 'nbf' JWT claim: %v", err) + return + } + if constraints.time < 0 { + constraints.time = time.Now().UnixNano() + } + // constraints.time is in nanoseconds but nbfVal is in seconds + if constraints.time/1000000000 < nbfVal { + return arr, nil + } + } + // Format the result + arr[0] = ast.BooleanTerm(true) + arr[1] = ast.NewTerm(token.decodedHeader) + arr[2] = ast.NewTerm(payload) + return arr, nil +} + +// -- Utilities -- + +func decodeJWT(a ast.Value) (*JSONWebToken, error) { + // Parse the JSON Web Token + astEncode, err := builtins.StringOperand(a, 1) + if err != nil { + return nil, err + } + + encoding := string(astEncode) + if !strings.Contains(encoding, ".") { + return nil, errors.New("encoded JWT had no period separators") + } + + parts := strings.Split(encoding, ".") + if len(parts) != 3 { + return nil, fmt.Errorf("encoded JWT must have 3 sections, found %d", len(parts)) + } + + return &JSONWebToken{header: parts[0], payload: parts[1], signature: parts[2]}, nil +} + +func (token *JSONWebToken) decodeSignature() (string, error) { + decodedSignature, err := builtinBase64UrlDecode(ast.String(token.signature)) + if err != nil { + return "", err + } + + signatureAst, err := builtins.StringOperand(decodedSignature, 1) + if err != nil { + return "", err + } + return string(signatureAst), err +} + +// Extract, validate and return the JWT header as an ast.Object. +func validateJWTHeader(h string) (ast.Object, error) { + header, err := extractJSONObject(h) + if err != nil { + return nil, fmt.Errorf("bad JWT header: %v", err) + } + + // There are two kinds of JWT tokens, a JSON Web Signature (JWS) and + // a JSON Web Encryption (JWE). The latter is very involved, and we + // won't support it for now. + // This code checks which kind of JWT we are dealing with according to + // RFC 7516 Section 9: https://tools.ietf.org/html/rfc7516#section-9 + if header.Get(jwtEncKey) != nil { + return nil, errors.New("JWT is a JWE object, which is not supported") + } + + return header, nil +} + +func extractJSONObject(s string) (ast.Object, error) { + // XXX: This code relies on undocumented behavior of Go's + // json.Unmarshal using the last occurrence of duplicate keys in a JSON + // Object. If duplicate keys are present in a JWT, the last must be + // used or the token rejected. Since detecting duplicates is tantamount + // to parsing it ourselves, we're relying on the Go implementation + // using the last occurring instance of the key, which is the behavior + // as of Go 1.8.1. + v, err := builtinJSONUnmarshal(ast.String(s)) + if err != nil { + return nil, fmt.Errorf("invalid JSON: %v", err) + } + + o, ok := v.(ast.Object) + if !ok { + return nil, errors.New("decoded JSON type was not an Object") + } + + return o, nil +} + +// getInputSha returns the SHA256 checksum of the input +func getInputSHA(input []byte) (hash []byte) { + hasher := sha256.New() + hasher.Write(input) + return hasher.Sum(nil) +} + +func init() { + RegisterFunctionalBuiltin1(ast.JWTDecode.Name, builtinJWTDecode) + RegisterFunctionalBuiltin2(ast.JWTVerifyRS256.Name, builtinJWTVerifyRS256) + RegisterFunctionalBuiltin2(ast.JWTVerifyPS256.Name, builtinJWTVerifyPS256) + RegisterFunctionalBuiltin2(ast.JWTVerifyES256.Name, builtinJWTVerifyES256) + RegisterFunctionalBuiltin2(ast.JWTVerifyHS256.Name, builtinJWTVerifyHS256) + RegisterFunctionalBuiltin2(ast.JWTDecodeVerify.Name, builtinJWTDecodeVerify) + RegisterFunctionalBuiltin3(ast.JWTEncodeSignRaw.Name, builtinJWTEncodeSignRaw) + RegisterFunctionalBuiltin3(ast.JWTEncodeSign.Name, builtinJWTEncodeSign) +} diff --git a/vendor/github.com/open-policy-agent/opa/topdown/trace.go b/vendor/github.com/open-policy-agent/opa/topdown/trace.go new file mode 100644 index 000000000..ba8fc925b --- /dev/null +++ b/vendor/github.com/open-policy-agent/opa/topdown/trace.go @@ -0,0 +1,304 @@ +// Copyright 2016 The OPA Authors. All rights reserved. +// Use of this source code is governed by an Apache2 +// license that can be found in the LICENSE file. + +package topdown + +import ( + "fmt" + "io" + "strings" + + "github.com/open-policy-agent/opa/ast" + "github.com/open-policy-agent/opa/topdown/builtins" +) + +// Op defines the types of tracing events. +type Op string + +const ( + // EnterOp is emitted when a new query is about to be evaluated. + EnterOp Op = "Enter" + + // ExitOp is emitted when a query has evaluated to true. + ExitOp Op = "Exit" + + // EvalOp is emitted when an expression is about to be evaluated. + EvalOp Op = "Eval" + + // RedoOp is emitted when an expression, rule, or query is being re-evaluated. + RedoOp Op = "Redo" + + // SaveOp is emitted when an expression is saved instead of evaluated + // during partial evaluation. + SaveOp Op = "Save" + + // FailOp is emitted when an expression evaluates to false. + FailOp Op = "Fail" + + // NoteOp is emitted when an expression invokes a tracing built-in function. + NoteOp Op = "Note" + + // IndexOp is emitted during an expression evaluation to represent lookup + // matches. + IndexOp Op = "Index" +) + +// VarMetadata provides some user facing information about +// a variable in some policy. +type VarMetadata struct { + Name ast.Var `json:"name"` + Location *ast.Location `json:"location"` +} + +// Event contains state associated with a tracing event. +type Event struct { + Op Op // Identifies type of event. + Node ast.Node // Contains AST node relevant to the event. + Location *ast.Location // The location of the Node this event relates to. + QueryID uint64 // Identifies the query this event belongs to. + ParentID uint64 // Identifies the parent query this event belongs to. + Locals *ast.ValueMap // Contains local variable bindings from the query context. + LocalMetadata map[ast.Var]VarMetadata // Contains metadata for the local variable bindings. + Message string // Contains message for Note events. +} + +// HasRule returns true if the Event contains an ast.Rule. +func (evt *Event) HasRule() bool { + _, ok := evt.Node.(*ast.Rule) + return ok +} + +// HasBody returns true if the Event contains an ast.Body. +func (evt *Event) HasBody() bool { + _, ok := evt.Node.(ast.Body) + return ok +} + +// HasExpr returns true if the Event contains an ast.Expr. +func (evt *Event) HasExpr() bool { + _, ok := evt.Node.(*ast.Expr) + return ok +} + +// Equal returns true if this event is equal to the other event. +func (evt *Event) Equal(other *Event) bool { + if evt.Op != other.Op { + return false + } + if evt.QueryID != other.QueryID { + return false + } + if evt.ParentID != other.ParentID { + return false + } + if !evt.equalNodes(other) { + return false + } + return evt.Locals.Equal(other.Locals) +} + +func (evt *Event) String() string { + return fmt.Sprintf("%v %v %v (qid=%v, pqid=%v)", evt.Op, evt.Node, evt.Locals, evt.QueryID, evt.ParentID) +} + +func (evt *Event) equalNodes(other *Event) bool { + switch a := evt.Node.(type) { + case ast.Body: + if b, ok := other.Node.(ast.Body); ok { + return a.Equal(b) + } + case *ast.Rule: + if b, ok := other.Node.(*ast.Rule); ok { + return a.Equal(b) + } + case *ast.Expr: + if b, ok := other.Node.(*ast.Expr); ok { + return a.Equal(b) + } + case nil: + return other.Node == nil + } + return false +} + +// Tracer defines the interface for tracing in the top-down evaluation engine. +type Tracer interface { + Enabled() bool + Trace(*Event) +} + +// BufferTracer implements the Tracer interface by simply buffering all events +// received. +type BufferTracer []*Event + +// NewBufferTracer returns a new BufferTracer. +func NewBufferTracer() *BufferTracer { + return &BufferTracer{} +} + +// Enabled always returns true if the BufferTracer is instantiated. +func (b *BufferTracer) Enabled() bool { + if b == nil { + return false + } + return true +} + +// Trace adds the event to the buffer. +func (b *BufferTracer) Trace(evt *Event) { + *b = append(*b, evt) +} + +// PrettyTrace pretty prints the trace to the writer. +func PrettyTrace(w io.Writer, trace []*Event) { + depths := depths{} + for _, event := range trace { + depth := depths.GetOrSet(event.QueryID, event.ParentID) + fmt.Fprintln(w, formatEvent(event, depth)) + } +} + +// PrettyTraceWithLocation prints the trace to the writer and includes location information +func PrettyTraceWithLocation(w io.Writer, trace []*Event) { + depths := depths{} + for _, event := range trace { + depth := depths.GetOrSet(event.QueryID, event.ParentID) + location := formatLocation(event) + fmt.Fprintln(w, fmt.Sprintf("%v %v", location, formatEvent(event, depth))) + } +} + +func formatEvent(event *Event, depth int) string { + padding := formatEventPadding(event, depth) + if event.Op == NoteOp { + return fmt.Sprintf("%v%v %q", padding, event.Op, event.Message) + } else if event.Message != "" { + return fmt.Sprintf("%v%v %v %v", padding, event.Op, event.Node, event.Message) + } else { + switch node := event.Node.(type) { + case *ast.Rule: + return fmt.Sprintf("%v%v %v", padding, event.Op, node.Path()) + default: + return fmt.Sprintf("%v%v %v", padding, event.Op, rewrite(event).Node) + } + } +} + +func formatEventPadding(event *Event, depth int) string { + spaces := formatEventSpaces(event, depth) + padding := "" + if spaces > 1 { + padding += strings.Repeat("| ", spaces-1) + } + return padding +} + +func formatEventSpaces(event *Event, depth int) int { + switch event.Op { + case EnterOp: + return depth + case RedoOp: + if _, ok := event.Node.(*ast.Expr); !ok { + return depth + } + } + return depth + 1 +} + +func formatLocation(event *Event) string { + if event.Op == NoteOp { + return fmt.Sprintf("%-19v", "note") + } + + location := event.Location + if location == nil { + return fmt.Sprintf("%-19v", "") + } + + if location.File == "" { + return fmt.Sprintf("%-19v", fmt.Sprintf("%.15v:%v", "query", location.Row)) + } + + return fmt.Sprintf("%-19v", fmt.Sprintf("%.15v:%v", location.File, location.Row)) +} + +// depths is a helper for computing the depth of an event. Events within the +// same query all have the same depth. The depth of query is +// depth(parent(query))+1. +type depths map[uint64]int + +func (ds depths) GetOrSet(qid uint64, pqid uint64) int { + depth := ds[qid] + if depth == 0 { + depth = ds[pqid] + depth++ + ds[qid] = depth + } + return depth +} + +func builtinTrace(bctx BuiltinContext, args []*ast.Term, iter func(*ast.Term) error) error { + + str, err := builtins.StringOperand(args[0].Value, 1) + if err != nil { + return handleBuiltinErr(ast.Trace.Name, bctx.Location, err) + } + + if !traceIsEnabled(bctx.Tracers) { + return iter(ast.BooleanTerm(true)) + } + + evt := &Event{ + Op: NoteOp, + QueryID: bctx.QueryID, + ParentID: bctx.ParentID, + Message: string(str), + } + + for i := range bctx.Tracers { + bctx.Tracers[i].Trace(evt) + } + + return iter(ast.BooleanTerm(true)) +} + +func traceIsEnabled(tracers []Tracer) bool { + for i := range tracers { + if tracers[i].Enabled() { + return true + } + } + return false +} + +func rewrite(event *Event) *Event { + + cpy := *event + + var node ast.Node + + switch v := event.Node.(type) { + case *ast.Expr: + node = v.Copy() + case ast.Body: + node = v.Copy() + case *ast.Rule: + node = v.Copy() + } + + ast.TransformVars(node, func(v ast.Var) (ast.Value, error) { + if meta, ok := cpy.LocalMetadata[v]; ok { + return meta.Name, nil + } + return v, nil + }) + + cpy.Node = node + + return &cpy +} + +func init() { + RegisterBuiltinFunc(ast.Trace.Name, builtinTrace) +} diff --git a/vendor/github.com/open-policy-agent/opa/topdown/type.go b/vendor/github.com/open-policy-agent/opa/topdown/type.go new file mode 100644 index 000000000..a4c31fc83 --- /dev/null +++ b/vendor/github.com/open-policy-agent/opa/topdown/type.go @@ -0,0 +1,82 @@ +// Copyright 2018 The OPA Authors. All rights reserved. +// Use of this source code is governed by an Apache2 +// license that can be found in the LICENSE file. + +package topdown + +import ( + "github.com/open-policy-agent/opa/ast" +) + +func builtinIsNumber(a ast.Value) (ast.Value, error) { + switch a.(type) { + case ast.Number: + return ast.Boolean(true), nil + default: + return nil, BuiltinEmpty{} + } +} + +func builtinIsString(a ast.Value) (ast.Value, error) { + switch a.(type) { + case ast.String: + return ast.Boolean(true), nil + default: + return nil, BuiltinEmpty{} + } +} + +func builtinIsBoolean(a ast.Value) (ast.Value, error) { + switch a.(type) { + case ast.Boolean: + return ast.Boolean(true), nil + default: + return nil, BuiltinEmpty{} + } +} + +func builtinIsArray(a ast.Value) (ast.Value, error) { + switch a.(type) { + case ast.Array: + return ast.Boolean(true), nil + default: + return nil, BuiltinEmpty{} + } +} + +func builtinIsSet(a ast.Value) (ast.Value, error) { + switch a.(type) { + case ast.Set: + return ast.Boolean(true), nil + default: + return nil, BuiltinEmpty{} + } +} + +func builtinIsObject(a ast.Value) (ast.Value, error) { + switch a.(type) { + case ast.Object: + return ast.Boolean(true), nil + default: + return nil, BuiltinEmpty{} + } +} + +func builtinIsNull(a ast.Value) (ast.Value, error) { + switch a.(type) { + case ast.Null: + return ast.Boolean(true), nil + default: + return nil, BuiltinEmpty{} + } +} + +func init() { + RegisterFunctionalBuiltin1(ast.IsNumber.Name, builtinIsNumber) + RegisterFunctionalBuiltin1(ast.IsString.Name, builtinIsString) + RegisterFunctionalBuiltin1(ast.IsBoolean.Name, builtinIsBoolean) + RegisterFunctionalBuiltin1(ast.IsArray.Name, builtinIsArray) + RegisterFunctionalBuiltin1(ast.IsSet.Name, builtinIsSet) + RegisterFunctionalBuiltin1(ast.IsObject.Name, builtinIsObject) + RegisterFunctionalBuiltin1(ast.IsNull.Name, builtinIsNull) +} diff --git a/vendor/github.com/open-policy-agent/opa/topdown/type_name.go b/vendor/github.com/open-policy-agent/opa/topdown/type_name.go new file mode 100644 index 000000000..5a77f5d31 --- /dev/null +++ b/vendor/github.com/open-policy-agent/opa/topdown/type_name.go @@ -0,0 +1,36 @@ +// Copyright 2018 The OPA Authors. All rights reserved. +// Use of this source code is governed by an Apache2 +// license that can be found in the LICENSE file. + +package topdown + +import ( + "fmt" + + "github.com/open-policy-agent/opa/ast" +) + +func builtinTypeName(a ast.Value) (ast.Value, error) { + switch a.(type) { + case ast.Null: + return ast.String("null"), nil + case ast.Boolean: + return ast.String("boolean"), nil + case ast.Number: + return ast.String("number"), nil + case ast.String: + return ast.String("string"), nil + case ast.Array: + return ast.String("array"), nil + case ast.Object: + return ast.String("object"), nil + case ast.Set: + return ast.String("set"), nil + } + + return nil, fmt.Errorf("illegal value") +} + +func init() { + RegisterFunctionalBuiltin1(ast.TypeNameBuiltin.Name, builtinTypeName) +} diff --git a/vendor/github.com/open-policy-agent/opa/topdown/walk.go b/vendor/github.com/open-policy-agent/opa/topdown/walk.go new file mode 100644 index 000000000..e40de8a12 --- /dev/null +++ b/vendor/github.com/open-policy-agent/opa/topdown/walk.go @@ -0,0 +1,84 @@ +// Copyright 2017 The OPA Authors. All rights reserved. +// Use of this source code is governed by an Apache2 +// license that can be found in the LICENSE file. + +package topdown + +import ( + "github.com/open-policy-agent/opa/ast" +) + +func evalWalk(bctx BuiltinContext, args []*ast.Term, iter func(*ast.Term) error) error { + input := args[0] + filter := getOutputPath(args) + var path ast.Array + return walk(filter, path, input, iter) +} + +func walk(filter, path ast.Array, input *ast.Term, iter func(*ast.Term) error) error { + + if len(filter) == 0 { + if err := iter(ast.ArrayTerm(ast.NewTerm(path), input)); err != nil { + return err + } + } + + if len(filter) > 0 { + key := filter[0] + filter = filter[1:] + if key.IsGround() { + if term := input.Get(key); term != nil { + return walk(filter, append(path, key), term, iter) + } + return nil + } + } + + switch v := input.Value.(type) { + case ast.Array: + for i := range v { + path = append(path, ast.IntNumberTerm(i)) + if err := walk(filter, path, v[i], iter); err != nil { + return err + } + path = path[:len(path)-1] + } + case ast.Object: + return v.Iter(func(k, v *ast.Term) error { + path = append(path, k) + if err := walk(filter, path, v, iter); err != nil { + return err + } + path = path[:len(path)-1] + return nil + }) + case ast.Set: + return v.Iter(func(elem *ast.Term) error { + path = append(path, elem) + if err := walk(filter, path, elem, iter); err != nil { + return err + } + path = path[:len(path)-1] + return nil + }) + } + + return nil +} + +func getOutputPath(args []*ast.Term) ast.Array { + if len(args) == 2 { + if arr, ok := args[1].Value.(ast.Array); ok { + if len(arr) == 2 { + if path, ok := arr[0].Value.(ast.Array); ok { + return path + } + } + } + } + return nil +} + +func init() { + RegisterBuiltinFunc(ast.WalkBuiltin.Name, evalWalk) +} diff --git a/vendor/github.com/open-policy-agent/opa/types/types.go b/vendor/github.com/open-policy-agent/opa/types/types.go new file mode 100644 index 000000000..f942c7b88 --- /dev/null +++ b/vendor/github.com/open-policy-agent/opa/types/types.go @@ -0,0 +1,869 @@ +// Copyright 2017 The OPA Authors. All rights reserved. +// Use of this source code is governed by an Apache2 +// license that can be found in the LICENSE file. + +// Package types declares data types for Rego values and helper functions to +// operate on these types. +package types + +import ( + "encoding/json" + "fmt" + "sort" + "strings" + + "github.com/open-policy-agent/opa/util" +) + +// Sprint returns the string representation of the type. +func Sprint(x Type) string { + if x == nil { + return "???" + } + return x.String() +} + +// Type represents a type of a term in the language. +type Type interface { + String() string + typeMarker() string + json.Marshaler +} + +func (Null) typeMarker() string { return "null" } +func (Boolean) typeMarker() string { return "boolean" } +func (Number) typeMarker() string { return "number" } +func (String) typeMarker() string { return "string" } +func (*Array) typeMarker() string { return "array" } +func (*Object) typeMarker() string { return "object" } +func (*Set) typeMarker() string { return "set" } +func (Any) typeMarker() string { return "any" } +func (Function) typeMarker() string { return "function" } + +// Null represents the null type. +type Null struct{} + +// NewNull returns a new Null type. +func NewNull() Null { + return Null{} +} + +// MarshalJSON returns the JSON encoding of t. +func (t Null) MarshalJSON() ([]byte, error) { + return json.Marshal(map[string]interface{}{ + "type": t.typeMarker(), + }) +} + +func (t Null) String() string { + return "null" +} + +// Boolean represents the boolean type. +type Boolean struct{} + +// B represents an instance of the boolean type. +var B = NewBoolean() + +// NewBoolean returns a new Boolean type. +func NewBoolean() Boolean { + return Boolean{} +} + +// MarshalJSON returns the JSON encoding of t. +func (t Boolean) MarshalJSON() ([]byte, error) { + repr := map[string]interface{}{ + "type": t.typeMarker(), + } + return json.Marshal(repr) +} + +func (t Boolean) String() string { + return t.typeMarker() +} + +// String represents the string type. +type String struct{} + +// S represents an instance of the string type. +var S = NewString() + +// NewString returns a new String type. +func NewString() String { + return String{} +} + +// MarshalJSON returns the JSON encoding of t. +func (t String) MarshalJSON() ([]byte, error) { + return json.Marshal(map[string]interface{}{ + "type": t.typeMarker(), + }) +} + +func (t String) String() string { + return "string" +} + +// Number represents the number type. +type Number struct{} + +// N represents an instance of the number type. +var N = NewNumber() + +// NewNumber returns a new Number type. +func NewNumber() Number { + return Number{} +} + +// MarshalJSON returns the JSON encoding of t. +func (t Number) MarshalJSON() ([]byte, error) { + return json.Marshal(map[string]interface{}{ + "type": t.typeMarker(), + }) +} + +func (Number) String() string { + return "number" +} + +// Array represents the array type. +type Array struct { + static []Type // static items + dynamic Type // dynamic items +} + +// NewArray returns a new Array type. +func NewArray(static []Type, dynamic Type) *Array { + return &Array{ + static: static, + dynamic: dynamic, + } +} + +// MarshalJSON returns the JSON encoding of t. +func (t *Array) MarshalJSON() ([]byte, error) { + repr := map[string]interface{}{ + "type": t.typeMarker(), + } + if len(t.static) != 0 { + repr["static"] = t.static + } + if t.dynamic != nil { + repr["dynamic"] = t.dynamic + } + return json.Marshal(repr) +} + +func (t *Array) String() string { + prefix := "array" + buf := []string{} + for _, tpe := range t.static { + buf = append(buf, Sprint(tpe)) + } + var repr = prefix + if len(buf) > 0 { + repr += "<" + strings.Join(buf, ", ") + ">" + } + if t.dynamic != nil { + repr += "[" + t.dynamic.String() + "]" + } + return repr +} + +// Dynamic returns the type of the array's dynamic elements. +func (t *Array) Dynamic() Type { + return t.dynamic +} + +// Len returns the number of static array elements. +func (t *Array) Len() int { + return len(t.static) +} + +// Select returns the type of element at the zero-based pos. +func (t *Array) Select(pos int) Type { + if pos >= 0 { + if len(t.static) > pos { + return t.static[pos] + } + if t.dynamic != nil { + return t.dynamic + } + } + return nil +} + +// Set represents the set type. +type Set struct { + of Type +} + +// NewSet returns a new Set type. +func NewSet(of Type) *Set { + return &Set{ + of: of, + } +} + +// MarshalJSON returns the JSON encoding of t. +func (t *Set) MarshalJSON() ([]byte, error) { + repr := map[string]interface{}{ + "type": t.typeMarker(), + } + if t.of != nil { + repr["of"] = t.of + } + return json.Marshal(repr) +} + +func (t *Set) String() string { + prefix := "set" + return prefix + "[" + Sprint(t.of) + "]" +} + +// StaticProperty represents a static object property. +type StaticProperty struct { + Key interface{} + Value Type +} + +// NewStaticProperty returns a new StaticProperty object. +func NewStaticProperty(key interface{}, value Type) *StaticProperty { + return &StaticProperty{ + Key: key, + Value: value, + } +} + +// MarshalJSON returns the JSON encoding of p. +func (p *StaticProperty) MarshalJSON() ([]byte, error) { + return json.Marshal(map[string]interface{}{ + "key": p.Key, + "value": p.Value, + }) +} + +// DynamicProperty represents a dynamic object property. +type DynamicProperty struct { + Key Type + Value Type +} + +// NewDynamicProperty returns a new DynamicProperty object. +func NewDynamicProperty(key, value Type) *DynamicProperty { + return &DynamicProperty{ + Key: key, + Value: value, + } +} + +// MarshalJSON returns the JSON encoding of p. +func (p *DynamicProperty) MarshalJSON() ([]byte, error) { + return json.Marshal(map[string]interface{}{ + "key": p.Key, + "value": p.Value, + }) +} + +func (p *DynamicProperty) String() string { + return fmt.Sprintf("%s: %s", Sprint(p.Key), Sprint(p.Value)) +} + +// Object represents the object type. +type Object struct { + static []*StaticProperty // constant properties + dynamic *DynamicProperty // dynamic properties +} + +// NewObject returns a new Object type. +func NewObject(static []*StaticProperty, dynamic *DynamicProperty) *Object { + sort.Slice(static, func(i, j int) bool { + cmp := util.Compare(static[i].Key, static[j].Key) + return cmp == -1 + }) + return &Object{ + static: static, + dynamic: dynamic, + } +} + +func (t *Object) String() string { + prefix := "object" + buf := make([]string, 0, len(t.static)) + for _, p := range t.static { + buf = append(buf, fmt.Sprintf("%v: %v", p.Key, Sprint(p.Value))) + } + var repr = prefix + if len(buf) > 0 { + repr += "<" + strings.Join(buf, ", ") + ">" + } + if t.dynamic != nil { + repr += "[" + t.dynamic.String() + "]" + } + return repr +} + +// DynamicValue returns the type of the object's dynamic elements. +func (t *Object) DynamicValue() Type { + if t.dynamic == nil { + return nil + } + return t.dynamic.Value +} + +// Keys returns the keys of the object's static elements. +func (t *Object) Keys() []interface{} { + sl := make([]interface{}, 0, len(t.static)) + for _, p := range t.static { + sl = append(sl, p.Key) + } + return sl +} + +// MarshalJSON returns the JSON encoding of t. +func (t *Object) MarshalJSON() ([]byte, error) { + repr := map[string]interface{}{ + "type": t.typeMarker(), + } + if len(t.static) != 0 { + repr["static"] = t.static + } + if t.dynamic != nil { + repr["dynamic"] = t.dynamic + } + return json.Marshal(repr) +} + +// Select returns the type of the named property. +func (t *Object) Select(name interface{}) Type { + for _, p := range t.static { + if util.Compare(p.Key, name) == 0 { + return p.Value + } + } + if t.dynamic != nil { + if Contains(t.dynamic.Key, TypeOf(name)) { + return t.dynamic.Value + } + } + return nil +} + +// Any represents a dynamic type. +type Any []Type + +// A represents the superset of all types. +var A = NewAny() + +// NewAny returns a new Any type. +func NewAny(of ...Type) Any { + sl := make(Any, len(of)) + for i := range sl { + sl[i] = of[i] + } + return sl +} + +// Contains returns true if t is a superset of other. +func (t Any) Contains(other Type) bool { + if _, ok := other.(*Function); ok { + return false + } + for i := range t { + if Compare(t[i], other) == 0 { + return true + } + } + return len(t) == 0 +} + +// MarshalJSON returns the JSON encoding of t. +func (t Any) MarshalJSON() ([]byte, error) { + return json.Marshal(map[string]interface{}{ + "type": t.typeMarker(), + "of": []Type(t), + }) +} + +// Merge return a new Any type that is the superset of t and other. +func (t Any) Merge(other Type) Any { + if otherAny, ok := other.(Any); ok { + return t.Union(otherAny) + } + if t.Contains(other) { + return t + } + return append(t, other) +} + +// Union returns a new Any type that is the union of the two Any types. +func (t Any) Union(other Any) Any { + if len(t) == 0 { + return t + } + if len(other) == 0 { + return other + } + cpy := make(Any, len(t)) + for i := range cpy { + cpy[i] = t[i] + } + for i := range other { + if !cpy.Contains(other[i]) { + cpy = append(cpy, other[i]) + } + } + return cpy +} + +func (t Any) String() string { + prefix := "any" + if len(t) == 0 { + return prefix + } + buf := make([]string, len(t)) + for i := range t { + buf[i] = Sprint(t[i]) + } + return prefix + "<" + strings.Join(buf, ", ") + ">" +} + +// Function represents a function type. +type Function struct { + args []Type + result Type +} + +// Args returns an argument list. +func Args(x ...Type) []Type { + return x +} + +// NewFunction returns a new Function object where xs[:len(xs)-1] are arguments +// and xs[len(xs)-1] is the result type. +func NewFunction(args []Type, result Type) *Function { + return &Function{ + args: args, + result: result, + } +} + +// Args returns the function's argument types. +func (t *Function) Args() []Type { + return t.args +} + +// Result returns the function's result type. +func (t *Function) Result() Type { + return t.result +} + +func (t *Function) String() string { + var args string + if len(t.args) != 1 { + args = "(" + } + buf := []string{} + for _, a := range t.Args() { + buf = append(buf, Sprint(a)) + } + args += strings.Join(buf, ", ") + if len(t.args) != 1 { + args += ")" + } + return fmt.Sprintf("%v => %v", args, Sprint(t.Result())) +} + +// MarshalJSON returns the JSON encoding of t. +func (t *Function) MarshalJSON() ([]byte, error) { + repr := map[string]interface{}{ + "type": t.typeMarker(), + } + if len(t.args) > 0 { + repr["args"] = t.args + } + if t.result != nil { + repr["result"] = t.result + } + return json.Marshal(repr) +} + +// Union returns a new function represnting the union of t and other. Functions +// must have the same arity to be unioned. +func (t *Function) Union(other *Function) *Function { + if other == nil { + return t + } else if t == nil { + return other + } + a := t.Args() + b := other.Args() + if len(a) != len(b) { + return nil + } + args := make([]Type, len(a)) + for i := range a { + args[i] = Or(a[i], b[i]) + } + + return NewFunction(args, Or(t.Result(), other.Result())) +} + +// Compare returns -1, 0, 1 based on comparison between a and b. +func Compare(a, b Type) int { + x := typeOrder(a) + y := typeOrder(b) + if x > y { + return 1 + } else if x < y { + return -1 + } + switch a.(type) { + case nil, Null, Boolean, Number, String: + return 0 + case *Array: + arrA := a.(*Array) + arrB := b.(*Array) + if arrA.dynamic != nil && arrB.dynamic == nil { + return 1 + } else if arrB.dynamic != nil && arrA.dynamic == nil { + return -1 + } + if arrB.dynamic != nil && arrA.dynamic != nil { + if cmp := Compare(arrA.dynamic, arrB.dynamic); cmp != 0 { + return cmp + } + } + return typeSliceCompare(arrA.static, arrB.static) + case *Object: + objA := a.(*Object) + objB := b.(*Object) + if objA.dynamic != nil && objB.dynamic == nil { + return 1 + } else if objB.dynamic != nil && objA.dynamic == nil { + return -1 + } + if objA.dynamic != nil && objB.dynamic != nil { + if cmp := Compare(objA.dynamic.Key, objB.dynamic.Key); cmp != 0 { + return cmp + } + if cmp := Compare(objA.dynamic.Value, objB.dynamic.Value); cmp != 0 { + return cmp + } + } + + lenStaticA := len(objA.static) + lenStaticB := len(objB.static) + + minLen := lenStaticA + if lenStaticB < minLen { + minLen = lenStaticB + } + + for i := 0; i < minLen; i++ { + if cmp := util.Compare(objA.static[i].Key, objB.static[i].Key); cmp != 0 { + return cmp + } + if cmp := Compare(objA.static[i].Value, objB.static[i].Value); cmp != 0 { + return cmp + } + } + + if lenStaticA < lenStaticB { + return -1 + } else if lenStaticB < lenStaticA { + return 1 + } + + return 0 + case *Set: + setA := a.(*Set) + setB := b.(*Set) + if setA.of == nil && setB.of == nil { + return 0 + } else if setA.of == nil { + return -1 + } else if setB.of == nil { + return 1 + } + return Compare(setA.of, setB.of) + case Any: + sl1 := typeSlice(a.(Any)) + sl2 := typeSlice(b.(Any)) + sort.Sort(sl1) + sort.Sort(sl2) + return typeSliceCompare(sl1, sl2) + case *Function: + fA := a.(*Function) + fB := b.(*Function) + if len(fA.args) < len(fB.args) { + return -1 + } else if len(fA.args) > len(fB.args) { + return 1 + } + for i := 0; i < len(fA.args); i++ { + if cmp := Compare(fA.args[i], fB.args[i]); cmp != 0 { + return cmp + } + } + return Compare(fA.result, fB.result) + default: + panic("unreachable") + } +} + +// Contains returns true if a is a superset or equal to b. +func Contains(a, b Type) bool { + if any, ok := a.(Any); ok { + return any.Contains(b) + } + return Compare(a, b) == 0 +} + +// Or returns a type that represents the union of a and b. If one type is a +// superset of the other, the superset is returned unchanged. +func Or(a, b Type) Type { + if a == nil { + return b + } else if b == nil { + return a + } + fA, ok1 := a.(*Function) + fB, ok2 := b.(*Function) + if ok1 && ok2 { + return fA.Union(fB) + } else if ok1 || ok2 { + return nil + } + anyA, ok1 := a.(Any) + anyB, ok2 := b.(Any) + if ok1 { + return anyA.Merge(b) + } + if ok2 { + return anyB.Merge(a) + } + if Compare(a, b) == 0 { + return a + } + return NewAny(a, b) +} + +// Select returns a property or item of a. +func Select(a Type, x interface{}) Type { + switch a := a.(type) { + case *Array: + n, ok := x.(json.Number) + if !ok { + return nil + } + pos, err := n.Int64() + if err != nil { + return nil + } + return a.Select(int(pos)) + case *Object: + return a.Select(x) + case *Set: + tpe := TypeOf(x) + if Compare(a.of, tpe) == 0 { + return a.of + } + if any, ok := a.of.(Any); ok { + if any.Contains(tpe) { + return tpe + } + } + return nil + case Any: + if Compare(a, A) == 0 { + return A + } + var tpe Type + for i := range a { + // TODO(tsandall): test nil/nil + tpe = Or(Select(a[i], x), tpe) + } + return tpe + default: + return nil + } +} + +// Keys returns the type of keys that can be enumerated for a. For arrays, the +// keys are always number types, for objects the keys are always string types, +// and for sets the keys are always the type of the set element. +func Keys(a Type) Type { + switch a := a.(type) { + case *Array: + return N + case *Object: + var tpe Type + for _, k := range a.Keys() { + tpe = Or(tpe, TypeOf(k)) + } + if a.dynamic != nil { + tpe = Or(tpe, a.dynamic.Key) + } + return tpe + case *Set: + return a.of + case Any: + // TODO(tsandall): ditto test + if Compare(a, A) == 0 { + return A + } + var tpe Type + for i := range a { + tpe = Or(Keys(a[i]), tpe) + } + return tpe + } + return nil +} + +// Values returns the type of values that can be enumerated for a. +func Values(a Type) Type { + switch a := a.(type) { + case *Array: + var tpe Type + for i := range a.static { + tpe = Or(tpe, a.static[i]) + } + return Or(tpe, a.dynamic) + case *Object: + var tpe Type + for _, v := range a.static { + tpe = Or(tpe, v.Value) + } + if a.dynamic != nil { + tpe = Or(tpe, a.dynamic.Value) + } + return tpe + case *Set: + return a.of + case Any: + if Compare(a, A) == 0 { + return A + } + var tpe Type + for i := range a { + tpe = Or(Values(a[i]), tpe) + } + return tpe + } + return nil +} + +// Nil returns true if a's type is unknown. +func Nil(a Type) bool { + switch a := a.(type) { + case nil: + return true + case *Function: + for i := range a.args { + if Nil(a.args[i]) { + return true + } + } + return Nil(a.result) + case *Array: + for i := range a.static { + if Nil(a.static[i]) { + return true + } + } + if a.dynamic != nil { + return Nil(a.dynamic) + } + case *Object: + for i := range a.static { + if Nil(a.static[i].Value) { + return true + } + } + if a.dynamic != nil { + return Nil(a.dynamic.Key) || Nil(a.dynamic.Value) + } + case *Set: + return Nil(a.of) + } + return false +} + +// TypeOf returns the type of the Golang native value. +func TypeOf(x interface{}) Type { + switch x := x.(type) { + case nil: + return NewNull() + case bool: + return B + case string: + return S + case json.Number: + return N + case map[interface{}]interface{}: + static := make([]*StaticProperty, 0, len(x)) + for k, v := range x { + static = append(static, NewStaticProperty(k, TypeOf(v))) + } + return NewObject(static, nil) + case []interface{}: + static := make([]Type, len(x)) + for i := range x { + static[i] = TypeOf(x[i]) + } + return NewArray(static, nil) + } + panic("unreachable") +} + +type typeSlice []Type + +func (s typeSlice) Less(i, j int) bool { return Compare(s[i], s[j]) < 0 } +func (s typeSlice) Swap(i, j int) { x := s[i]; s[i] = s[j]; s[j] = x } +func (s typeSlice) Len() int { return len(s) } + +func typeSliceCompare(a, b []Type) int { + minLen := len(a) + if len(b) < minLen { + minLen = len(b) + } + for i := 0; i < minLen; i++ { + if cmp := Compare(a[i], b[i]); cmp != 0 { + return cmp + } + } + if len(a) < len(b) { + return -1 + } else if len(b) < len(a) { + return 1 + } + return 0 +} + +func typeOrder(x Type) int { + switch x.(type) { + case Null: + return 0 + case Boolean: + return 1 + case Number: + return 2 + case String: + return 3 + case *Array: + return 4 + case *Object: + return 5 + case *Set: + return 6 + case Any: + return 7 + case *Function: + return 8 + case nil: + return -1 + } + panic("unreachable") +} diff --git a/vendor/github.com/open-policy-agent/opa/util/backoff.go b/vendor/github.com/open-policy-agent/opa/util/backoff.go new file mode 100644 index 000000000..9e3a37257 --- /dev/null +++ b/vendor/github.com/open-policy-agent/opa/util/backoff.go @@ -0,0 +1,42 @@ +// Copyright 2018 The OPA Authors. All rights reserved. +// Use of this source code is governed by an Apache2 +// license that can be found in the LICENSE file. + +package util + +import ( + "math/rand" + "time" +) + +// DefaultBackoff returns a delay with an exponential backoff based on the +// number of retries. +func DefaultBackoff(base, max float64, retries int) time.Duration { + return Backoff(base, max, .2, 1.6, retries) +} + +// Backoff returns a delay with an exponential backoff based on the number of +// retries. Same algorithm used in gRPC. +func Backoff(base, max, jitter, factor float64, retries int) time.Duration { + if retries == 0 { + return 0 + } + + backoff, max := float64(base), float64(max) + for backoff < max && retries > 0 { + backoff *= factor + retries-- + } + if backoff > max { + backoff = max + } + + // Randomize backoff delays so that if a cluster of requests start at + // the same time, they won't operate in lockstep. + backoff *= 1 + jitter*(rand.Float64()*2-1) + if backoff < 0 { + return 0 + } + + return time.Duration(backoff) +} diff --git a/vendor/github.com/open-policy-agent/opa/util/close.go b/vendor/github.com/open-policy-agent/opa/util/close.go new file mode 100644 index 000000000..97c917731 --- /dev/null +++ b/vendor/github.com/open-policy-agent/opa/util/close.go @@ -0,0 +1,23 @@ +// Copyright 2018 The OPA Authors. All rights reserved. +// Use of this source code is governed by an Apache2 +// license that can be found in the LICENSE file. + +package util + +import ( + "io" + "io/ioutil" + "net/http" +) + +// Close reads the remaining bytes from the response and then closes it to +// ensure that the connection is freed. If the body is not read and closed, a +// leak can occur. +func Close(resp *http.Response) { + if resp != nil && resp.Body != nil { + if _, err := io.Copy(ioutil.Discard, resp.Body); err != nil { + return + } + resp.Body.Close() + } +} diff --git a/vendor/github.com/open-policy-agent/opa/util/compare.go b/vendor/github.com/open-policy-agent/opa/util/compare.go new file mode 100644 index 000000000..dd187590b --- /dev/null +++ b/vendor/github.com/open-policy-agent/opa/util/compare.go @@ -0,0 +1,161 @@ +// Copyright 2016 The OPA Authors. All rights reserved. +// Use of this source code is governed by an Apache2 +// license that can be found in the LICENSE file. + +package util + +import ( + "encoding/json" + "fmt" + "math/big" + "sort" +) + +// Compare returns 0 if a equals b, -1 if a is less than b, and 1 if b is than a. +// +// For comparison between values of different types, the following ordering is used: +// nil < bool < float64 < string < []interface{} < map[string]interface{}. Slices and maps +// are compared recursively. If one slice or map is a subset of the other slice or map +// it is considered "less than". Nil is always equal to nil. +// +func Compare(a, b interface{}) int { + aSortOrder := sortOrder(a) + bSortOrder := sortOrder(b) + if aSortOrder < bSortOrder { + return -1 + } else if bSortOrder < aSortOrder { + return 1 + } + switch a := a.(type) { + case nil: + return 0 + case bool: + switch b := b.(type) { + case bool: + if a == b { + return 0 + } + if !a { + return -1 + } + return 1 + } + case json.Number: + switch b := b.(type) { + case json.Number: + return compareJSONNumber(a, b) + } + case string: + switch b := b.(type) { + case string: + if a == b { + return 0 + } else if a < b { + return -1 + } + return 1 + } + case []interface{}: + switch b := b.(type) { + case []interface{}: + bLen := len(b) + aLen := len(a) + minLen := aLen + if bLen < minLen { + minLen = bLen + } + for i := 0; i < minLen; i++ { + cmp := Compare(a[i], b[i]) + if cmp != 0 { + return cmp + } + } + if aLen == bLen { + return 0 + } else if aLen < bLen { + return -1 + } + return 1 + } + case map[string]interface{}: + switch b := b.(type) { + case map[string]interface{}: + var aKeys []string + for k := range a { + aKeys = append(aKeys, k) + } + var bKeys []string + for k := range b { + bKeys = append(bKeys, k) + } + sort.Strings(aKeys) + sort.Strings(bKeys) + aLen := len(aKeys) + bLen := len(bKeys) + minLen := aLen + if bLen < minLen { + minLen = bLen + } + for i := 0; i < minLen; i++ { + if aKeys[i] < bKeys[i] { + return -1 + } else if bKeys[i] < aKeys[i] { + return 1 + } + aVal := a[aKeys[i]] + bVal := b[bKeys[i]] + cmp := Compare(aVal, bVal) + if cmp != 0 { + return cmp + } + } + if aLen == bLen { + return 0 + } else if aLen < bLen { + return -1 + } + return 1 + } + } + + panic(fmt.Sprintf("illegal arguments of type %T and type %T", a, b)) +} + +const ( + nilSort = iota + boolSort = iota + numberSort = iota + stringSort = iota + arraySort = iota + objectSort = iota +) + +func compareJSONNumber(a, b json.Number) int { + bigA, ok := new(big.Float).SetString(string(a)) + if !ok { + panic("illegal value") + } + bigB, ok := new(big.Float).SetString(string(b)) + if !ok { + panic("illegal value") + } + return bigA.Cmp(bigB) +} + +func sortOrder(v interface{}) int { + switch v.(type) { + case nil: + return nilSort + case bool: + return boolSort + case json.Number: + return numberSort + case string: + return stringSort + case []interface{}: + return arraySort + case map[string]interface{}: + return objectSort + } + panic(fmt.Sprintf("illegal argument of type %T", v)) +} diff --git a/vendor/github.com/open-policy-agent/opa/util/doc.go b/vendor/github.com/open-policy-agent/opa/util/doc.go new file mode 100644 index 000000000..900dff8c1 --- /dev/null +++ b/vendor/github.com/open-policy-agent/opa/util/doc.go @@ -0,0 +1,6 @@ +// Copyright 2016 The OPA Authors. All rights reserved. +// Use of this source code is governed by an Apache2 +// license that can be found in the LICENSE file. + +// Package util provides generic utilities used throughout the policy engine. +package util diff --git a/vendor/github.com/open-policy-agent/opa/util/enumflag.go b/vendor/github.com/open-policy-agent/opa/util/enumflag.go new file mode 100644 index 000000000..02a06a53f --- /dev/null +++ b/vendor/github.com/open-policy-agent/opa/util/enumflag.go @@ -0,0 +1,54 @@ +// Copyright 2017 The OPA Authors. All rights reserved. +// Use of this source code is governed by an Apache2 +// license that can be found in the LICENSE file. + +package util + +import ( + "fmt" + "strings" +) + +// EnumFlag implements the pflag.Value interface to provide enumerated command +// line parameter values. +type EnumFlag struct { + defaultValue string + vs []string + i int +} + +// NewEnumFlag returns a new EnumFlag that has a defaultValue and vs enumerated +// values. +func NewEnumFlag(defaultValue string, vs []string) *EnumFlag { + f := &EnumFlag{ + i: -1, + vs: vs, + defaultValue: defaultValue, + } + return f +} + +// Type returns the valid enumeration values. +func (f *EnumFlag) Type() string { + return "{" + strings.Join(f.vs, ",") + "}" +} + +// String returns the EnumValue's value as string. +func (f *EnumFlag) String() string { + if f.i == -1 { + return f.defaultValue + } + return f.vs[f.i] +} + +// Set sets the enum value. If s is not a valid enum value, an error is +// returned. +func (f *EnumFlag) Set(s string) error { + for i := range f.vs { + if f.vs[i] == s { + f.i = i + return nil + } + } + return fmt.Errorf("must be one of %v", f.Type()) +} diff --git a/vendor/github.com/open-policy-agent/opa/util/graph.go b/vendor/github.com/open-policy-agent/opa/util/graph.go new file mode 100644 index 000000000..f0e824245 --- /dev/null +++ b/vendor/github.com/open-policy-agent/opa/util/graph.go @@ -0,0 +1,90 @@ +// Copyright 2016 The OPA Authors. All rights reserved. +// Use of this source code is governed by an Apache2 +// license that can be found in the LICENSE file. + +package util + +// Traversal defines a basic interface to perform traversals. +type Traversal interface { + + // Edges should return the neighbours of node "u". + Edges(u T) []T + + // Visited should return true if node "u" has already been visited in this + // traversal. If the same traversal is used multiple times, the state that + // tracks visited nodes should be reset. + Visited(u T) bool +} + +// Equals should return true if node "u" equals node "v". +type Equals func(u T, v T) bool + +// Iter should return true to indicate stop. +type Iter func(u T) bool + +// DFS performs a depth first traversal calling f for each node starting from u. +// If f returns true, traversal stops and DFS returns true. +func DFS(t Traversal, f Iter, u T) bool { + lifo := NewLIFO(u) + for lifo.Size() > 0 { + next, _ := lifo.Pop() + if t.Visited(next) { + continue + } + if f(next) { + return true + } + for _, v := range t.Edges(next) { + lifo.Push(v) + } + } + return false +} + +// BFS performs a breadth first traversal calling f for each node starting from +// u. If f returns true, traversal stops and BFS returns true. +func BFS(t Traversal, f Iter, u T) bool { + fifo := NewFIFO(u) + for fifo.Size() > 0 { + next, _ := fifo.Pop() + if t.Visited(next) { + continue + } + if f(next) { + return true + } + for _, v := range t.Edges(next) { + fifo.Push(v) + } + } + return false +} + +// DFSPath returns a path from node a to node z found by performing +// a depth first traversal. If no path is found, an empty slice is returned. +func DFSPath(t Traversal, eq Equals, a, z T) []T { + p := dfsRecursive(t, eq, a, z, []T{}) + for i := len(p)/2 - 1; i >= 0; i-- { + o := len(p) - i - 1 + p[i], p[o] = p[o], p[i] + } + return p +} + +func dfsRecursive(t Traversal, eq Equals, u, z T, path []T) []T { + if t.Visited(u) { + return path + } + for _, v := range t.Edges(u) { + if eq(v, z) { + path = append(path, z) + path = append(path, u) + return path + } + if p := dfsRecursive(t, eq, v, z, path); len(p) > 0 { + path = append(p, u) + return path + } + } + return path +} diff --git a/vendor/github.com/open-policy-agent/opa/util/hashmap.go b/vendor/github.com/open-policy-agent/opa/util/hashmap.go new file mode 100644 index 000000000..11e7dca40 --- /dev/null +++ b/vendor/github.com/open-policy-agent/opa/util/hashmap.go @@ -0,0 +1,157 @@ +// Copyright 2016 The OPA Authors. All rights reserved. +// Use of this source code is governed by an Apache2 +// license that can be found in the LICENSE file. + +package util + +import ( + "fmt" + "strings" +) + +// T is a concise way to refer to T. +type T interface{} + +type hashEntry struct { + k T + v T + next *hashEntry +} + +// HashMap represents a key/value map. +type HashMap struct { + eq func(T, T) bool + hash func(T) int + table map[int]*hashEntry + size int +} + +// NewHashMap returns a new empty HashMap. +func NewHashMap(eq func(T, T) bool, hash func(T) int) *HashMap { + return &HashMap{ + eq: eq, + hash: hash, + table: make(map[int]*hashEntry), + size: 0, + } +} + +// Copy returns a shallow copy of this HashMap. +func (h *HashMap) Copy() *HashMap { + cpy := NewHashMap(h.eq, h.hash) + h.Iter(func(k, v T) bool { + cpy.Put(k, v) + return false + }) + return cpy +} + +// Equal returns true if this HashMap equals the other HashMap. +// Two hash maps are equal if they contain the same key/value pairs. +func (h *HashMap) Equal(other *HashMap) bool { + if h.Len() != other.Len() { + return false + } + return !h.Iter(func(k, v T) bool { + ov, ok := other.Get(k) + if !ok { + return true + } + return !h.eq(v, ov) + }) +} + +// Get returns the value for k. +func (h *HashMap) Get(k T) (T, bool) { + hash := h.hash(k) + for entry := h.table[hash]; entry != nil; entry = entry.next { + if h.eq(entry.k, k) { + return entry.v, true + } + } + return nil, false +} + +// Delete removes the the key k. +func (h *HashMap) Delete(k T) { + hash := h.hash(k) + var prev *hashEntry + for entry := h.table[hash]; entry != nil; entry = entry.next { + if h.eq(entry.k, k) { + if prev != nil { + prev.next = entry.next + } else { + h.table[hash] = entry.next + } + h.size-- + return + } + prev = entry + } +} + +// Hash returns the hash code for this hash map. +func (h *HashMap) Hash() int { + var hash int + h.Iter(func(k, v T) bool { + hash += h.hash(k) + h.hash(v) + return false + }) + return hash +} + +// Iter invokes the iter function for each element in the HashMap. +// If the iter function returns true, iteration stops and the return value is true. +// If the iter function never returns true, iteration proceeds through all elements +// and the return value is false. +func (h *HashMap) Iter(iter func(T, T) bool) bool { + for _, entry := range h.table { + for ; entry != nil; entry = entry.next { + if iter(entry.k, entry.v) { + return true + } + } + } + return false +} + +// Len returns the current size of this HashMap. +func (h *HashMap) Len() int { + return h.size +} + +// Put inserts a key/value pair into this HashMap. If the key is already present, the existing +// value is overwritten. +func (h *HashMap) Put(k T, v T) { + hash := h.hash(k) + head := h.table[hash] + for entry := head; entry != nil; entry = entry.next { + if h.eq(entry.k, k) { + entry.v = v + return + } + } + h.table[hash] = &hashEntry{k: k, v: v, next: head} + h.size++ +} + +func (h *HashMap) String() string { + var buf []string + h.Iter(func(k T, v T) bool { + buf = append(buf, fmt.Sprintf("%v: %v", k, v)) + return false + }) + return "{" + strings.Join(buf, ", ") + "}" +} + +// Update returns a new HashMap with elements from the other HashMap put into this HashMap. +// If the other HashMap contains elements with the same key as this HashMap, the value +// from the other HashMap overwrites the value from this HashMap. +func (h *HashMap) Update(other *HashMap) *HashMap { + updated := h.Copy() + other.Iter(func(k, v T) bool { + updated.Put(k, v) + return false + }) + return updated +} diff --git a/vendor/github.com/open-policy-agent/opa/util/json.go b/vendor/github.com/open-policy-agent/opa/util/json.go new file mode 100644 index 000000000..c65dac9a4 --- /dev/null +++ b/vendor/github.com/open-policy-agent/opa/util/json.go @@ -0,0 +1,99 @@ +// Copyright 2016 The OPA Authors. All rights reserved. +// Use of this source code is governed by an Apache2 +// license that can be found in the LICENSE file. + +package util + +import ( + "bytes" + "encoding/json" + "io" + "reflect" + + "github.com/ghodss/yaml" +) + +// UnmarshalJSON parses the JSON encoded data and stores the result in the value +// pointed to by x. +// +// This function is intended to be used in place of the standard json.Marshal +// function when json.Number is required. +func UnmarshalJSON(bs []byte, x interface{}) (err error) { + buf := bytes.NewBuffer(bs) + decoder := NewJSONDecoder(buf) + return decoder.Decode(x) +} + +// NewJSONDecoder returns a new decoder that reads from r. +// +// This function is intended to be used in place of the standard json.NewDecoder +// when json.Number is required. +func NewJSONDecoder(r io.Reader) *json.Decoder { + decoder := json.NewDecoder(r) + decoder.UseNumber() + return decoder +} + +// MustUnmarshalJSON parse the JSON encoded data and returns the result. +// +// If the data cannot be decoded, this function will panic. This function is for +// test purposes. +func MustUnmarshalJSON(bs []byte) interface{} { + var x interface{} + if err := UnmarshalJSON(bs, &x); err != nil { + panic(err) + } + return x +} + +// MustMarshalJSON returns the JSON encoding of x +// +// If the data cannot be encoded, this function will panic. This function is for +// test purposes. +func MustMarshalJSON(x interface{}) []byte { + bs, err := json.Marshal(x) + if err != nil { + panic(err) + } + return bs +} + +// RoundTrip encodes to JSON, and decodes the result again. +// +// Thereby, it is converting its argument to the representation expected by +// rego.Input and inmem's Write operations. Works with both references and +// values. +func RoundTrip(x *interface{}) error { + bs, err := json.Marshal(x) + if err != nil { + return err + } + return UnmarshalJSON(bs, x) +} + +// Reference returns a pointer to its argument unless the argument already is +// a pointer. If the argument is **t, or ***t, etc, it will return *t. +// +// Used for preparing Go types (including pointers to structs) into values to be +// put through util.RoundTrip(). +func Reference(x interface{}) *interface{} { + var y interface{} + rv := reflect.ValueOf(x) + if rv.Kind() == reflect.Ptr { + return Reference(rv.Elem().Interface()) + } + if rv.Kind() != reflect.Invalid { + y = rv.Interface() + return &y + } + return &x +} + +// Unmarshal decodes a YAML or JSON value into the specified type. +func Unmarshal(bs []byte, v interface{}) error { + bs, err := yaml.YAMLToJSON(bs) + if err != nil { + return err + } + return UnmarshalJSON(bs, v) +} diff --git a/vendor/github.com/open-policy-agent/opa/util/queue.go b/vendor/github.com/open-policy-agent/opa/util/queue.go new file mode 100644 index 000000000..63a2ffc16 --- /dev/null +++ b/vendor/github.com/open-policy-agent/opa/util/queue.go @@ -0,0 +1,113 @@ +// Copyright 2017 The OPA Authors. All rights reserved. +// Use of this source code is governed by an Apache2 +// license that can be found in the LICENSE file. + +package util + +// LIFO represents a simple LIFO queue. +type LIFO struct { + top *queueNode + size int +} + +type queueNode struct { + v T + next *queueNode +} + +// NewLIFO returns a new LIFO queue containing elements ts starting with the +// left-most argument at the bottom. +func NewLIFO(ts ...T) *LIFO { + s := &LIFO{} + for i := range ts { + s.Push(ts[i]) + } + return s +} + +// Push adds a new element onto the LIFO. +func (s *LIFO) Push(t T) { + node := &queueNode{v: t, next: s.top} + s.top = node + s.size++ +} + +// Peek returns the top of the LIFO. If LIFO is empty, returns nil, false. +func (s *LIFO) Peek() (T, bool) { + if s.top == nil { + return nil, false + } + return s.top.v, true +} + +// Pop returns the top of the LIFO and removes it. If LIFO is empty returns +// nil, false. +func (s *LIFO) Pop() (T, bool) { + if s.top == nil { + return nil, false + } + node := s.top + s.top = node.next + s.size-- + return node.v, true +} + +// Size returns the size of the LIFO. +func (s *LIFO) Size() int { + return s.size +} + +// FIFO represents a simple FIFO queue. +type FIFO struct { + front *queueNode + back *queueNode + size int +} + +// NewFIFO returns a new FIFO queue containing elements ts starting with the +// left-most argument at the front. +func NewFIFO(ts ...T) *FIFO { + s := &FIFO{} + for i := range ts { + s.Push(ts[i]) + } + return s +} + +// Push adds a new element onto the LIFO. +func (s *FIFO) Push(t T) { + node := &queueNode{v: t, next: nil} + if s.front == nil { + s.front = node + s.back = node + } else { + s.back.next = node + s.back = node + } + s.size++ +} + +// Peek returns the top of the LIFO. If LIFO is empty, returns nil, false. +func (s *FIFO) Peek() (T, bool) { + if s.front == nil { + return nil, false + } + return s.front.v, true +} + +// Pop returns the top of the LIFO and removes it. If LIFO is empty returns +// nil, false. +func (s *FIFO) Pop() (T, bool) { + if s.front == nil { + return nil, false + } + node := s.front + s.front = node.next + s.size-- + return node.v, true +} + +// Size returns the size of the LIFO. +func (s *FIFO) Size() int { + return s.size +} diff --git a/vendor/github.com/open-policy-agent/opa/version/version.go b/vendor/github.com/open-policy-agent/opa/version/version.go new file mode 100644 index 000000000..e1cdb4b15 --- /dev/null +++ b/vendor/github.com/open-policy-agent/opa/version/version.go @@ -0,0 +1,15 @@ +// Copyright 2016 The OPA Authors. All rights reserved. +// Use of this source code is governed by an Apache2 +// license that can be found in the LICENSE file. + +// Package version contains version information that is set at build time. +package version + +// Version information that is displayed by the "version" command and used to +// identify the version of running instances of OPA. +var ( + Version = "" + Vcs = "" + Timestamp = "" + Hostname = "" +) diff --git a/vendor/github.com/rcrowley/go-metrics/.gitignore b/vendor/github.com/rcrowley/go-metrics/.gitignore new file mode 100644 index 000000000..83c8f8237 --- /dev/null +++ b/vendor/github.com/rcrowley/go-metrics/.gitignore @@ -0,0 +1,9 @@ +*.[68] +*.a +*.out +*.swp +_obj +_testmain.go +cmd/metrics-bench/metrics-bench +cmd/metrics-example/metrics-example +cmd/never-read/never-read diff --git a/vendor/github.com/rcrowley/go-metrics/.travis.yml b/vendor/github.com/rcrowley/go-metrics/.travis.yml new file mode 100644 index 000000000..117763e65 --- /dev/null +++ b/vendor/github.com/rcrowley/go-metrics/.travis.yml @@ -0,0 +1,17 @@ +language: go + +go: + - 1.3 + - 1.4 + - 1.5 + - 1.6 + - 1.7 + - 1.8 + - 1.9 + +script: + - ./validate.sh + +# this should give us faster builds according to +# http://docs.travis-ci.com/user/migrating-from-legacy/ +sudo: false diff --git a/vendor/github.com/rcrowley/go-metrics/LICENSE b/vendor/github.com/rcrowley/go-metrics/LICENSE new file mode 100644 index 000000000..363fa9ee7 --- /dev/null +++ b/vendor/github.com/rcrowley/go-metrics/LICENSE @@ -0,0 +1,29 @@ +Copyright 2012 Richard Crowley. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + 1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + +THIS SOFTWARE IS PROVIDED BY RICHARD CROWLEY ``AS IS'' AND ANY EXPRESS +OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL RICHARD CROWLEY OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF +THE POSSIBILITY OF SUCH DAMAGE. + +The views and conclusions contained in the software and documentation +are those of the authors and should not be interpreted as representing +official policies, either expressed or implied, of Richard Crowley. diff --git a/vendor/github.com/rcrowley/go-metrics/README.md b/vendor/github.com/rcrowley/go-metrics/README.md new file mode 100644 index 000000000..b7356b5fc --- /dev/null +++ b/vendor/github.com/rcrowley/go-metrics/README.md @@ -0,0 +1,168 @@ +go-metrics +========== + +![travis build status](https://travis-ci.org/rcrowley/go-metrics.svg?branch=master) + +Go port of Coda Hale's Metrics library: . + +Documentation: . + +Usage +----- + +Create and update metrics: + +```go +c := metrics.NewCounter() +metrics.Register("foo", c) +c.Inc(47) + +g := metrics.NewGauge() +metrics.Register("bar", g) +g.Update(47) + +r := NewRegistry() +g := metrics.NewRegisteredFunctionalGauge("cache-evictions", r, func() int64 { return cache.getEvictionsCount() }) + +s := metrics.NewExpDecaySample(1028, 0.015) // or metrics.NewUniformSample(1028) +h := metrics.NewHistogram(s) +metrics.Register("baz", h) +h.Update(47) + +m := metrics.NewMeter() +metrics.Register("quux", m) +m.Mark(47) + +t := metrics.NewTimer() +metrics.Register("bang", t) +t.Time(func() {}) +t.Update(47) +``` + +Register() is not threadsafe. For threadsafe metric registration use +GetOrRegister: + +```go +t := metrics.GetOrRegisterTimer("account.create.latency", nil) +t.Time(func() {}) +t.Update(47) +``` + +**NOTE:** Be sure to unregister short-lived meters and timers otherwise they will +leak memory: + +```go +// Will call Stop() on the Meter to allow for garbage collection +metrics.Unregister("quux") +// Or similarly for a Timer that embeds a Meter +metrics.Unregister("bang") +``` + +Periodically log every metric in human-readable form to standard error: + +```go +go metrics.Log(metrics.DefaultRegistry, 5 * time.Second, log.New(os.Stderr, "metrics: ", log.Lmicroseconds)) +``` + +Periodically log every metric in slightly-more-parseable form to syslog: + +```go +w, _ := syslog.Dial("unixgram", "/dev/log", syslog.LOG_INFO, "metrics") +go metrics.Syslog(metrics.DefaultRegistry, 60e9, w) +``` + +Periodically emit every metric to Graphite using the [Graphite client](https://github.com/cyberdelia/go-metrics-graphite): + +```go + +import "github.com/cyberdelia/go-metrics-graphite" + +addr, _ := net.ResolveTCPAddr("tcp", "127.0.0.1:2003") +go graphite.Graphite(metrics.DefaultRegistry, 10e9, "metrics", addr) +``` + +Periodically emit every metric into InfluxDB: + +**NOTE:** this has been pulled out of the library due to constant fluctuations +in the InfluxDB API. In fact, all client libraries are on their way out. see +issues [#121](https://github.com/rcrowley/go-metrics/issues/121) and +[#124](https://github.com/rcrowley/go-metrics/issues/124) for progress and details. + +```go +import "github.com/vrischmann/go-metrics-influxdb" + +go influxdb.InfluxDB(metrics.DefaultRegistry, + 10e9, + "127.0.0.1:8086", + "database-name", + "username", + "password" +) +``` + +Periodically upload every metric to Librato using the [Librato client](https://github.com/mihasya/go-metrics-librato): + +**Note**: the client included with this repository under the `librato` package +has been deprecated and moved to the repository linked above. + +```go +import "github.com/mihasya/go-metrics-librato" + +go librato.Librato(metrics.DefaultRegistry, + 10e9, // interval + "example@example.com", // account owner email address + "token", // Librato API token + "hostname", // source + []float64{0.95}, // percentiles to send + time.Millisecond, // time unit +) +``` + +Periodically emit every metric to StatHat: + +```go +import "github.com/rcrowley/go-metrics/stathat" + +go stathat.Stathat(metrics.DefaultRegistry, 10e9, "example@example.com") +``` + +Maintain all metrics along with expvars at `/debug/metrics`: + +This uses the same mechanism as [the official expvar](http://golang.org/pkg/expvar/) +but exposed under `/debug/metrics`, which shows a json representation of all your usual expvars +as well as all your go-metrics. + + +```go +import "github.com/rcrowley/go-metrics/exp" + +exp.Exp(metrics.DefaultRegistry) +``` + +Installation +------------ + +```sh +go get github.com/rcrowley/go-metrics +``` + +StatHat support additionally requires their Go client: + +```sh +go get github.com/stathat/go +``` + +Publishing Metrics +------------------ + +Clients are available for the following destinations: + +* Librato - https://github.com/mihasya/go-metrics-librato +* Graphite - https://github.com/cyberdelia/go-metrics-graphite +* InfluxDB - https://github.com/vrischmann/go-metrics-influxdb +* Ganglia - https://github.com/appscode/metlia +* Prometheus - https://github.com/deathowl/go-metrics-prometheus +* DataDog - https://github.com/syntaqx/go-metrics-datadog +* SignalFX - https://github.com/pascallouisperez/go-metrics-signalfx +* Honeycomb - https://github.com/getspine/go-metrics-honeycomb +* Wavefront - https://github.com/wavefrontHQ/go-metrics-wavefront diff --git a/vendor/github.com/rcrowley/go-metrics/counter.go b/vendor/github.com/rcrowley/go-metrics/counter.go new file mode 100644 index 000000000..bb7b039cb --- /dev/null +++ b/vendor/github.com/rcrowley/go-metrics/counter.go @@ -0,0 +1,112 @@ +package metrics + +import "sync/atomic" + +// Counters hold an int64 value that can be incremented and decremented. +type Counter interface { + Clear() + Count() int64 + Dec(int64) + Inc(int64) + Snapshot() Counter +} + +// GetOrRegisterCounter returns an existing Counter or constructs and registers +// a new StandardCounter. +func GetOrRegisterCounter(name string, r Registry) Counter { + if nil == r { + r = DefaultRegistry + } + return r.GetOrRegister(name, NewCounter).(Counter) +} + +// NewCounter constructs a new StandardCounter. +func NewCounter() Counter { + if UseNilMetrics { + return NilCounter{} + } + return &StandardCounter{0} +} + +// NewRegisteredCounter constructs and registers a new StandardCounter. +func NewRegisteredCounter(name string, r Registry) Counter { + c := NewCounter() + if nil == r { + r = DefaultRegistry + } + r.Register(name, c) + return c +} + +// CounterSnapshot is a read-only copy of another Counter. +type CounterSnapshot int64 + +// Clear panics. +func (CounterSnapshot) Clear() { + panic("Clear called on a CounterSnapshot") +} + +// Count returns the count at the time the snapshot was taken. +func (c CounterSnapshot) Count() int64 { return int64(c) } + +// Dec panics. +func (CounterSnapshot) Dec(int64) { + panic("Dec called on a CounterSnapshot") +} + +// Inc panics. +func (CounterSnapshot) Inc(int64) { + panic("Inc called on a CounterSnapshot") +} + +// Snapshot returns the snapshot. +func (c CounterSnapshot) Snapshot() Counter { return c } + +// NilCounter is a no-op Counter. +type NilCounter struct{} + +// Clear is a no-op. +func (NilCounter) Clear() {} + +// Count is a no-op. +func (NilCounter) Count() int64 { return 0 } + +// Dec is a no-op. +func (NilCounter) Dec(i int64) {} + +// Inc is a no-op. +func (NilCounter) Inc(i int64) {} + +// Snapshot is a no-op. +func (NilCounter) Snapshot() Counter { return NilCounter{} } + +// StandardCounter is the standard implementation of a Counter and uses the +// sync/atomic package to manage a single int64 value. +type StandardCounter struct { + count int64 +} + +// Clear sets the counter to zero. +func (c *StandardCounter) Clear() { + atomic.StoreInt64(&c.count, 0) +} + +// Count returns the current count. +func (c *StandardCounter) Count() int64 { + return atomic.LoadInt64(&c.count) +} + +// Dec decrements the counter by the given amount. +func (c *StandardCounter) Dec(i int64) { + atomic.AddInt64(&c.count, -i) +} + +// Inc increments the counter by the given amount. +func (c *StandardCounter) Inc(i int64) { + atomic.AddInt64(&c.count, i) +} + +// Snapshot returns a read-only copy of the counter. +func (c *StandardCounter) Snapshot() Counter { + return CounterSnapshot(c.Count()) +} diff --git a/vendor/github.com/rcrowley/go-metrics/debug.go b/vendor/github.com/rcrowley/go-metrics/debug.go new file mode 100644 index 000000000..043ccefab --- /dev/null +++ b/vendor/github.com/rcrowley/go-metrics/debug.go @@ -0,0 +1,76 @@ +package metrics + +import ( + "runtime/debug" + "time" +) + +var ( + debugMetrics struct { + GCStats struct { + LastGC Gauge + NumGC Gauge + Pause Histogram + //PauseQuantiles Histogram + PauseTotal Gauge + } + ReadGCStats Timer + } + gcStats debug.GCStats +) + +// Capture new values for the Go garbage collector statistics exported in +// debug.GCStats. This is designed to be called as a goroutine. +func CaptureDebugGCStats(r Registry, d time.Duration) { + for _ = range time.Tick(d) { + CaptureDebugGCStatsOnce(r) + } +} + +// Capture new values for the Go garbage collector statistics exported in +// debug.GCStats. This is designed to be called in a background goroutine. +// Giving a registry which has not been given to RegisterDebugGCStats will +// panic. +// +// Be careful (but much less so) with this because debug.ReadGCStats calls +// the C function runtime·lock(runtime·mheap) which, while not a stop-the-world +// operation, isn't something you want to be doing all the time. +func CaptureDebugGCStatsOnce(r Registry) { + lastGC := gcStats.LastGC + t := time.Now() + debug.ReadGCStats(&gcStats) + debugMetrics.ReadGCStats.UpdateSince(t) + + debugMetrics.GCStats.LastGC.Update(int64(gcStats.LastGC.UnixNano())) + debugMetrics.GCStats.NumGC.Update(int64(gcStats.NumGC)) + if lastGC != gcStats.LastGC && 0 < len(gcStats.Pause) { + debugMetrics.GCStats.Pause.Update(int64(gcStats.Pause[0])) + } + //debugMetrics.GCStats.PauseQuantiles.Update(gcStats.PauseQuantiles) + debugMetrics.GCStats.PauseTotal.Update(int64(gcStats.PauseTotal)) +} + +// Register metrics for the Go garbage collector statistics exported in +// debug.GCStats. The metrics are named by their fully-qualified Go symbols, +// i.e. debug.GCStats.PauseTotal. +func RegisterDebugGCStats(r Registry) { + debugMetrics.GCStats.LastGC = NewGauge() + debugMetrics.GCStats.NumGC = NewGauge() + debugMetrics.GCStats.Pause = NewHistogram(NewExpDecaySample(1028, 0.015)) + //debugMetrics.GCStats.PauseQuantiles = NewHistogram(NewExpDecaySample(1028, 0.015)) + debugMetrics.GCStats.PauseTotal = NewGauge() + debugMetrics.ReadGCStats = NewTimer() + + r.Register("debug.GCStats.LastGC", debugMetrics.GCStats.LastGC) + r.Register("debug.GCStats.NumGC", debugMetrics.GCStats.NumGC) + r.Register("debug.GCStats.Pause", debugMetrics.GCStats.Pause) + //r.Register("debug.GCStats.PauseQuantiles", debugMetrics.GCStats.PauseQuantiles) + r.Register("debug.GCStats.PauseTotal", debugMetrics.GCStats.PauseTotal) + r.Register("debug.ReadGCStats", debugMetrics.ReadGCStats) +} + +// Allocate an initial slice for gcStats.Pause to avoid allocations during +// normal operation. +func init() { + gcStats.Pause = make([]time.Duration, 11) +} diff --git a/vendor/github.com/rcrowley/go-metrics/ewma.go b/vendor/github.com/rcrowley/go-metrics/ewma.go new file mode 100644 index 000000000..a8183dd7e --- /dev/null +++ b/vendor/github.com/rcrowley/go-metrics/ewma.go @@ -0,0 +1,138 @@ +package metrics + +import ( + "math" + "sync" + "sync/atomic" +) + +// EWMAs continuously calculate an exponentially-weighted moving average +// based on an outside source of clock ticks. +type EWMA interface { + Rate() float64 + Snapshot() EWMA + Tick() + Update(int64) +} + +// NewEWMA constructs a new EWMA with the given alpha. +func NewEWMA(alpha float64) EWMA { + if UseNilMetrics { + return NilEWMA{} + } + return &StandardEWMA{alpha: alpha} +} + +// NewEWMA1 constructs a new EWMA for a one-minute moving average. +func NewEWMA1() EWMA { + return NewEWMA(1 - math.Exp(-5.0/60.0/1)) +} + +// NewEWMA5 constructs a new EWMA for a five-minute moving average. +func NewEWMA5() EWMA { + return NewEWMA(1 - math.Exp(-5.0/60.0/5)) +} + +// NewEWMA15 constructs a new EWMA for a fifteen-minute moving average. +func NewEWMA15() EWMA { + return NewEWMA(1 - math.Exp(-5.0/60.0/15)) +} + +// EWMASnapshot is a read-only copy of another EWMA. +type EWMASnapshot float64 + +// Rate returns the rate of events per second at the time the snapshot was +// taken. +func (a EWMASnapshot) Rate() float64 { return float64(a) } + +// Snapshot returns the snapshot. +func (a EWMASnapshot) Snapshot() EWMA { return a } + +// Tick panics. +func (EWMASnapshot) Tick() { + panic("Tick called on an EWMASnapshot") +} + +// Update panics. +func (EWMASnapshot) Update(int64) { + panic("Update called on an EWMASnapshot") +} + +// NilEWMA is a no-op EWMA. +type NilEWMA struct{} + +// Rate is a no-op. +func (NilEWMA) Rate() float64 { return 0.0 } + +// Snapshot is a no-op. +func (NilEWMA) Snapshot() EWMA { return NilEWMA{} } + +// Tick is a no-op. +func (NilEWMA) Tick() {} + +// Update is a no-op. +func (NilEWMA) Update(n int64) {} + +// StandardEWMA is the standard implementation of an EWMA and tracks the number +// of uncounted events and processes them on each tick. It uses the +// sync/atomic package to manage uncounted events. +type StandardEWMA struct { + uncounted int64 // /!\ this should be the first member to ensure 64-bit alignment + alpha float64 + rate uint64 + init uint32 + mutex sync.Mutex +} + +// Rate returns the moving average rate of events per second. +func (a *StandardEWMA) Rate() float64 { + currentRate := math.Float64frombits(atomic.LoadUint64(&a.rate)) * float64(1e9) + return currentRate +} + +// Snapshot returns a read-only copy of the EWMA. +func (a *StandardEWMA) Snapshot() EWMA { + return EWMASnapshot(a.Rate()) +} + +// Tick ticks the clock to update the moving average. It assumes it is called +// every five seconds. +func (a *StandardEWMA) Tick() { + // Optimization to avoid mutex locking in the hot-path. + if atomic.LoadUint32(&a.init) == 1 { + a.updateRate(a.fetchInstantRate()) + } else { + // Slow-path: this is only needed on the first Tick() and preserves transactional updating + // of init and rate in the else block. The first conditional is needed below because + // a different thread could have set a.init = 1 between the time of the first atomic load and when + // the lock was acquired. + a.mutex.Lock() + if atomic.LoadUint32(&a.init) == 1 { + // The fetchInstantRate() uses atomic loading, which is unecessary in this critical section + // but again, this section is only invoked on the first successful Tick() operation. + a.updateRate(a.fetchInstantRate()) + } else { + atomic.StoreUint32(&a.init, 1) + atomic.StoreUint64(&a.rate, math.Float64bits(a.fetchInstantRate())) + } + a.mutex.Unlock() + } +} + +func (a *StandardEWMA) fetchInstantRate() float64 { + count := atomic.LoadInt64(&a.uncounted) + atomic.AddInt64(&a.uncounted, -count) + instantRate := float64(count) / float64(5e9) + return instantRate +} + +func (a *StandardEWMA) updateRate(instantRate float64) { + currentRate := math.Float64frombits(atomic.LoadUint64(&a.rate)) + currentRate += a.alpha * (instantRate - currentRate) + atomic.StoreUint64(&a.rate, math.Float64bits(currentRate)) +} + +// Update adds n uncounted events. +func (a *StandardEWMA) Update(n int64) { + atomic.AddInt64(&a.uncounted, n) +} diff --git a/vendor/github.com/rcrowley/go-metrics/gauge.go b/vendor/github.com/rcrowley/go-metrics/gauge.go new file mode 100644 index 000000000..cb57a9388 --- /dev/null +++ b/vendor/github.com/rcrowley/go-metrics/gauge.go @@ -0,0 +1,120 @@ +package metrics + +import "sync/atomic" + +// Gauges hold an int64 value that can be set arbitrarily. +type Gauge interface { + Snapshot() Gauge + Update(int64) + Value() int64 +} + +// GetOrRegisterGauge returns an existing Gauge or constructs and registers a +// new StandardGauge. +func GetOrRegisterGauge(name string, r Registry) Gauge { + if nil == r { + r = DefaultRegistry + } + return r.GetOrRegister(name, NewGauge).(Gauge) +} + +// NewGauge constructs a new StandardGauge. +func NewGauge() Gauge { + if UseNilMetrics { + return NilGauge{} + } + return &StandardGauge{0} +} + +// NewRegisteredGauge constructs and registers a new StandardGauge. +func NewRegisteredGauge(name string, r Registry) Gauge { + c := NewGauge() + if nil == r { + r = DefaultRegistry + } + r.Register(name, c) + return c +} + +// NewFunctionalGauge constructs a new FunctionalGauge. +func NewFunctionalGauge(f func() int64) Gauge { + if UseNilMetrics { + return NilGauge{} + } + return &FunctionalGauge{value: f} +} + +// NewRegisteredFunctionalGauge constructs and registers a new StandardGauge. +func NewRegisteredFunctionalGauge(name string, r Registry, f func() int64) Gauge { + c := NewFunctionalGauge(f) + if nil == r { + r = DefaultRegistry + } + r.Register(name, c) + return c +} + +// GaugeSnapshot is a read-only copy of another Gauge. +type GaugeSnapshot int64 + +// Snapshot returns the snapshot. +func (g GaugeSnapshot) Snapshot() Gauge { return g } + +// Update panics. +func (GaugeSnapshot) Update(int64) { + panic("Update called on a GaugeSnapshot") +} + +// Value returns the value at the time the snapshot was taken. +func (g GaugeSnapshot) Value() int64 { return int64(g) } + +// NilGauge is a no-op Gauge. +type NilGauge struct{} + +// Snapshot is a no-op. +func (NilGauge) Snapshot() Gauge { return NilGauge{} } + +// Update is a no-op. +func (NilGauge) Update(v int64) {} + +// Value is a no-op. +func (NilGauge) Value() int64 { return 0 } + +// StandardGauge is the standard implementation of a Gauge and uses the +// sync/atomic package to manage a single int64 value. +type StandardGauge struct { + value int64 +} + +// Snapshot returns a read-only copy of the gauge. +func (g *StandardGauge) Snapshot() Gauge { + return GaugeSnapshot(g.Value()) +} + +// Update updates the gauge's value. +func (g *StandardGauge) Update(v int64) { + atomic.StoreInt64(&g.value, v) +} + +// Value returns the gauge's current value. +func (g *StandardGauge) Value() int64 { + return atomic.LoadInt64(&g.value) +} + +// FunctionalGauge returns value from given function +type FunctionalGauge struct { + value func() int64 +} + +// Value returns the gauge's current value. +func (g FunctionalGauge) Value() int64 { + return g.value() +} + +// Snapshot returns the snapshot. +func (g FunctionalGauge) Snapshot() Gauge { return GaugeSnapshot(g.Value()) } + +// Update panics. +func (FunctionalGauge) Update(int64) { + panic("Update called on a FunctionalGauge") +} diff --git a/vendor/github.com/rcrowley/go-metrics/gauge_float64.go b/vendor/github.com/rcrowley/go-metrics/gauge_float64.go new file mode 100644 index 000000000..3962e6db0 --- /dev/null +++ b/vendor/github.com/rcrowley/go-metrics/gauge_float64.go @@ -0,0 +1,125 @@ +package metrics + +import ( + "math" + "sync/atomic" +) + +// GaugeFloat64s hold a float64 value that can be set arbitrarily. +type GaugeFloat64 interface { + Snapshot() GaugeFloat64 + Update(float64) + Value() float64 +} + +// GetOrRegisterGaugeFloat64 returns an existing GaugeFloat64 or constructs and registers a +// new StandardGaugeFloat64. +func GetOrRegisterGaugeFloat64(name string, r Registry) GaugeFloat64 { + if nil == r { + r = DefaultRegistry + } + return r.GetOrRegister(name, NewGaugeFloat64()).(GaugeFloat64) +} + +// NewGaugeFloat64 constructs a new StandardGaugeFloat64. +func NewGaugeFloat64() GaugeFloat64 { + if UseNilMetrics { + return NilGaugeFloat64{} + } + return &StandardGaugeFloat64{ + value: 0.0, + } +} + +// NewRegisteredGaugeFloat64 constructs and registers a new StandardGaugeFloat64. +func NewRegisteredGaugeFloat64(name string, r Registry) GaugeFloat64 { + c := NewGaugeFloat64() + if nil == r { + r = DefaultRegistry + } + r.Register(name, c) + return c +} + +// NewFunctionalGauge constructs a new FunctionalGauge. +func NewFunctionalGaugeFloat64(f func() float64) GaugeFloat64 { + if UseNilMetrics { + return NilGaugeFloat64{} + } + return &FunctionalGaugeFloat64{value: f} +} + +// NewRegisteredFunctionalGauge constructs and registers a new StandardGauge. +func NewRegisteredFunctionalGaugeFloat64(name string, r Registry, f func() float64) GaugeFloat64 { + c := NewFunctionalGaugeFloat64(f) + if nil == r { + r = DefaultRegistry + } + r.Register(name, c) + return c +} + +// GaugeFloat64Snapshot is a read-only copy of another GaugeFloat64. +type GaugeFloat64Snapshot float64 + +// Snapshot returns the snapshot. +func (g GaugeFloat64Snapshot) Snapshot() GaugeFloat64 { return g } + +// Update panics. +func (GaugeFloat64Snapshot) Update(float64) { + panic("Update called on a GaugeFloat64Snapshot") +} + +// Value returns the value at the time the snapshot was taken. +func (g GaugeFloat64Snapshot) Value() float64 { return float64(g) } + +// NilGauge is a no-op Gauge. +type NilGaugeFloat64 struct{} + +// Snapshot is a no-op. +func (NilGaugeFloat64) Snapshot() GaugeFloat64 { return NilGaugeFloat64{} } + +// Update is a no-op. +func (NilGaugeFloat64) Update(v float64) {} + +// Value is a no-op. +func (NilGaugeFloat64) Value() float64 { return 0.0 } + +// StandardGaugeFloat64 is the standard implementation of a GaugeFloat64 and uses +// sync.Mutex to manage a single float64 value. +type StandardGaugeFloat64 struct { + value uint64 +} + +// Snapshot returns a read-only copy of the gauge. +func (g *StandardGaugeFloat64) Snapshot() GaugeFloat64 { + return GaugeFloat64Snapshot(g.Value()) +} + +// Update updates the gauge's value. +func (g *StandardGaugeFloat64) Update(v float64) { + atomic.StoreUint64(&g.value, math.Float64bits(v)) +} + +// Value returns the gauge's current value. +func (g *StandardGaugeFloat64) Value() float64 { + return math.Float64frombits(atomic.LoadUint64(&g.value)) +} + +// FunctionalGaugeFloat64 returns value from given function +type FunctionalGaugeFloat64 struct { + value func() float64 +} + +// Value returns the gauge's current value. +func (g FunctionalGaugeFloat64) Value() float64 { + return g.value() +} + +// Snapshot returns the snapshot. +func (g FunctionalGaugeFloat64) Snapshot() GaugeFloat64 { return GaugeFloat64Snapshot(g.Value()) } + +// Update panics. +func (FunctionalGaugeFloat64) Update(float64) { + panic("Update called on a FunctionalGaugeFloat64") +} diff --git a/vendor/github.com/rcrowley/go-metrics/graphite.go b/vendor/github.com/rcrowley/go-metrics/graphite.go new file mode 100644 index 000000000..abd0a7d29 --- /dev/null +++ b/vendor/github.com/rcrowley/go-metrics/graphite.go @@ -0,0 +1,113 @@ +package metrics + +import ( + "bufio" + "fmt" + "log" + "net" + "strconv" + "strings" + "time" +) + +// GraphiteConfig provides a container with configuration parameters for +// the Graphite exporter +type GraphiteConfig struct { + Addr *net.TCPAddr // Network address to connect to + Registry Registry // Registry to be exported + FlushInterval time.Duration // Flush interval + DurationUnit time.Duration // Time conversion unit for durations + Prefix string // Prefix to be prepended to metric names + Percentiles []float64 // Percentiles to export from timers and histograms +} + +// Graphite is a blocking exporter function which reports metrics in r +// to a graphite server located at addr, flushing them every d duration +// and prepending metric names with prefix. +func Graphite(r Registry, d time.Duration, prefix string, addr *net.TCPAddr) { + GraphiteWithConfig(GraphiteConfig{ + Addr: addr, + Registry: r, + FlushInterval: d, + DurationUnit: time.Nanosecond, + Prefix: prefix, + Percentiles: []float64{0.5, 0.75, 0.95, 0.99, 0.999}, + }) +} + +// GraphiteWithConfig is a blocking exporter function just like Graphite, +// but it takes a GraphiteConfig instead. +func GraphiteWithConfig(c GraphiteConfig) { + log.Printf("WARNING: This go-metrics client has been DEPRECATED! It has been moved to https://github.com/cyberdelia/go-metrics-graphite and will be removed from rcrowley/go-metrics on August 12th 2015") + for _ = range time.Tick(c.FlushInterval) { + if err := graphite(&c); nil != err { + log.Println(err) + } + } +} + +// GraphiteOnce performs a single submission to Graphite, returning a +// non-nil error on failed connections. This can be used in a loop +// similar to GraphiteWithConfig for custom error handling. +func GraphiteOnce(c GraphiteConfig) error { + log.Printf("WARNING: This go-metrics client has been DEPRECATED! It has been moved to https://github.com/cyberdelia/go-metrics-graphite and will be removed from rcrowley/go-metrics on August 12th 2015") + return graphite(&c) +} + +func graphite(c *GraphiteConfig) error { + now := time.Now().Unix() + du := float64(c.DurationUnit) + conn, err := net.DialTCP("tcp", nil, c.Addr) + if nil != err { + return err + } + defer conn.Close() + w := bufio.NewWriter(conn) + c.Registry.Each(func(name string, i interface{}) { + switch metric := i.(type) { + case Counter: + fmt.Fprintf(w, "%s.%s.count %d %d\n", c.Prefix, name, metric.Count(), now) + case Gauge: + fmt.Fprintf(w, "%s.%s.value %d %d\n", c.Prefix, name, metric.Value(), now) + case GaugeFloat64: + fmt.Fprintf(w, "%s.%s.value %f %d\n", c.Prefix, name, metric.Value(), now) + case Histogram: + h := metric.Snapshot() + ps := h.Percentiles(c.Percentiles) + fmt.Fprintf(w, "%s.%s.count %d %d\n", c.Prefix, name, h.Count(), now) + fmt.Fprintf(w, "%s.%s.min %d %d\n", c.Prefix, name, h.Min(), now) + fmt.Fprintf(w, "%s.%s.max %d %d\n", c.Prefix, name, h.Max(), now) + fmt.Fprintf(w, "%s.%s.mean %.2f %d\n", c.Prefix, name, h.Mean(), now) + fmt.Fprintf(w, "%s.%s.std-dev %.2f %d\n", c.Prefix, name, h.StdDev(), now) + for psIdx, psKey := range c.Percentiles { + key := strings.Replace(strconv.FormatFloat(psKey*100.0, 'f', -1, 64), ".", "", 1) + fmt.Fprintf(w, "%s.%s.%s-percentile %.2f %d\n", c.Prefix, name, key, ps[psIdx], now) + } + case Meter: + m := metric.Snapshot() + fmt.Fprintf(w, "%s.%s.count %d %d\n", c.Prefix, name, m.Count(), now) + fmt.Fprintf(w, "%s.%s.one-minute %.2f %d\n", c.Prefix, name, m.Rate1(), now) + fmt.Fprintf(w, "%s.%s.five-minute %.2f %d\n", c.Prefix, name, m.Rate5(), now) + fmt.Fprintf(w, "%s.%s.fifteen-minute %.2f %d\n", c.Prefix, name, m.Rate15(), now) + fmt.Fprintf(w, "%s.%s.mean %.2f %d\n", c.Prefix, name, m.RateMean(), now) + case Timer: + t := metric.Snapshot() + ps := t.Percentiles(c.Percentiles) + fmt.Fprintf(w, "%s.%s.count %d %d\n", c.Prefix, name, t.Count(), now) + fmt.Fprintf(w, "%s.%s.min %d %d\n", c.Prefix, name, t.Min()/int64(du), now) + fmt.Fprintf(w, "%s.%s.max %d %d\n", c.Prefix, name, t.Max()/int64(du), now) + fmt.Fprintf(w, "%s.%s.mean %.2f %d\n", c.Prefix, name, t.Mean()/du, now) + fmt.Fprintf(w, "%s.%s.std-dev %.2f %d\n", c.Prefix, name, t.StdDev()/du, now) + for psIdx, psKey := range c.Percentiles { + key := strings.Replace(strconv.FormatFloat(psKey*100.0, 'f', -1, 64), ".", "", 1) + fmt.Fprintf(w, "%s.%s.%s-percentile %.2f %d\n", c.Prefix, name, key, ps[psIdx], now) + } + fmt.Fprintf(w, "%s.%s.one-minute %.2f %d\n", c.Prefix, name, t.Rate1(), now) + fmt.Fprintf(w, "%s.%s.five-minute %.2f %d\n", c.Prefix, name, t.Rate5(), now) + fmt.Fprintf(w, "%s.%s.fifteen-minute %.2f %d\n", c.Prefix, name, t.Rate15(), now) + fmt.Fprintf(w, "%s.%s.mean-rate %.2f %d\n", c.Prefix, name, t.RateMean(), now) + } + w.Flush() + }) + return nil +} diff --git a/vendor/github.com/rcrowley/go-metrics/healthcheck.go b/vendor/github.com/rcrowley/go-metrics/healthcheck.go new file mode 100644 index 000000000..445131cae --- /dev/null +++ b/vendor/github.com/rcrowley/go-metrics/healthcheck.go @@ -0,0 +1,61 @@ +package metrics + +// Healthchecks hold an error value describing an arbitrary up/down status. +type Healthcheck interface { + Check() + Error() error + Healthy() + Unhealthy(error) +} + +// NewHealthcheck constructs a new Healthcheck which will use the given +// function to update its status. +func NewHealthcheck(f func(Healthcheck)) Healthcheck { + if UseNilMetrics { + return NilHealthcheck{} + } + return &StandardHealthcheck{nil, f} +} + +// NilHealthcheck is a no-op. +type NilHealthcheck struct{} + +// Check is a no-op. +func (NilHealthcheck) Check() {} + +// Error is a no-op. +func (NilHealthcheck) Error() error { return nil } + +// Healthy is a no-op. +func (NilHealthcheck) Healthy() {} + +// Unhealthy is a no-op. +func (NilHealthcheck) Unhealthy(error) {} + +// StandardHealthcheck is the standard implementation of a Healthcheck and +// stores the status and a function to call to update the status. +type StandardHealthcheck struct { + err error + f func(Healthcheck) +} + +// Check runs the healthcheck function to update the healthcheck's status. +func (h *StandardHealthcheck) Check() { + h.f(h) +} + +// Error returns the healthcheck's status, which will be nil if it is healthy. +func (h *StandardHealthcheck) Error() error { + return h.err +} + +// Healthy marks the healthcheck as healthy. +func (h *StandardHealthcheck) Healthy() { + h.err = nil +} + +// Unhealthy marks the healthcheck as unhealthy. The error is stored and +// may be retrieved by the Error method. +func (h *StandardHealthcheck) Unhealthy(err error) { + h.err = err +} diff --git a/vendor/github.com/rcrowley/go-metrics/histogram.go b/vendor/github.com/rcrowley/go-metrics/histogram.go new file mode 100644 index 000000000..dbc837fe4 --- /dev/null +++ b/vendor/github.com/rcrowley/go-metrics/histogram.go @@ -0,0 +1,202 @@ +package metrics + +// Histograms calculate distribution statistics from a series of int64 values. +type Histogram interface { + Clear() + Count() int64 + Max() int64 + Mean() float64 + Min() int64 + Percentile(float64) float64 + Percentiles([]float64) []float64 + Sample() Sample + Snapshot() Histogram + StdDev() float64 + Sum() int64 + Update(int64) + Variance() float64 +} + +// GetOrRegisterHistogram returns an existing Histogram or constructs and +// registers a new StandardHistogram. +func GetOrRegisterHistogram(name string, r Registry, s Sample) Histogram { + if nil == r { + r = DefaultRegistry + } + return r.GetOrRegister(name, func() Histogram { return NewHistogram(s) }).(Histogram) +} + +// NewHistogram constructs a new StandardHistogram from a Sample. +func NewHistogram(s Sample) Histogram { + if UseNilMetrics { + return NilHistogram{} + } + return &StandardHistogram{sample: s} +} + +// NewRegisteredHistogram constructs and registers a new StandardHistogram from +// a Sample. +func NewRegisteredHistogram(name string, r Registry, s Sample) Histogram { + c := NewHistogram(s) + if nil == r { + r = DefaultRegistry + } + r.Register(name, c) + return c +} + +// HistogramSnapshot is a read-only copy of another Histogram. +type HistogramSnapshot struct { + sample *SampleSnapshot +} + +// Clear panics. +func (*HistogramSnapshot) Clear() { + panic("Clear called on a HistogramSnapshot") +} + +// Count returns the number of samples recorded at the time the snapshot was +// taken. +func (h *HistogramSnapshot) Count() int64 { return h.sample.Count() } + +// Max returns the maximum value in the sample at the time the snapshot was +// taken. +func (h *HistogramSnapshot) Max() int64 { return h.sample.Max() } + +// Mean returns the mean of the values in the sample at the time the snapshot +// was taken. +func (h *HistogramSnapshot) Mean() float64 { return h.sample.Mean() } + +// Min returns the minimum value in the sample at the time the snapshot was +// taken. +func (h *HistogramSnapshot) Min() int64 { return h.sample.Min() } + +// Percentile returns an arbitrary percentile of values in the sample at the +// time the snapshot was taken. +func (h *HistogramSnapshot) Percentile(p float64) float64 { + return h.sample.Percentile(p) +} + +// Percentiles returns a slice of arbitrary percentiles of values in the sample +// at the time the snapshot was taken. +func (h *HistogramSnapshot) Percentiles(ps []float64) []float64 { + return h.sample.Percentiles(ps) +} + +// Sample returns the Sample underlying the histogram. +func (h *HistogramSnapshot) Sample() Sample { return h.sample } + +// Snapshot returns the snapshot. +func (h *HistogramSnapshot) Snapshot() Histogram { return h } + +// StdDev returns the standard deviation of the values in the sample at the +// time the snapshot was taken. +func (h *HistogramSnapshot) StdDev() float64 { return h.sample.StdDev() } + +// Sum returns the sum in the sample at the time the snapshot was taken. +func (h *HistogramSnapshot) Sum() int64 { return h.sample.Sum() } + +// Update panics. +func (*HistogramSnapshot) Update(int64) { + panic("Update called on a HistogramSnapshot") +} + +// Variance returns the variance of inputs at the time the snapshot was taken. +func (h *HistogramSnapshot) Variance() float64 { return h.sample.Variance() } + +// NilHistogram is a no-op Histogram. +type NilHistogram struct{} + +// Clear is a no-op. +func (NilHistogram) Clear() {} + +// Count is a no-op. +func (NilHistogram) Count() int64 { return 0 } + +// Max is a no-op. +func (NilHistogram) Max() int64 { return 0 } + +// Mean is a no-op. +func (NilHistogram) Mean() float64 { return 0.0 } + +// Min is a no-op. +func (NilHistogram) Min() int64 { return 0 } + +// Percentile is a no-op. +func (NilHistogram) Percentile(p float64) float64 { return 0.0 } + +// Percentiles is a no-op. +func (NilHistogram) Percentiles(ps []float64) []float64 { + return make([]float64, len(ps)) +} + +// Sample is a no-op. +func (NilHistogram) Sample() Sample { return NilSample{} } + +// Snapshot is a no-op. +func (NilHistogram) Snapshot() Histogram { return NilHistogram{} } + +// StdDev is a no-op. +func (NilHistogram) StdDev() float64 { return 0.0 } + +// Sum is a no-op. +func (NilHistogram) Sum() int64 { return 0 } + +// Update is a no-op. +func (NilHistogram) Update(v int64) {} + +// Variance is a no-op. +func (NilHistogram) Variance() float64 { return 0.0 } + +// StandardHistogram is the standard implementation of a Histogram and uses a +// Sample to bound its memory use. +type StandardHistogram struct { + sample Sample +} + +// Clear clears the histogram and its sample. +func (h *StandardHistogram) Clear() { h.sample.Clear() } + +// Count returns the number of samples recorded since the histogram was last +// cleared. +func (h *StandardHistogram) Count() int64 { return h.sample.Count() } + +// Max returns the maximum value in the sample. +func (h *StandardHistogram) Max() int64 { return h.sample.Max() } + +// Mean returns the mean of the values in the sample. +func (h *StandardHistogram) Mean() float64 { return h.sample.Mean() } + +// Min returns the minimum value in the sample. +func (h *StandardHistogram) Min() int64 { return h.sample.Min() } + +// Percentile returns an arbitrary percentile of the values in the sample. +func (h *StandardHistogram) Percentile(p float64) float64 { + return h.sample.Percentile(p) +} + +// Percentiles returns a slice of arbitrary percentiles of the values in the +// sample. +func (h *StandardHistogram) Percentiles(ps []float64) []float64 { + return h.sample.Percentiles(ps) +} + +// Sample returns the Sample underlying the histogram. +func (h *StandardHistogram) Sample() Sample { return h.sample } + +// Snapshot returns a read-only copy of the histogram. +func (h *StandardHistogram) Snapshot() Histogram { + return &HistogramSnapshot{sample: h.sample.Snapshot().(*SampleSnapshot)} +} + +// StdDev returns the standard deviation of the values in the sample. +func (h *StandardHistogram) StdDev() float64 { return h.sample.StdDev() } + +// Sum returns the sum in the sample. +func (h *StandardHistogram) Sum() int64 { return h.sample.Sum() } + +// Update samples a new value. +func (h *StandardHistogram) Update(v int64) { h.sample.Update(v) } + +// Variance returns the variance of the values in the sample. +func (h *StandardHistogram) Variance() float64 { return h.sample.Variance() } diff --git a/vendor/github.com/rcrowley/go-metrics/json.go b/vendor/github.com/rcrowley/go-metrics/json.go new file mode 100644 index 000000000..174b9477e --- /dev/null +++ b/vendor/github.com/rcrowley/go-metrics/json.go @@ -0,0 +1,31 @@ +package metrics + +import ( + "encoding/json" + "io" + "time" +) + +// MarshalJSON returns a byte slice containing a JSON representation of all +// the metrics in the Registry. +func (r *StandardRegistry) MarshalJSON() ([]byte, error) { + return json.Marshal(r.GetAll()) +} + +// WriteJSON writes metrics from the given registry periodically to the +// specified io.Writer as JSON. +func WriteJSON(r Registry, d time.Duration, w io.Writer) { + for _ = range time.Tick(d) { + WriteJSONOnce(r, w) + } +} + +// WriteJSONOnce writes metrics from the given registry to the specified +// io.Writer as JSON. +func WriteJSONOnce(r Registry, w io.Writer) { + json.NewEncoder(w).Encode(r) +} + +func (p *PrefixedRegistry) MarshalJSON() ([]byte, error) { + return json.Marshal(p.GetAll()) +} diff --git a/vendor/github.com/rcrowley/go-metrics/log.go b/vendor/github.com/rcrowley/go-metrics/log.go new file mode 100644 index 000000000..f8074c045 --- /dev/null +++ b/vendor/github.com/rcrowley/go-metrics/log.go @@ -0,0 +1,80 @@ +package metrics + +import ( + "time" +) + +type Logger interface { + Printf(format string, v ...interface{}) +} + +func Log(r Registry, freq time.Duration, l Logger) { + LogScaled(r, freq, time.Nanosecond, l) +} + +// Output each metric in the given registry periodically using the given +// logger. Print timings in `scale` units (eg time.Millisecond) rather than nanos. +func LogScaled(r Registry, freq time.Duration, scale time.Duration, l Logger) { + du := float64(scale) + duSuffix := scale.String()[1:] + + for _ = range time.Tick(freq) { + r.Each(func(name string, i interface{}) { + switch metric := i.(type) { + case Counter: + l.Printf("counter %s\n", name) + l.Printf(" count: %9d\n", metric.Count()) + case Gauge: + l.Printf("gauge %s\n", name) + l.Printf(" value: %9d\n", metric.Value()) + case GaugeFloat64: + l.Printf("gauge %s\n", name) + l.Printf(" value: %f\n", metric.Value()) + case Healthcheck: + metric.Check() + l.Printf("healthcheck %s\n", name) + l.Printf(" error: %v\n", metric.Error()) + case Histogram: + h := metric.Snapshot() + ps := h.Percentiles([]float64{0.5, 0.75, 0.95, 0.99, 0.999}) + l.Printf("histogram %s\n", name) + l.Printf(" count: %9d\n", h.Count()) + l.Printf(" min: %9d\n", h.Min()) + l.Printf(" max: %9d\n", h.Max()) + l.Printf(" mean: %12.2f\n", h.Mean()) + l.Printf(" stddev: %12.2f\n", h.StdDev()) + l.Printf(" median: %12.2f\n", ps[0]) + l.Printf(" 75%%: %12.2f\n", ps[1]) + l.Printf(" 95%%: %12.2f\n", ps[2]) + l.Printf(" 99%%: %12.2f\n", ps[3]) + l.Printf(" 99.9%%: %12.2f\n", ps[4]) + case Meter: + m := metric.Snapshot() + l.Printf("meter %s\n", name) + l.Printf(" count: %9d\n", m.Count()) + l.Printf(" 1-min rate: %12.2f\n", m.Rate1()) + l.Printf(" 5-min rate: %12.2f\n", m.Rate5()) + l.Printf(" 15-min rate: %12.2f\n", m.Rate15()) + l.Printf(" mean rate: %12.2f\n", m.RateMean()) + case Timer: + t := metric.Snapshot() + ps := t.Percentiles([]float64{0.5, 0.75, 0.95, 0.99, 0.999}) + l.Printf("timer %s\n", name) + l.Printf(" count: %9d\n", t.Count()) + l.Printf(" min: %12.2f%s\n", float64(t.Min())/du, duSuffix) + l.Printf(" max: %12.2f%s\n", float64(t.Max())/du, duSuffix) + l.Printf(" mean: %12.2f%s\n", t.Mean()/du, duSuffix) + l.Printf(" stddev: %12.2f%s\n", t.StdDev()/du, duSuffix) + l.Printf(" median: %12.2f%s\n", ps[0]/du, duSuffix) + l.Printf(" 75%%: %12.2f%s\n", ps[1]/du, duSuffix) + l.Printf(" 95%%: %12.2f%s\n", ps[2]/du, duSuffix) + l.Printf(" 99%%: %12.2f%s\n", ps[3]/du, duSuffix) + l.Printf(" 99.9%%: %12.2f%s\n", ps[4]/du, duSuffix) + l.Printf(" 1-min rate: %12.2f\n", t.Rate1()) + l.Printf(" 5-min rate: %12.2f\n", t.Rate5()) + l.Printf(" 15-min rate: %12.2f\n", t.Rate15()) + l.Printf(" mean rate: %12.2f\n", t.RateMean()) + } + }) + } +} diff --git a/vendor/github.com/rcrowley/go-metrics/memory.md b/vendor/github.com/rcrowley/go-metrics/memory.md new file mode 100644 index 000000000..47454f54b --- /dev/null +++ b/vendor/github.com/rcrowley/go-metrics/memory.md @@ -0,0 +1,285 @@ +Memory usage +============ + +(Highly unscientific.) + +Command used to gather static memory usage: + +```sh +grep ^Vm "/proc/$(ps fax | grep [m]etrics-bench | awk '{print $1}')/status" +``` + +Program used to gather baseline memory usage: + +```go +package main + +import "time" + +func main() { + time.Sleep(600e9) +} +``` + +Baseline +-------- + +``` +VmPeak: 42604 kB +VmSize: 42604 kB +VmLck: 0 kB +VmHWM: 1120 kB +VmRSS: 1120 kB +VmData: 35460 kB +VmStk: 136 kB +VmExe: 1020 kB +VmLib: 1848 kB +VmPTE: 36 kB +VmSwap: 0 kB +``` + +Program used to gather metric memory usage (with other metrics being similar): + +```go +package main + +import ( + "fmt" + "metrics" + "time" +) + +func main() { + fmt.Sprintf("foo") + metrics.NewRegistry() + time.Sleep(600e9) +} +``` + +1000 counters registered +------------------------ + +``` +VmPeak: 44016 kB +VmSize: 44016 kB +VmLck: 0 kB +VmHWM: 1928 kB +VmRSS: 1928 kB +VmData: 36868 kB +VmStk: 136 kB +VmExe: 1024 kB +VmLib: 1848 kB +VmPTE: 40 kB +VmSwap: 0 kB +``` + +**1.412 kB virtual, TODO 0.808 kB resident per counter.** + +100000 counters registered +-------------------------- + +``` +VmPeak: 55024 kB +VmSize: 55024 kB +VmLck: 0 kB +VmHWM: 12440 kB +VmRSS: 12440 kB +VmData: 47876 kB +VmStk: 136 kB +VmExe: 1024 kB +VmLib: 1848 kB +VmPTE: 64 kB +VmSwap: 0 kB +``` + +**0.1242 kB virtual, 0.1132 kB resident per counter.** + +1000 gauges registered +---------------------- + +``` +VmPeak: 44012 kB +VmSize: 44012 kB +VmLck: 0 kB +VmHWM: 1928 kB +VmRSS: 1928 kB +VmData: 36868 kB +VmStk: 136 kB +VmExe: 1020 kB +VmLib: 1848 kB +VmPTE: 40 kB +VmSwap: 0 kB +``` + +**1.408 kB virtual, 0.808 kB resident per counter.** + +100000 gauges registered +------------------------ + +``` +VmPeak: 55020 kB +VmSize: 55020 kB +VmLck: 0 kB +VmHWM: 12432 kB +VmRSS: 12432 kB +VmData: 47876 kB +VmStk: 136 kB +VmExe: 1020 kB +VmLib: 1848 kB +VmPTE: 60 kB +VmSwap: 0 kB +``` + +**0.12416 kB virtual, 0.11312 resident per gauge.** + +1000 histograms with a uniform sample size of 1028 +-------------------------------------------------- + +``` +VmPeak: 72272 kB +VmSize: 72272 kB +VmLck: 0 kB +VmHWM: 16204 kB +VmRSS: 16204 kB +VmData: 65100 kB +VmStk: 136 kB +VmExe: 1048 kB +VmLib: 1848 kB +VmPTE: 80 kB +VmSwap: 0 kB +``` + +**29.668 kB virtual, TODO 15.084 resident per histogram.** + +10000 histograms with a uniform sample size of 1028 +--------------------------------------------------- + +``` +VmPeak: 256912 kB +VmSize: 256912 kB +VmLck: 0 kB +VmHWM: 146204 kB +VmRSS: 146204 kB +VmData: 249740 kB +VmStk: 136 kB +VmExe: 1048 kB +VmLib: 1848 kB +VmPTE: 448 kB +VmSwap: 0 kB +``` + +**21.4308 kB virtual, 14.5084 kB resident per histogram.** + +50000 histograms with a uniform sample size of 1028 +--------------------------------------------------- + +``` +VmPeak: 908112 kB +VmSize: 908112 kB +VmLck: 0 kB +VmHWM: 645832 kB +VmRSS: 645588 kB +VmData: 900940 kB +VmStk: 136 kB +VmExe: 1048 kB +VmLib: 1848 kB +VmPTE: 1716 kB +VmSwap: 1544 kB +``` + +**17.31016 kB virtual, 12.88936 kB resident per histogram.** + +1000 histograms with an exponentially-decaying sample size of 1028 and alpha of 0.015 +------------------------------------------------------------------------------------- + +``` +VmPeak: 62480 kB +VmSize: 62480 kB +VmLck: 0 kB +VmHWM: 11572 kB +VmRSS: 11572 kB +VmData: 55308 kB +VmStk: 136 kB +VmExe: 1048 kB +VmLib: 1848 kB +VmPTE: 64 kB +VmSwap: 0 kB +``` + +**19.876 kB virtual, 10.452 kB resident per histogram.** + +10000 histograms with an exponentially-decaying sample size of 1028 and alpha of 0.015 +-------------------------------------------------------------------------------------- + +``` +VmPeak: 153296 kB +VmSize: 153296 kB +VmLck: 0 kB +VmHWM: 101176 kB +VmRSS: 101176 kB +VmData: 146124 kB +VmStk: 136 kB +VmExe: 1048 kB +VmLib: 1848 kB +VmPTE: 240 kB +VmSwap: 0 kB +``` + +**11.0692 kB virtual, 10.0056 kB resident per histogram.** + +50000 histograms with an exponentially-decaying sample size of 1028 and alpha of 0.015 +-------------------------------------------------------------------------------------- + +``` +VmPeak: 557264 kB +VmSize: 557264 kB +VmLck: 0 kB +VmHWM: 501056 kB +VmRSS: 501056 kB +VmData: 550092 kB +VmStk: 136 kB +VmExe: 1048 kB +VmLib: 1848 kB +VmPTE: 1032 kB +VmSwap: 0 kB +``` + +**10.2932 kB virtual, 9.99872 kB resident per histogram.** + +1000 meters +----------- + +``` +VmPeak: 74504 kB +VmSize: 74504 kB +VmLck: 0 kB +VmHWM: 24124 kB +VmRSS: 24124 kB +VmData: 67340 kB +VmStk: 136 kB +VmExe: 1040 kB +VmLib: 1848 kB +VmPTE: 92 kB +VmSwap: 0 kB +``` + +**31.9 kB virtual, 23.004 kB resident per meter.** + +10000 meters +------------ + +``` +VmPeak: 278920 kB +VmSize: 278920 kB +VmLck: 0 kB +VmHWM: 227300 kB +VmRSS: 227300 kB +VmData: 271756 kB +VmStk: 136 kB +VmExe: 1040 kB +VmLib: 1848 kB +VmPTE: 488 kB +VmSwap: 0 kB +``` + +**23.6316 kB virtual, 22.618 kB resident per meter.** diff --git a/vendor/github.com/rcrowley/go-metrics/meter.go b/vendor/github.com/rcrowley/go-metrics/meter.go new file mode 100644 index 000000000..223669bcb --- /dev/null +++ b/vendor/github.com/rcrowley/go-metrics/meter.go @@ -0,0 +1,251 @@ +package metrics + +import ( + "math" + "sync" + "sync/atomic" + "time" +) + +// Meters count events to produce exponentially-weighted moving average rates +// at one-, five-, and fifteen-minutes and a mean rate. +type Meter interface { + Count() int64 + Mark(int64) + Rate1() float64 + Rate5() float64 + Rate15() float64 + RateMean() float64 + Snapshot() Meter + Stop() +} + +// GetOrRegisterMeter returns an existing Meter or constructs and registers a +// new StandardMeter. +// Be sure to unregister the meter from the registry once it is of no use to +// allow for garbage collection. +func GetOrRegisterMeter(name string, r Registry) Meter { + if nil == r { + r = DefaultRegistry + } + return r.GetOrRegister(name, NewMeter).(Meter) +} + +// NewMeter constructs a new StandardMeter and launches a goroutine. +// Be sure to call Stop() once the meter is of no use to allow for garbage collection. +func NewMeter() Meter { + if UseNilMetrics { + return NilMeter{} + } + m := newStandardMeter() + arbiter.Lock() + defer arbiter.Unlock() + arbiter.meters[m] = struct{}{} + if !arbiter.started { + arbiter.started = true + go arbiter.tick() + } + return m +} + +// NewMeter constructs and registers a new StandardMeter and launches a +// goroutine. +// Be sure to unregister the meter from the registry once it is of no use to +// allow for garbage collection. +func NewRegisteredMeter(name string, r Registry) Meter { + c := NewMeter() + if nil == r { + r = DefaultRegistry + } + r.Register(name, c) + return c +} + +// MeterSnapshot is a read-only copy of another Meter. +type MeterSnapshot struct { + count int64 + rate1, rate5, rate15, rateMean uint64 +} + +// Count returns the count of events at the time the snapshot was taken. +func (m *MeterSnapshot) Count() int64 { return m.count } + +// Mark panics. +func (*MeterSnapshot) Mark(n int64) { + panic("Mark called on a MeterSnapshot") +} + +// Rate1 returns the one-minute moving average rate of events per second at the +// time the snapshot was taken. +func (m *MeterSnapshot) Rate1() float64 { return math.Float64frombits(m.rate1) } + +// Rate5 returns the five-minute moving average rate of events per second at +// the time the snapshot was taken. +func (m *MeterSnapshot) Rate5() float64 { return math.Float64frombits(m.rate5) } + +// Rate15 returns the fifteen-minute moving average rate of events per second +// at the time the snapshot was taken. +func (m *MeterSnapshot) Rate15() float64 { return math.Float64frombits(m.rate15) } + +// RateMean returns the meter's mean rate of events per second at the time the +// snapshot was taken. +func (m *MeterSnapshot) RateMean() float64 { return math.Float64frombits(m.rateMean) } + +// Snapshot returns the snapshot. +func (m *MeterSnapshot) Snapshot() Meter { return m } + +// Stop is a no-op. +func (m *MeterSnapshot) Stop() {} + +// NilMeter is a no-op Meter. +type NilMeter struct{} + +// Count is a no-op. +func (NilMeter) Count() int64 { return 0 } + +// Mark is a no-op. +func (NilMeter) Mark(n int64) {} + +// Rate1 is a no-op. +func (NilMeter) Rate1() float64 { return 0.0 } + +// Rate5 is a no-op. +func (NilMeter) Rate5() float64 { return 0.0 } + +// Rate15is a no-op. +func (NilMeter) Rate15() float64 { return 0.0 } + +// RateMean is a no-op. +func (NilMeter) RateMean() float64 { return 0.0 } + +// Snapshot is a no-op. +func (NilMeter) Snapshot() Meter { return NilMeter{} } + +// Stop is a no-op. +func (NilMeter) Stop() {} + +// StandardMeter is the standard implementation of a Meter. +type StandardMeter struct { + snapshot *MeterSnapshot + a1, a5, a15 EWMA + startTime time.Time + stopped uint32 +} + +func newStandardMeter() *StandardMeter { + return &StandardMeter{ + snapshot: &MeterSnapshot{}, + a1: NewEWMA1(), + a5: NewEWMA5(), + a15: NewEWMA15(), + startTime: time.Now(), + } +} + +// Stop stops the meter, Mark() will be a no-op if you use it after being stopped. +func (m *StandardMeter) Stop() { + if atomic.CompareAndSwapUint32(&m.stopped, 0, 1) { + arbiter.Lock() + delete(arbiter.meters, m) + arbiter.Unlock() + } +} + +// Count returns the number of events recorded. +func (m *StandardMeter) Count() int64 { + return atomic.LoadInt64(&m.snapshot.count) +} + +// Mark records the occurance of n events. +func (m *StandardMeter) Mark(n int64) { + if atomic.LoadUint32(&m.stopped) == 1 { + return + } + + atomic.AddInt64(&m.snapshot.count, n) + + m.a1.Update(n) + m.a5.Update(n) + m.a15.Update(n) + m.updateSnapshot() +} + +// Rate1 returns the one-minute moving average rate of events per second. +func (m *StandardMeter) Rate1() float64 { + return math.Float64frombits(atomic.LoadUint64(&m.snapshot.rate1)) +} + +// Rate5 returns the five-minute moving average rate of events per second. +func (m *StandardMeter) Rate5() float64 { + return math.Float64frombits(atomic.LoadUint64(&m.snapshot.rate5)) +} + +// Rate15 returns the fifteen-minute moving average rate of events per second. +func (m *StandardMeter) Rate15() float64 { + return math.Float64frombits(atomic.LoadUint64(&m.snapshot.rate15)) +} + +// RateMean returns the meter's mean rate of events per second. +func (m *StandardMeter) RateMean() float64 { + return math.Float64frombits(atomic.LoadUint64(&m.snapshot.rateMean)) +} + +// Snapshot returns a read-only copy of the meter. +func (m *StandardMeter) Snapshot() Meter { + copiedSnapshot := MeterSnapshot{ + count: atomic.LoadInt64(&m.snapshot.count), + rate1: atomic.LoadUint64(&m.snapshot.rate1), + rate5: atomic.LoadUint64(&m.snapshot.rate5), + rate15: atomic.LoadUint64(&m.snapshot.rate15), + rateMean: atomic.LoadUint64(&m.snapshot.rateMean), + } + return &copiedSnapshot +} + +func (m *StandardMeter) updateSnapshot() { + rate1 := math.Float64bits(m.a1.Rate()) + rate5 := math.Float64bits(m.a5.Rate()) + rate15 := math.Float64bits(m.a15.Rate()) + rateMean := math.Float64bits(float64(m.Count()) / time.Since(m.startTime).Seconds()) + + atomic.StoreUint64(&m.snapshot.rate1, rate1) + atomic.StoreUint64(&m.snapshot.rate5, rate5) + atomic.StoreUint64(&m.snapshot.rate15, rate15) + atomic.StoreUint64(&m.snapshot.rateMean, rateMean) +} + +func (m *StandardMeter) tick() { + m.a1.Tick() + m.a5.Tick() + m.a15.Tick() + m.updateSnapshot() +} + +// meterArbiter ticks meters every 5s from a single goroutine. +// meters are references in a set for future stopping. +type meterArbiter struct { + sync.RWMutex + started bool + meters map[*StandardMeter]struct{} + ticker *time.Ticker +} + +var arbiter = meterArbiter{ticker: time.NewTicker(5e9), meters: make(map[*StandardMeter]struct{})} + +// Ticks meters on the scheduled interval +func (ma *meterArbiter) tick() { + for { + select { + case <-ma.ticker.C: + ma.tickMeters() + } + } +} + +func (ma *meterArbiter) tickMeters() { + ma.RLock() + defer ma.RUnlock() + for meter := range ma.meters { + meter.tick() + } +} diff --git a/vendor/github.com/rcrowley/go-metrics/metrics.go b/vendor/github.com/rcrowley/go-metrics/metrics.go new file mode 100644 index 000000000..b97a49ed1 --- /dev/null +++ b/vendor/github.com/rcrowley/go-metrics/metrics.go @@ -0,0 +1,13 @@ +// Go port of Coda Hale's Metrics library +// +// +// +// Coda Hale's original work: +package metrics + +// UseNilMetrics is checked by the constructor functions for all of the +// standard metrics. If it is true, the metric returned is a stub. +// +// This global kill-switch helps quantify the observer effect and makes +// for less cluttered pprof profiles. +var UseNilMetrics bool = false diff --git a/vendor/github.com/rcrowley/go-metrics/opentsdb.go b/vendor/github.com/rcrowley/go-metrics/opentsdb.go new file mode 100644 index 000000000..266b6c93d --- /dev/null +++ b/vendor/github.com/rcrowley/go-metrics/opentsdb.go @@ -0,0 +1,119 @@ +package metrics + +import ( + "bufio" + "fmt" + "log" + "net" + "os" + "strings" + "time" +) + +var shortHostName string = "" + +// OpenTSDBConfig provides a container with configuration parameters for +// the OpenTSDB exporter +type OpenTSDBConfig struct { + Addr *net.TCPAddr // Network address to connect to + Registry Registry // Registry to be exported + FlushInterval time.Duration // Flush interval + DurationUnit time.Duration // Time conversion unit for durations + Prefix string // Prefix to be prepended to metric names +} + +// OpenTSDB is a blocking exporter function which reports metrics in r +// to a TSDB server located at addr, flushing them every d duration +// and prepending metric names with prefix. +func OpenTSDB(r Registry, d time.Duration, prefix string, addr *net.TCPAddr) { + OpenTSDBWithConfig(OpenTSDBConfig{ + Addr: addr, + Registry: r, + FlushInterval: d, + DurationUnit: time.Nanosecond, + Prefix: prefix, + }) +} + +// OpenTSDBWithConfig is a blocking exporter function just like OpenTSDB, +// but it takes a OpenTSDBConfig instead. +func OpenTSDBWithConfig(c OpenTSDBConfig) { + for _ = range time.Tick(c.FlushInterval) { + if err := openTSDB(&c); nil != err { + log.Println(err) + } + } +} + +func getShortHostname() string { + if shortHostName == "" { + host, _ := os.Hostname() + if index := strings.Index(host, "."); index > 0 { + shortHostName = host[:index] + } else { + shortHostName = host + } + } + return shortHostName +} + +func openTSDB(c *OpenTSDBConfig) error { + shortHostname := getShortHostname() + now := time.Now().Unix() + du := float64(c.DurationUnit) + conn, err := net.DialTCP("tcp", nil, c.Addr) + if nil != err { + return err + } + defer conn.Close() + w := bufio.NewWriter(conn) + c.Registry.Each(func(name string, i interface{}) { + switch metric := i.(type) { + case Counter: + fmt.Fprintf(w, "put %s.%s.count %d %d host=%s\n", c.Prefix, name, now, metric.Count(), shortHostname) + case Gauge: + fmt.Fprintf(w, "put %s.%s.value %d %d host=%s\n", c.Prefix, name, now, metric.Value(), shortHostname) + case GaugeFloat64: + fmt.Fprintf(w, "put %s.%s.value %d %f host=%s\n", c.Prefix, name, now, metric.Value(), shortHostname) + case Histogram: + h := metric.Snapshot() + ps := h.Percentiles([]float64{0.5, 0.75, 0.95, 0.99, 0.999}) + fmt.Fprintf(w, "put %s.%s.count %d %d host=%s\n", c.Prefix, name, now, h.Count(), shortHostname) + fmt.Fprintf(w, "put %s.%s.min %d %d host=%s\n", c.Prefix, name, now, h.Min(), shortHostname) + fmt.Fprintf(w, "put %s.%s.max %d %d host=%s\n", c.Prefix, name, now, h.Max(), shortHostname) + fmt.Fprintf(w, "put %s.%s.mean %d %.2f host=%s\n", c.Prefix, name, now, h.Mean(), shortHostname) + fmt.Fprintf(w, "put %s.%s.std-dev %d %.2f host=%s\n", c.Prefix, name, now, h.StdDev(), shortHostname) + fmt.Fprintf(w, "put %s.%s.50-percentile %d %.2f host=%s\n", c.Prefix, name, now, ps[0], shortHostname) + fmt.Fprintf(w, "put %s.%s.75-percentile %d %.2f host=%s\n", c.Prefix, name, now, ps[1], shortHostname) + fmt.Fprintf(w, "put %s.%s.95-percentile %d %.2f host=%s\n", c.Prefix, name, now, ps[2], shortHostname) + fmt.Fprintf(w, "put %s.%s.99-percentile %d %.2f host=%s\n", c.Prefix, name, now, ps[3], shortHostname) + fmt.Fprintf(w, "put %s.%s.999-percentile %d %.2f host=%s\n", c.Prefix, name, now, ps[4], shortHostname) + case Meter: + m := metric.Snapshot() + fmt.Fprintf(w, "put %s.%s.count %d %d host=%s\n", c.Prefix, name, now, m.Count(), shortHostname) + fmt.Fprintf(w, "put %s.%s.one-minute %d %.2f host=%s\n", c.Prefix, name, now, m.Rate1(), shortHostname) + fmt.Fprintf(w, "put %s.%s.five-minute %d %.2f host=%s\n", c.Prefix, name, now, m.Rate5(), shortHostname) + fmt.Fprintf(w, "put %s.%s.fifteen-minute %d %.2f host=%s\n", c.Prefix, name, now, m.Rate15(), shortHostname) + fmt.Fprintf(w, "put %s.%s.mean %d %.2f host=%s\n", c.Prefix, name, now, m.RateMean(), shortHostname) + case Timer: + t := metric.Snapshot() + ps := t.Percentiles([]float64{0.5, 0.75, 0.95, 0.99, 0.999}) + fmt.Fprintf(w, "put %s.%s.count %d %d host=%s\n", c.Prefix, name, now, t.Count(), shortHostname) + fmt.Fprintf(w, "put %s.%s.min %d %d host=%s\n", c.Prefix, name, now, t.Min()/int64(du), shortHostname) + fmt.Fprintf(w, "put %s.%s.max %d %d host=%s\n", c.Prefix, name, now, t.Max()/int64(du), shortHostname) + fmt.Fprintf(w, "put %s.%s.mean %d %.2f host=%s\n", c.Prefix, name, now, t.Mean()/du, shortHostname) + fmt.Fprintf(w, "put %s.%s.std-dev %d %.2f host=%s\n", c.Prefix, name, now, t.StdDev()/du, shortHostname) + fmt.Fprintf(w, "put %s.%s.50-percentile %d %.2f host=%s\n", c.Prefix, name, now, ps[0]/du, shortHostname) + fmt.Fprintf(w, "put %s.%s.75-percentile %d %.2f host=%s\n", c.Prefix, name, now, ps[1]/du, shortHostname) + fmt.Fprintf(w, "put %s.%s.95-percentile %d %.2f host=%s\n", c.Prefix, name, now, ps[2]/du, shortHostname) + fmt.Fprintf(w, "put %s.%s.99-percentile %d %.2f host=%s\n", c.Prefix, name, now, ps[3]/du, shortHostname) + fmt.Fprintf(w, "put %s.%s.999-percentile %d %.2f host=%s\n", c.Prefix, name, now, ps[4]/du, shortHostname) + fmt.Fprintf(w, "put %s.%s.one-minute %d %.2f host=%s\n", c.Prefix, name, now, t.Rate1(), shortHostname) + fmt.Fprintf(w, "put %s.%s.five-minute %d %.2f host=%s\n", c.Prefix, name, now, t.Rate5(), shortHostname) + fmt.Fprintf(w, "put %s.%s.fifteen-minute %d %.2f host=%s\n", c.Prefix, name, now, t.Rate15(), shortHostname) + fmt.Fprintf(w, "put %s.%s.mean-rate %d %.2f host=%s\n", c.Prefix, name, now, t.RateMean(), shortHostname) + } + w.Flush() + }) + return nil +} diff --git a/vendor/github.com/rcrowley/go-metrics/registry.go b/vendor/github.com/rcrowley/go-metrics/registry.go new file mode 100644 index 000000000..b3bab64e1 --- /dev/null +++ b/vendor/github.com/rcrowley/go-metrics/registry.go @@ -0,0 +1,363 @@ +package metrics + +import ( + "fmt" + "reflect" + "strings" + "sync" +) + +// DuplicateMetric is the error returned by Registry.Register when a metric +// already exists. If you mean to Register that metric you must first +// Unregister the existing metric. +type DuplicateMetric string + +func (err DuplicateMetric) Error() string { + return fmt.Sprintf("duplicate metric: %s", string(err)) +} + +// A Registry holds references to a set of metrics by name and can iterate +// over them, calling callback functions provided by the user. +// +// This is an interface so as to encourage other structs to implement +// the Registry API as appropriate. +type Registry interface { + + // Call the given function for each registered metric. + Each(func(string, interface{})) + + // Get the metric by the given name or nil if none is registered. + Get(string) interface{} + + // GetAll metrics in the Registry. + GetAll() map[string]map[string]interface{} + + // Gets an existing metric or registers the given one. + // The interface can be the metric to register if not found in registry, + // or a function returning the metric for lazy instantiation. + GetOrRegister(string, interface{}) interface{} + + // Register the given metric under the given name. + Register(string, interface{}) error + + // Run all registered healthchecks. + RunHealthchecks() + + // Unregister the metric with the given name. + Unregister(string) + + // Unregister all metrics. (Mostly for testing.) + UnregisterAll() +} + +// The standard implementation of a Registry is a mutex-protected map +// of names to metrics. +type StandardRegistry struct { + metrics map[string]interface{} + mutex sync.RWMutex +} + +// Create a new registry. +func NewRegistry() Registry { + return &StandardRegistry{metrics: make(map[string]interface{})} +} + +// Call the given function for each registered metric. +func (r *StandardRegistry) Each(f func(string, interface{})) { + for name, i := range r.registered() { + f(name, i) + } +} + +// Get the metric by the given name or nil if none is registered. +func (r *StandardRegistry) Get(name string) interface{} { + r.mutex.RLock() + defer r.mutex.RUnlock() + return r.metrics[name] +} + +// Gets an existing metric or creates and registers a new one. Threadsafe +// alternative to calling Get and Register on failure. +// The interface can be the metric to register if not found in registry, +// or a function returning the metric for lazy instantiation. +func (r *StandardRegistry) GetOrRegister(name string, i interface{}) interface{} { + // access the read lock first which should be re-entrant + r.mutex.RLock() + metric, ok := r.metrics[name] + r.mutex.RUnlock() + if ok { + return metric + } + + // only take the write lock if we'll be modifying the metrics map + r.mutex.Lock() + defer r.mutex.Unlock() + if metric, ok := r.metrics[name]; ok { + return metric + } + if v := reflect.ValueOf(i); v.Kind() == reflect.Func { + i = v.Call(nil)[0].Interface() + } + r.register(name, i) + return i +} + +// Register the given metric under the given name. Returns a DuplicateMetric +// if a metric by the given name is already registered. +func (r *StandardRegistry) Register(name string, i interface{}) error { + r.mutex.Lock() + defer r.mutex.Unlock() + return r.register(name, i) +} + +// Run all registered healthchecks. +func (r *StandardRegistry) RunHealthchecks() { + r.mutex.RLock() + defer r.mutex.RUnlock() + for _, i := range r.metrics { + if h, ok := i.(Healthcheck); ok { + h.Check() + } + } +} + +// GetAll metrics in the Registry +func (r *StandardRegistry) GetAll() map[string]map[string]interface{} { + data := make(map[string]map[string]interface{}) + r.Each(func(name string, i interface{}) { + values := make(map[string]interface{}) + switch metric := i.(type) { + case Counter: + values["count"] = metric.Count() + case Gauge: + values["value"] = metric.Value() + case GaugeFloat64: + values["value"] = metric.Value() + case Healthcheck: + values["error"] = nil + metric.Check() + if err := metric.Error(); nil != err { + values["error"] = metric.Error().Error() + } + case Histogram: + h := metric.Snapshot() + ps := h.Percentiles([]float64{0.5, 0.75, 0.95, 0.99, 0.999}) + values["count"] = h.Count() + values["min"] = h.Min() + values["max"] = h.Max() + values["mean"] = h.Mean() + values["stddev"] = h.StdDev() + values["median"] = ps[0] + values["75%"] = ps[1] + values["95%"] = ps[2] + values["99%"] = ps[3] + values["99.9%"] = ps[4] + case Meter: + m := metric.Snapshot() + values["count"] = m.Count() + values["1m.rate"] = m.Rate1() + values["5m.rate"] = m.Rate5() + values["15m.rate"] = m.Rate15() + values["mean.rate"] = m.RateMean() + case Timer: + t := metric.Snapshot() + ps := t.Percentiles([]float64{0.5, 0.75, 0.95, 0.99, 0.999}) + values["count"] = t.Count() + values["min"] = t.Min() + values["max"] = t.Max() + values["mean"] = t.Mean() + values["stddev"] = t.StdDev() + values["median"] = ps[0] + values["75%"] = ps[1] + values["95%"] = ps[2] + values["99%"] = ps[3] + values["99.9%"] = ps[4] + values["1m.rate"] = t.Rate1() + values["5m.rate"] = t.Rate5() + values["15m.rate"] = t.Rate15() + values["mean.rate"] = t.RateMean() + } + data[name] = values + }) + return data +} + +// Unregister the metric with the given name. +func (r *StandardRegistry) Unregister(name string) { + r.mutex.Lock() + defer r.mutex.Unlock() + r.stop(name) + delete(r.metrics, name) +} + +// Unregister all metrics. (Mostly for testing.) +func (r *StandardRegistry) UnregisterAll() { + r.mutex.Lock() + defer r.mutex.Unlock() + for name, _ := range r.metrics { + r.stop(name) + delete(r.metrics, name) + } +} + +func (r *StandardRegistry) register(name string, i interface{}) error { + if _, ok := r.metrics[name]; ok { + return DuplicateMetric(name) + } + switch i.(type) { + case Counter, Gauge, GaugeFloat64, Healthcheck, Histogram, Meter, Timer: + r.metrics[name] = i + } + return nil +} + +func (r *StandardRegistry) registered() map[string]interface{} { + r.mutex.Lock() + defer r.mutex.Unlock() + metrics := make(map[string]interface{}, len(r.metrics)) + for name, i := range r.metrics { + metrics[name] = i + } + return metrics +} + +func (r *StandardRegistry) stop(name string) { + if i, ok := r.metrics[name]; ok { + if s, ok := i.(Stoppable); ok { + s.Stop() + } + } +} + +// Stoppable defines the metrics which has to be stopped. +type Stoppable interface { + Stop() +} + +type PrefixedRegistry struct { + underlying Registry + prefix string +} + +func NewPrefixedRegistry(prefix string) Registry { + return &PrefixedRegistry{ + underlying: NewRegistry(), + prefix: prefix, + } +} + +func NewPrefixedChildRegistry(parent Registry, prefix string) Registry { + return &PrefixedRegistry{ + underlying: parent, + prefix: prefix, + } +} + +// Call the given function for each registered metric. +func (r *PrefixedRegistry) Each(fn func(string, interface{})) { + wrappedFn := func(prefix string) func(string, interface{}) { + return func(name string, iface interface{}) { + if strings.HasPrefix(name, prefix) { + fn(name, iface) + } else { + return + } + } + } + + baseRegistry, prefix := findPrefix(r, "") + baseRegistry.Each(wrappedFn(prefix)) +} + +func findPrefix(registry Registry, prefix string) (Registry, string) { + switch r := registry.(type) { + case *PrefixedRegistry: + return findPrefix(r.underlying, r.prefix+prefix) + case *StandardRegistry: + return r, prefix + } + return nil, "" +} + +// Get the metric by the given name or nil if none is registered. +func (r *PrefixedRegistry) Get(name string) interface{} { + realName := r.prefix + name + return r.underlying.Get(realName) +} + +// Gets an existing metric or registers the given one. +// The interface can be the metric to register if not found in registry, +// or a function returning the metric for lazy instantiation. +func (r *PrefixedRegistry) GetOrRegister(name string, metric interface{}) interface{} { + realName := r.prefix + name + return r.underlying.GetOrRegister(realName, metric) +} + +// Register the given metric under the given name. The name will be prefixed. +func (r *PrefixedRegistry) Register(name string, metric interface{}) error { + realName := r.prefix + name + return r.underlying.Register(realName, metric) +} + +// Run all registered healthchecks. +func (r *PrefixedRegistry) RunHealthchecks() { + r.underlying.RunHealthchecks() +} + +// GetAll metrics in the Registry +func (r *PrefixedRegistry) GetAll() map[string]map[string]interface{} { + return r.underlying.GetAll() +} + +// Unregister the metric with the given name. The name will be prefixed. +func (r *PrefixedRegistry) Unregister(name string) { + realName := r.prefix + name + r.underlying.Unregister(realName) +} + +// Unregister all metrics. (Mostly for testing.) +func (r *PrefixedRegistry) UnregisterAll() { + r.underlying.UnregisterAll() +} + +var DefaultRegistry Registry = NewRegistry() + +// Call the given function for each registered metric. +func Each(f func(string, interface{})) { + DefaultRegistry.Each(f) +} + +// Get the metric by the given name or nil if none is registered. +func Get(name string) interface{} { + return DefaultRegistry.Get(name) +} + +// Gets an existing metric or creates and registers a new one. Threadsafe +// alternative to calling Get and Register on failure. +func GetOrRegister(name string, i interface{}) interface{} { + return DefaultRegistry.GetOrRegister(name, i) +} + +// Register the given metric under the given name. Returns a DuplicateMetric +// if a metric by the given name is already registered. +func Register(name string, i interface{}) error { + return DefaultRegistry.Register(name, i) +} + +// Register the given metric under the given name. Panics if a metric by the +// given name is already registered. +func MustRegister(name string, i interface{}) { + if err := Register(name, i); err != nil { + panic(err) + } +} + +// Run all registered healthchecks. +func RunHealthchecks() { + DefaultRegistry.RunHealthchecks() +} + +// Unregister the metric with the given name. +func Unregister(name string) { + DefaultRegistry.Unregister(name) +} diff --git a/vendor/github.com/rcrowley/go-metrics/runtime.go b/vendor/github.com/rcrowley/go-metrics/runtime.go new file mode 100644 index 000000000..11c6b785a --- /dev/null +++ b/vendor/github.com/rcrowley/go-metrics/runtime.go @@ -0,0 +1,212 @@ +package metrics + +import ( + "runtime" + "runtime/pprof" + "time" +) + +var ( + memStats runtime.MemStats + runtimeMetrics struct { + MemStats struct { + Alloc Gauge + BuckHashSys Gauge + DebugGC Gauge + EnableGC Gauge + Frees Gauge + HeapAlloc Gauge + HeapIdle Gauge + HeapInuse Gauge + HeapObjects Gauge + HeapReleased Gauge + HeapSys Gauge + LastGC Gauge + Lookups Gauge + Mallocs Gauge + MCacheInuse Gauge + MCacheSys Gauge + MSpanInuse Gauge + MSpanSys Gauge + NextGC Gauge + NumGC Gauge + GCCPUFraction GaugeFloat64 + PauseNs Histogram + PauseTotalNs Gauge + StackInuse Gauge + StackSys Gauge + Sys Gauge + TotalAlloc Gauge + } + NumCgoCall Gauge + NumGoroutine Gauge + NumThread Gauge + ReadMemStats Timer + } + frees uint64 + lookups uint64 + mallocs uint64 + numGC uint32 + numCgoCalls int64 + + threadCreateProfile = pprof.Lookup("threadcreate") +) + +// Capture new values for the Go runtime statistics exported in +// runtime.MemStats. This is designed to be called as a goroutine. +func CaptureRuntimeMemStats(r Registry, d time.Duration) { + for _ = range time.Tick(d) { + CaptureRuntimeMemStatsOnce(r) + } +} + +// Capture new values for the Go runtime statistics exported in +// runtime.MemStats. This is designed to be called in a background +// goroutine. Giving a registry which has not been given to +// RegisterRuntimeMemStats will panic. +// +// Be very careful with this because runtime.ReadMemStats calls the C +// functions runtime·semacquire(&runtime·worldsema) and runtime·stoptheworld() +// and that last one does what it says on the tin. +func CaptureRuntimeMemStatsOnce(r Registry) { + t := time.Now() + runtime.ReadMemStats(&memStats) // This takes 50-200us. + runtimeMetrics.ReadMemStats.UpdateSince(t) + + runtimeMetrics.MemStats.Alloc.Update(int64(memStats.Alloc)) + runtimeMetrics.MemStats.BuckHashSys.Update(int64(memStats.BuckHashSys)) + if memStats.DebugGC { + runtimeMetrics.MemStats.DebugGC.Update(1) + } else { + runtimeMetrics.MemStats.DebugGC.Update(0) + } + if memStats.EnableGC { + runtimeMetrics.MemStats.EnableGC.Update(1) + } else { + runtimeMetrics.MemStats.EnableGC.Update(0) + } + + runtimeMetrics.MemStats.Frees.Update(int64(memStats.Frees - frees)) + runtimeMetrics.MemStats.HeapAlloc.Update(int64(memStats.HeapAlloc)) + runtimeMetrics.MemStats.HeapIdle.Update(int64(memStats.HeapIdle)) + runtimeMetrics.MemStats.HeapInuse.Update(int64(memStats.HeapInuse)) + runtimeMetrics.MemStats.HeapObjects.Update(int64(memStats.HeapObjects)) + runtimeMetrics.MemStats.HeapReleased.Update(int64(memStats.HeapReleased)) + runtimeMetrics.MemStats.HeapSys.Update(int64(memStats.HeapSys)) + runtimeMetrics.MemStats.LastGC.Update(int64(memStats.LastGC)) + runtimeMetrics.MemStats.Lookups.Update(int64(memStats.Lookups - lookups)) + runtimeMetrics.MemStats.Mallocs.Update(int64(memStats.Mallocs - mallocs)) + runtimeMetrics.MemStats.MCacheInuse.Update(int64(memStats.MCacheInuse)) + runtimeMetrics.MemStats.MCacheSys.Update(int64(memStats.MCacheSys)) + runtimeMetrics.MemStats.MSpanInuse.Update(int64(memStats.MSpanInuse)) + runtimeMetrics.MemStats.MSpanSys.Update(int64(memStats.MSpanSys)) + runtimeMetrics.MemStats.NextGC.Update(int64(memStats.NextGC)) + runtimeMetrics.MemStats.NumGC.Update(int64(memStats.NumGC - numGC)) + runtimeMetrics.MemStats.GCCPUFraction.Update(gcCPUFraction(&memStats)) + + // + i := numGC % uint32(len(memStats.PauseNs)) + ii := memStats.NumGC % uint32(len(memStats.PauseNs)) + if memStats.NumGC-numGC >= uint32(len(memStats.PauseNs)) { + for i = 0; i < uint32(len(memStats.PauseNs)); i++ { + runtimeMetrics.MemStats.PauseNs.Update(int64(memStats.PauseNs[i])) + } + } else { + if i > ii { + for ; i < uint32(len(memStats.PauseNs)); i++ { + runtimeMetrics.MemStats.PauseNs.Update(int64(memStats.PauseNs[i])) + } + i = 0 + } + for ; i < ii; i++ { + runtimeMetrics.MemStats.PauseNs.Update(int64(memStats.PauseNs[i])) + } + } + frees = memStats.Frees + lookups = memStats.Lookups + mallocs = memStats.Mallocs + numGC = memStats.NumGC + + runtimeMetrics.MemStats.PauseTotalNs.Update(int64(memStats.PauseTotalNs)) + runtimeMetrics.MemStats.StackInuse.Update(int64(memStats.StackInuse)) + runtimeMetrics.MemStats.StackSys.Update(int64(memStats.StackSys)) + runtimeMetrics.MemStats.Sys.Update(int64(memStats.Sys)) + runtimeMetrics.MemStats.TotalAlloc.Update(int64(memStats.TotalAlloc)) + + currentNumCgoCalls := numCgoCall() + runtimeMetrics.NumCgoCall.Update(currentNumCgoCalls - numCgoCalls) + numCgoCalls = currentNumCgoCalls + + runtimeMetrics.NumGoroutine.Update(int64(runtime.NumGoroutine())) + + runtimeMetrics.NumThread.Update(int64(threadCreateProfile.Count())) +} + +// Register runtimeMetrics for the Go runtime statistics exported in runtime and +// specifically runtime.MemStats. The runtimeMetrics are named by their +// fully-qualified Go symbols, i.e. runtime.MemStats.Alloc. +func RegisterRuntimeMemStats(r Registry) { + runtimeMetrics.MemStats.Alloc = NewGauge() + runtimeMetrics.MemStats.BuckHashSys = NewGauge() + runtimeMetrics.MemStats.DebugGC = NewGauge() + runtimeMetrics.MemStats.EnableGC = NewGauge() + runtimeMetrics.MemStats.Frees = NewGauge() + runtimeMetrics.MemStats.HeapAlloc = NewGauge() + runtimeMetrics.MemStats.HeapIdle = NewGauge() + runtimeMetrics.MemStats.HeapInuse = NewGauge() + runtimeMetrics.MemStats.HeapObjects = NewGauge() + runtimeMetrics.MemStats.HeapReleased = NewGauge() + runtimeMetrics.MemStats.HeapSys = NewGauge() + runtimeMetrics.MemStats.LastGC = NewGauge() + runtimeMetrics.MemStats.Lookups = NewGauge() + runtimeMetrics.MemStats.Mallocs = NewGauge() + runtimeMetrics.MemStats.MCacheInuse = NewGauge() + runtimeMetrics.MemStats.MCacheSys = NewGauge() + runtimeMetrics.MemStats.MSpanInuse = NewGauge() + runtimeMetrics.MemStats.MSpanSys = NewGauge() + runtimeMetrics.MemStats.NextGC = NewGauge() + runtimeMetrics.MemStats.NumGC = NewGauge() + runtimeMetrics.MemStats.GCCPUFraction = NewGaugeFloat64() + runtimeMetrics.MemStats.PauseNs = NewHistogram(NewExpDecaySample(1028, 0.015)) + runtimeMetrics.MemStats.PauseTotalNs = NewGauge() + runtimeMetrics.MemStats.StackInuse = NewGauge() + runtimeMetrics.MemStats.StackSys = NewGauge() + runtimeMetrics.MemStats.Sys = NewGauge() + runtimeMetrics.MemStats.TotalAlloc = NewGauge() + runtimeMetrics.NumCgoCall = NewGauge() + runtimeMetrics.NumGoroutine = NewGauge() + runtimeMetrics.NumThread = NewGauge() + runtimeMetrics.ReadMemStats = NewTimer() + + r.Register("runtime.MemStats.Alloc", runtimeMetrics.MemStats.Alloc) + r.Register("runtime.MemStats.BuckHashSys", runtimeMetrics.MemStats.BuckHashSys) + r.Register("runtime.MemStats.DebugGC", runtimeMetrics.MemStats.DebugGC) + r.Register("runtime.MemStats.EnableGC", runtimeMetrics.MemStats.EnableGC) + r.Register("runtime.MemStats.Frees", runtimeMetrics.MemStats.Frees) + r.Register("runtime.MemStats.HeapAlloc", runtimeMetrics.MemStats.HeapAlloc) + r.Register("runtime.MemStats.HeapIdle", runtimeMetrics.MemStats.HeapIdle) + r.Register("runtime.MemStats.HeapInuse", runtimeMetrics.MemStats.HeapInuse) + r.Register("runtime.MemStats.HeapObjects", runtimeMetrics.MemStats.HeapObjects) + r.Register("runtime.MemStats.HeapReleased", runtimeMetrics.MemStats.HeapReleased) + r.Register("runtime.MemStats.HeapSys", runtimeMetrics.MemStats.HeapSys) + r.Register("runtime.MemStats.LastGC", runtimeMetrics.MemStats.LastGC) + r.Register("runtime.MemStats.Lookups", runtimeMetrics.MemStats.Lookups) + r.Register("runtime.MemStats.Mallocs", runtimeMetrics.MemStats.Mallocs) + r.Register("runtime.MemStats.MCacheInuse", runtimeMetrics.MemStats.MCacheInuse) + r.Register("runtime.MemStats.MCacheSys", runtimeMetrics.MemStats.MCacheSys) + r.Register("runtime.MemStats.MSpanInuse", runtimeMetrics.MemStats.MSpanInuse) + r.Register("runtime.MemStats.MSpanSys", runtimeMetrics.MemStats.MSpanSys) + r.Register("runtime.MemStats.NextGC", runtimeMetrics.MemStats.NextGC) + r.Register("runtime.MemStats.NumGC", runtimeMetrics.MemStats.NumGC) + r.Register("runtime.MemStats.GCCPUFraction", runtimeMetrics.MemStats.GCCPUFraction) + r.Register("runtime.MemStats.PauseNs", runtimeMetrics.MemStats.PauseNs) + r.Register("runtime.MemStats.PauseTotalNs", runtimeMetrics.MemStats.PauseTotalNs) + r.Register("runtime.MemStats.StackInuse", runtimeMetrics.MemStats.StackInuse) + r.Register("runtime.MemStats.StackSys", runtimeMetrics.MemStats.StackSys) + r.Register("runtime.MemStats.Sys", runtimeMetrics.MemStats.Sys) + r.Register("runtime.MemStats.TotalAlloc", runtimeMetrics.MemStats.TotalAlloc) + r.Register("runtime.NumCgoCall", runtimeMetrics.NumCgoCall) + r.Register("runtime.NumGoroutine", runtimeMetrics.NumGoroutine) + r.Register("runtime.NumThread", runtimeMetrics.NumThread) + r.Register("runtime.ReadMemStats", runtimeMetrics.ReadMemStats) +} diff --git a/vendor/github.com/rcrowley/go-metrics/runtime_cgo.go b/vendor/github.com/rcrowley/go-metrics/runtime_cgo.go new file mode 100644 index 000000000..e3391f4e8 --- /dev/null +++ b/vendor/github.com/rcrowley/go-metrics/runtime_cgo.go @@ -0,0 +1,10 @@ +// +build cgo +// +build !appengine + +package metrics + +import "runtime" + +func numCgoCall() int64 { + return runtime.NumCgoCall() +} diff --git a/vendor/github.com/rcrowley/go-metrics/runtime_gccpufraction.go b/vendor/github.com/rcrowley/go-metrics/runtime_gccpufraction.go new file mode 100644 index 000000000..ca12c05ba --- /dev/null +++ b/vendor/github.com/rcrowley/go-metrics/runtime_gccpufraction.go @@ -0,0 +1,9 @@ +// +build go1.5 + +package metrics + +import "runtime" + +func gcCPUFraction(memStats *runtime.MemStats) float64 { + return memStats.GCCPUFraction +} diff --git a/vendor/github.com/rcrowley/go-metrics/runtime_no_cgo.go b/vendor/github.com/rcrowley/go-metrics/runtime_no_cgo.go new file mode 100644 index 000000000..616a3b475 --- /dev/null +++ b/vendor/github.com/rcrowley/go-metrics/runtime_no_cgo.go @@ -0,0 +1,7 @@ +// +build !cgo appengine + +package metrics + +func numCgoCall() int64 { + return 0 +} diff --git a/vendor/github.com/rcrowley/go-metrics/runtime_no_gccpufraction.go b/vendor/github.com/rcrowley/go-metrics/runtime_no_gccpufraction.go new file mode 100644 index 000000000..be96aa6f1 --- /dev/null +++ b/vendor/github.com/rcrowley/go-metrics/runtime_no_gccpufraction.go @@ -0,0 +1,9 @@ +// +build !go1.5 + +package metrics + +import "runtime" + +func gcCPUFraction(memStats *runtime.MemStats) float64 { + return 0 +} diff --git a/vendor/github.com/rcrowley/go-metrics/sample.go b/vendor/github.com/rcrowley/go-metrics/sample.go new file mode 100644 index 000000000..fecee5ef6 --- /dev/null +++ b/vendor/github.com/rcrowley/go-metrics/sample.go @@ -0,0 +1,616 @@ +package metrics + +import ( + "math" + "math/rand" + "sort" + "sync" + "time" +) + +const rescaleThreshold = time.Hour + +// Samples maintain a statistically-significant selection of values from +// a stream. +type Sample interface { + Clear() + Count() int64 + Max() int64 + Mean() float64 + Min() int64 + Percentile(float64) float64 + Percentiles([]float64) []float64 + Size() int + Snapshot() Sample + StdDev() float64 + Sum() int64 + Update(int64) + Values() []int64 + Variance() float64 +} + +// ExpDecaySample is an exponentially-decaying sample using a forward-decaying +// priority reservoir. See Cormode et al's "Forward Decay: A Practical Time +// Decay Model for Streaming Systems". +// +// +type ExpDecaySample struct { + alpha float64 + count int64 + mutex sync.Mutex + reservoirSize int + t0, t1 time.Time + values *expDecaySampleHeap +} + +// NewExpDecaySample constructs a new exponentially-decaying sample with the +// given reservoir size and alpha. +func NewExpDecaySample(reservoirSize int, alpha float64) Sample { + if UseNilMetrics { + return NilSample{} + } + s := &ExpDecaySample{ + alpha: alpha, + reservoirSize: reservoirSize, + t0: time.Now(), + values: newExpDecaySampleHeap(reservoirSize), + } + s.t1 = s.t0.Add(rescaleThreshold) + return s +} + +// Clear clears all samples. +func (s *ExpDecaySample) Clear() { + s.mutex.Lock() + defer s.mutex.Unlock() + s.count = 0 + s.t0 = time.Now() + s.t1 = s.t0.Add(rescaleThreshold) + s.values.Clear() +} + +// Count returns the number of samples recorded, which may exceed the +// reservoir size. +func (s *ExpDecaySample) Count() int64 { + s.mutex.Lock() + defer s.mutex.Unlock() + return s.count +} + +// Max returns the maximum value in the sample, which may not be the maximum +// value ever to be part of the sample. +func (s *ExpDecaySample) Max() int64 { + return SampleMax(s.Values()) +} + +// Mean returns the mean of the values in the sample. +func (s *ExpDecaySample) Mean() float64 { + return SampleMean(s.Values()) +} + +// Min returns the minimum value in the sample, which may not be the minimum +// value ever to be part of the sample. +func (s *ExpDecaySample) Min() int64 { + return SampleMin(s.Values()) +} + +// Percentile returns an arbitrary percentile of values in the sample. +func (s *ExpDecaySample) Percentile(p float64) float64 { + return SamplePercentile(s.Values(), p) +} + +// Percentiles returns a slice of arbitrary percentiles of values in the +// sample. +func (s *ExpDecaySample) Percentiles(ps []float64) []float64 { + return SamplePercentiles(s.Values(), ps) +} + +// Size returns the size of the sample, which is at most the reservoir size. +func (s *ExpDecaySample) Size() int { + s.mutex.Lock() + defer s.mutex.Unlock() + return s.values.Size() +} + +// Snapshot returns a read-only copy of the sample. +func (s *ExpDecaySample) Snapshot() Sample { + s.mutex.Lock() + defer s.mutex.Unlock() + vals := s.values.Values() + values := make([]int64, len(vals)) + for i, v := range vals { + values[i] = v.v + } + return &SampleSnapshot{ + count: s.count, + values: values, + } +} + +// StdDev returns the standard deviation of the values in the sample. +func (s *ExpDecaySample) StdDev() float64 { + return SampleStdDev(s.Values()) +} + +// Sum returns the sum of the values in the sample. +func (s *ExpDecaySample) Sum() int64 { + return SampleSum(s.Values()) +} + +// Update samples a new value. +func (s *ExpDecaySample) Update(v int64) { + s.update(time.Now(), v) +} + +// Values returns a copy of the values in the sample. +func (s *ExpDecaySample) Values() []int64 { + s.mutex.Lock() + defer s.mutex.Unlock() + vals := s.values.Values() + values := make([]int64, len(vals)) + for i, v := range vals { + values[i] = v.v + } + return values +} + +// Variance returns the variance of the values in the sample. +func (s *ExpDecaySample) Variance() float64 { + return SampleVariance(s.Values()) +} + +// update samples a new value at a particular timestamp. This is a method all +// its own to facilitate testing. +func (s *ExpDecaySample) update(t time.Time, v int64) { + s.mutex.Lock() + defer s.mutex.Unlock() + s.count++ + if s.values.Size() == s.reservoirSize { + s.values.Pop() + } + s.values.Push(expDecaySample{ + k: math.Exp(t.Sub(s.t0).Seconds()*s.alpha) / rand.Float64(), + v: v, + }) + if t.After(s.t1) { + values := s.values.Values() + t0 := s.t0 + s.values.Clear() + s.t0 = t + s.t1 = s.t0.Add(rescaleThreshold) + for _, v := range values { + v.k = v.k * math.Exp(-s.alpha*s.t0.Sub(t0).Seconds()) + s.values.Push(v) + } + } +} + +// NilSample is a no-op Sample. +type NilSample struct{} + +// Clear is a no-op. +func (NilSample) Clear() {} + +// Count is a no-op. +func (NilSample) Count() int64 { return 0 } + +// Max is a no-op. +func (NilSample) Max() int64 { return 0 } + +// Mean is a no-op. +func (NilSample) Mean() float64 { return 0.0 } + +// Min is a no-op. +func (NilSample) Min() int64 { return 0 } + +// Percentile is a no-op. +func (NilSample) Percentile(p float64) float64 { return 0.0 } + +// Percentiles is a no-op. +func (NilSample) Percentiles(ps []float64) []float64 { + return make([]float64, len(ps)) +} + +// Size is a no-op. +func (NilSample) Size() int { return 0 } + +// Sample is a no-op. +func (NilSample) Snapshot() Sample { return NilSample{} } + +// StdDev is a no-op. +func (NilSample) StdDev() float64 { return 0.0 } + +// Sum is a no-op. +func (NilSample) Sum() int64 { return 0 } + +// Update is a no-op. +func (NilSample) Update(v int64) {} + +// Values is a no-op. +func (NilSample) Values() []int64 { return []int64{} } + +// Variance is a no-op. +func (NilSample) Variance() float64 { return 0.0 } + +// SampleMax returns the maximum value of the slice of int64. +func SampleMax(values []int64) int64 { + if 0 == len(values) { + return 0 + } + var max int64 = math.MinInt64 + for _, v := range values { + if max < v { + max = v + } + } + return max +} + +// SampleMean returns the mean value of the slice of int64. +func SampleMean(values []int64) float64 { + if 0 == len(values) { + return 0.0 + } + return float64(SampleSum(values)) / float64(len(values)) +} + +// SampleMin returns the minimum value of the slice of int64. +func SampleMin(values []int64) int64 { + if 0 == len(values) { + return 0 + } + var min int64 = math.MaxInt64 + for _, v := range values { + if min > v { + min = v + } + } + return min +} + +// SamplePercentiles returns an arbitrary percentile of the slice of int64. +func SamplePercentile(values int64Slice, p float64) float64 { + return SamplePercentiles(values, []float64{p})[0] +} + +// SamplePercentiles returns a slice of arbitrary percentiles of the slice of +// int64. +func SamplePercentiles(values int64Slice, ps []float64) []float64 { + scores := make([]float64, len(ps)) + size := len(values) + if size > 0 { + sort.Sort(values) + for i, p := range ps { + pos := p * float64(size+1) + if pos < 1.0 { + scores[i] = float64(values[0]) + } else if pos >= float64(size) { + scores[i] = float64(values[size-1]) + } else { + lower := float64(values[int(pos)-1]) + upper := float64(values[int(pos)]) + scores[i] = lower + (pos-math.Floor(pos))*(upper-lower) + } + } + } + return scores +} + +// SampleSnapshot is a read-only copy of another Sample. +type SampleSnapshot struct { + count int64 + values []int64 +} + +func NewSampleSnapshot(count int64, values []int64) *SampleSnapshot { + return &SampleSnapshot{ + count: count, + values: values, + } +} + +// Clear panics. +func (*SampleSnapshot) Clear() { + panic("Clear called on a SampleSnapshot") +} + +// Count returns the count of inputs at the time the snapshot was taken. +func (s *SampleSnapshot) Count() int64 { return s.count } + +// Max returns the maximal value at the time the snapshot was taken. +func (s *SampleSnapshot) Max() int64 { return SampleMax(s.values) } + +// Mean returns the mean value at the time the snapshot was taken. +func (s *SampleSnapshot) Mean() float64 { return SampleMean(s.values) } + +// Min returns the minimal value at the time the snapshot was taken. +func (s *SampleSnapshot) Min() int64 { return SampleMin(s.values) } + +// Percentile returns an arbitrary percentile of values at the time the +// snapshot was taken. +func (s *SampleSnapshot) Percentile(p float64) float64 { + return SamplePercentile(s.values, p) +} + +// Percentiles returns a slice of arbitrary percentiles of values at the time +// the snapshot was taken. +func (s *SampleSnapshot) Percentiles(ps []float64) []float64 { + return SamplePercentiles(s.values, ps) +} + +// Size returns the size of the sample at the time the snapshot was taken. +func (s *SampleSnapshot) Size() int { return len(s.values) } + +// Snapshot returns the snapshot. +func (s *SampleSnapshot) Snapshot() Sample { return s } + +// StdDev returns the standard deviation of values at the time the snapshot was +// taken. +func (s *SampleSnapshot) StdDev() float64 { return SampleStdDev(s.values) } + +// Sum returns the sum of values at the time the snapshot was taken. +func (s *SampleSnapshot) Sum() int64 { return SampleSum(s.values) } + +// Update panics. +func (*SampleSnapshot) Update(int64) { + panic("Update called on a SampleSnapshot") +} + +// Values returns a copy of the values in the sample. +func (s *SampleSnapshot) Values() []int64 { + values := make([]int64, len(s.values)) + copy(values, s.values) + return values +} + +// Variance returns the variance of values at the time the snapshot was taken. +func (s *SampleSnapshot) Variance() float64 { return SampleVariance(s.values) } + +// SampleStdDev returns the standard deviation of the slice of int64. +func SampleStdDev(values []int64) float64 { + return math.Sqrt(SampleVariance(values)) +} + +// SampleSum returns the sum of the slice of int64. +func SampleSum(values []int64) int64 { + var sum int64 + for _, v := range values { + sum += v + } + return sum +} + +// SampleVariance returns the variance of the slice of int64. +func SampleVariance(values []int64) float64 { + if 0 == len(values) { + return 0.0 + } + m := SampleMean(values) + var sum float64 + for _, v := range values { + d := float64(v) - m + sum += d * d + } + return sum / float64(len(values)) +} + +// A uniform sample using Vitter's Algorithm R. +// +// +type UniformSample struct { + count int64 + mutex sync.Mutex + reservoirSize int + values []int64 +} + +// NewUniformSample constructs a new uniform sample with the given reservoir +// size. +func NewUniformSample(reservoirSize int) Sample { + if UseNilMetrics { + return NilSample{} + } + return &UniformSample{ + reservoirSize: reservoirSize, + values: make([]int64, 0, reservoirSize), + } +} + +// Clear clears all samples. +func (s *UniformSample) Clear() { + s.mutex.Lock() + defer s.mutex.Unlock() + s.count = 0 + s.values = make([]int64, 0, s.reservoirSize) +} + +// Count returns the number of samples recorded, which may exceed the +// reservoir size. +func (s *UniformSample) Count() int64 { + s.mutex.Lock() + defer s.mutex.Unlock() + return s.count +} + +// Max returns the maximum value in the sample, which may not be the maximum +// value ever to be part of the sample. +func (s *UniformSample) Max() int64 { + s.mutex.Lock() + defer s.mutex.Unlock() + return SampleMax(s.values) +} + +// Mean returns the mean of the values in the sample. +func (s *UniformSample) Mean() float64 { + s.mutex.Lock() + defer s.mutex.Unlock() + return SampleMean(s.values) +} + +// Min returns the minimum value in the sample, which may not be the minimum +// value ever to be part of the sample. +func (s *UniformSample) Min() int64 { + s.mutex.Lock() + defer s.mutex.Unlock() + return SampleMin(s.values) +} + +// Percentile returns an arbitrary percentile of values in the sample. +func (s *UniformSample) Percentile(p float64) float64 { + s.mutex.Lock() + defer s.mutex.Unlock() + return SamplePercentile(s.values, p) +} + +// Percentiles returns a slice of arbitrary percentiles of values in the +// sample. +func (s *UniformSample) Percentiles(ps []float64) []float64 { + s.mutex.Lock() + defer s.mutex.Unlock() + return SamplePercentiles(s.values, ps) +} + +// Size returns the size of the sample, which is at most the reservoir size. +func (s *UniformSample) Size() int { + s.mutex.Lock() + defer s.mutex.Unlock() + return len(s.values) +} + +// Snapshot returns a read-only copy of the sample. +func (s *UniformSample) Snapshot() Sample { + s.mutex.Lock() + defer s.mutex.Unlock() + values := make([]int64, len(s.values)) + copy(values, s.values) + return &SampleSnapshot{ + count: s.count, + values: values, + } +} + +// StdDev returns the standard deviation of the values in the sample. +func (s *UniformSample) StdDev() float64 { + s.mutex.Lock() + defer s.mutex.Unlock() + return SampleStdDev(s.values) +} + +// Sum returns the sum of the values in the sample. +func (s *UniformSample) Sum() int64 { + s.mutex.Lock() + defer s.mutex.Unlock() + return SampleSum(s.values) +} + +// Update samples a new value. +func (s *UniformSample) Update(v int64) { + s.mutex.Lock() + defer s.mutex.Unlock() + s.count++ + if len(s.values) < s.reservoirSize { + s.values = append(s.values, v) + } else { + r := rand.Int63n(s.count) + if r < int64(len(s.values)) { + s.values[int(r)] = v + } + } +} + +// Values returns a copy of the values in the sample. +func (s *UniformSample) Values() []int64 { + s.mutex.Lock() + defer s.mutex.Unlock() + values := make([]int64, len(s.values)) + copy(values, s.values) + return values +} + +// Variance returns the variance of the values in the sample. +func (s *UniformSample) Variance() float64 { + s.mutex.Lock() + defer s.mutex.Unlock() + return SampleVariance(s.values) +} + +// expDecaySample represents an individual sample in a heap. +type expDecaySample struct { + k float64 + v int64 +} + +func newExpDecaySampleHeap(reservoirSize int) *expDecaySampleHeap { + return &expDecaySampleHeap{make([]expDecaySample, 0, reservoirSize)} +} + +// expDecaySampleHeap is a min-heap of expDecaySamples. +// The internal implementation is copied from the standard library's container/heap +type expDecaySampleHeap struct { + s []expDecaySample +} + +func (h *expDecaySampleHeap) Clear() { + h.s = h.s[:0] +} + +func (h *expDecaySampleHeap) Push(s expDecaySample) { + n := len(h.s) + h.s = h.s[0 : n+1] + h.s[n] = s + h.up(n) +} + +func (h *expDecaySampleHeap) Pop() expDecaySample { + n := len(h.s) - 1 + h.s[0], h.s[n] = h.s[n], h.s[0] + h.down(0, n) + + n = len(h.s) + s := h.s[n-1] + h.s = h.s[0 : n-1] + return s +} + +func (h *expDecaySampleHeap) Size() int { + return len(h.s) +} + +func (h *expDecaySampleHeap) Values() []expDecaySample { + return h.s +} + +func (h *expDecaySampleHeap) up(j int) { + for { + i := (j - 1) / 2 // parent + if i == j || !(h.s[j].k < h.s[i].k) { + break + } + h.s[i], h.s[j] = h.s[j], h.s[i] + j = i + } +} + +func (h *expDecaySampleHeap) down(i, n int) { + for { + j1 := 2*i + 1 + if j1 >= n || j1 < 0 { // j1 < 0 after int overflow + break + } + j := j1 // left child + if j2 := j1 + 1; j2 < n && !(h.s[j1].k < h.s[j2].k) { + j = j2 // = 2*i + 2 // right child + } + if !(h.s[j].k < h.s[i].k) { + break + } + h.s[i], h.s[j] = h.s[j], h.s[i] + i = j + } +} + +type int64Slice []int64 + +func (p int64Slice) Len() int { return len(p) } +func (p int64Slice) Less(i, j int) bool { return p[i] < p[j] } +func (p int64Slice) Swap(i, j int) { p[i], p[j] = p[j], p[i] } diff --git a/vendor/github.com/rcrowley/go-metrics/syslog.go b/vendor/github.com/rcrowley/go-metrics/syslog.go new file mode 100644 index 000000000..693f19085 --- /dev/null +++ b/vendor/github.com/rcrowley/go-metrics/syslog.go @@ -0,0 +1,78 @@ +// +build !windows + +package metrics + +import ( + "fmt" + "log/syslog" + "time" +) + +// Output each metric in the given registry to syslog periodically using +// the given syslogger. +func Syslog(r Registry, d time.Duration, w *syslog.Writer) { + for _ = range time.Tick(d) { + r.Each(func(name string, i interface{}) { + switch metric := i.(type) { + case Counter: + w.Info(fmt.Sprintf("counter %s: count: %d", name, metric.Count())) + case Gauge: + w.Info(fmt.Sprintf("gauge %s: value: %d", name, metric.Value())) + case GaugeFloat64: + w.Info(fmt.Sprintf("gauge %s: value: %f", name, metric.Value())) + case Healthcheck: + metric.Check() + w.Info(fmt.Sprintf("healthcheck %s: error: %v", name, metric.Error())) + case Histogram: + h := metric.Snapshot() + ps := h.Percentiles([]float64{0.5, 0.75, 0.95, 0.99, 0.999}) + w.Info(fmt.Sprintf( + "histogram %s: count: %d min: %d max: %d mean: %.2f stddev: %.2f median: %.2f 75%%: %.2f 95%%: %.2f 99%%: %.2f 99.9%%: %.2f", + name, + h.Count(), + h.Min(), + h.Max(), + h.Mean(), + h.StdDev(), + ps[0], + ps[1], + ps[2], + ps[3], + ps[4], + )) + case Meter: + m := metric.Snapshot() + w.Info(fmt.Sprintf( + "meter %s: count: %d 1-min: %.2f 5-min: %.2f 15-min: %.2f mean: %.2f", + name, + m.Count(), + m.Rate1(), + m.Rate5(), + m.Rate15(), + m.RateMean(), + )) + case Timer: + t := metric.Snapshot() + ps := t.Percentiles([]float64{0.5, 0.75, 0.95, 0.99, 0.999}) + w.Info(fmt.Sprintf( + "timer %s: count: %d min: %d max: %d mean: %.2f stddev: %.2f median: %.2f 75%%: %.2f 95%%: %.2f 99%%: %.2f 99.9%%: %.2f 1-min: %.2f 5-min: %.2f 15-min: %.2f mean-rate: %.2f", + name, + t.Count(), + t.Min(), + t.Max(), + t.Mean(), + t.StdDev(), + ps[0], + ps[1], + ps[2], + ps[3], + ps[4], + t.Rate1(), + t.Rate5(), + t.Rate15(), + t.RateMean(), + )) + } + }) + } +} diff --git a/vendor/github.com/rcrowley/go-metrics/timer.go b/vendor/github.com/rcrowley/go-metrics/timer.go new file mode 100644 index 000000000..d6ec4c626 --- /dev/null +++ b/vendor/github.com/rcrowley/go-metrics/timer.go @@ -0,0 +1,329 @@ +package metrics + +import ( + "sync" + "time" +) + +// Timers capture the duration and rate of events. +type Timer interface { + Count() int64 + Max() int64 + Mean() float64 + Min() int64 + Percentile(float64) float64 + Percentiles([]float64) []float64 + Rate1() float64 + Rate5() float64 + Rate15() float64 + RateMean() float64 + Snapshot() Timer + StdDev() float64 + Stop() + Sum() int64 + Time(func()) + Update(time.Duration) + UpdateSince(time.Time) + Variance() float64 +} + +// GetOrRegisterTimer returns an existing Timer or constructs and registers a +// new StandardTimer. +// Be sure to unregister the meter from the registry once it is of no use to +// allow for garbage collection. +func GetOrRegisterTimer(name string, r Registry) Timer { + if nil == r { + r = DefaultRegistry + } + return r.GetOrRegister(name, NewTimer).(Timer) +} + +// NewCustomTimer constructs a new StandardTimer from a Histogram and a Meter. +// Be sure to call Stop() once the timer is of no use to allow for garbage collection. +func NewCustomTimer(h Histogram, m Meter) Timer { + if UseNilMetrics { + return NilTimer{} + } + return &StandardTimer{ + histogram: h, + meter: m, + } +} + +// NewRegisteredTimer constructs and registers a new StandardTimer. +// Be sure to unregister the meter from the registry once it is of no use to +// allow for garbage collection. +func NewRegisteredTimer(name string, r Registry) Timer { + c := NewTimer() + if nil == r { + r = DefaultRegistry + } + r.Register(name, c) + return c +} + +// NewTimer constructs a new StandardTimer using an exponentially-decaying +// sample with the same reservoir size and alpha as UNIX load averages. +// Be sure to call Stop() once the timer is of no use to allow for garbage collection. +func NewTimer() Timer { + if UseNilMetrics { + return NilTimer{} + } + return &StandardTimer{ + histogram: NewHistogram(NewExpDecaySample(1028, 0.015)), + meter: NewMeter(), + } +} + +// NilTimer is a no-op Timer. +type NilTimer struct { + h Histogram + m Meter +} + +// Count is a no-op. +func (NilTimer) Count() int64 { return 0 } + +// Max is a no-op. +func (NilTimer) Max() int64 { return 0 } + +// Mean is a no-op. +func (NilTimer) Mean() float64 { return 0.0 } + +// Min is a no-op. +func (NilTimer) Min() int64 { return 0 } + +// Percentile is a no-op. +func (NilTimer) Percentile(p float64) float64 { return 0.0 } + +// Percentiles is a no-op. +func (NilTimer) Percentiles(ps []float64) []float64 { + return make([]float64, len(ps)) +} + +// Rate1 is a no-op. +func (NilTimer) Rate1() float64 { return 0.0 } + +// Rate5 is a no-op. +func (NilTimer) Rate5() float64 { return 0.0 } + +// Rate15 is a no-op. +func (NilTimer) Rate15() float64 { return 0.0 } + +// RateMean is a no-op. +func (NilTimer) RateMean() float64 { return 0.0 } + +// Snapshot is a no-op. +func (NilTimer) Snapshot() Timer { return NilTimer{} } + +// StdDev is a no-op. +func (NilTimer) StdDev() float64 { return 0.0 } + +// Stop is a no-op. +func (NilTimer) Stop() {} + +// Sum is a no-op. +func (NilTimer) Sum() int64 { return 0 } + +// Time is a no-op. +func (NilTimer) Time(func()) {} + +// Update is a no-op. +func (NilTimer) Update(time.Duration) {} + +// UpdateSince is a no-op. +func (NilTimer) UpdateSince(time.Time) {} + +// Variance is a no-op. +func (NilTimer) Variance() float64 { return 0.0 } + +// StandardTimer is the standard implementation of a Timer and uses a Histogram +// and Meter. +type StandardTimer struct { + histogram Histogram + meter Meter + mutex sync.Mutex +} + +// Count returns the number of events recorded. +func (t *StandardTimer) Count() int64 { + return t.histogram.Count() +} + +// Max returns the maximum value in the sample. +func (t *StandardTimer) Max() int64 { + return t.histogram.Max() +} + +// Mean returns the mean of the values in the sample. +func (t *StandardTimer) Mean() float64 { + return t.histogram.Mean() +} + +// Min returns the minimum value in the sample. +func (t *StandardTimer) Min() int64 { + return t.histogram.Min() +} + +// Percentile returns an arbitrary percentile of the values in the sample. +func (t *StandardTimer) Percentile(p float64) float64 { + return t.histogram.Percentile(p) +} + +// Percentiles returns a slice of arbitrary percentiles of the values in the +// sample. +func (t *StandardTimer) Percentiles(ps []float64) []float64 { + return t.histogram.Percentiles(ps) +} + +// Rate1 returns the one-minute moving average rate of events per second. +func (t *StandardTimer) Rate1() float64 { + return t.meter.Rate1() +} + +// Rate5 returns the five-minute moving average rate of events per second. +func (t *StandardTimer) Rate5() float64 { + return t.meter.Rate5() +} + +// Rate15 returns the fifteen-minute moving average rate of events per second. +func (t *StandardTimer) Rate15() float64 { + return t.meter.Rate15() +} + +// RateMean returns the meter's mean rate of events per second. +func (t *StandardTimer) RateMean() float64 { + return t.meter.RateMean() +} + +// Snapshot returns a read-only copy of the timer. +func (t *StandardTimer) Snapshot() Timer { + t.mutex.Lock() + defer t.mutex.Unlock() + return &TimerSnapshot{ + histogram: t.histogram.Snapshot().(*HistogramSnapshot), + meter: t.meter.Snapshot().(*MeterSnapshot), + } +} + +// StdDev returns the standard deviation of the values in the sample. +func (t *StandardTimer) StdDev() float64 { + return t.histogram.StdDev() +} + +// Stop stops the meter. +func (t *StandardTimer) Stop() { + t.meter.Stop() +} + +// Sum returns the sum in the sample. +func (t *StandardTimer) Sum() int64 { + return t.histogram.Sum() +} + +// Record the duration of the execution of the given function. +func (t *StandardTimer) Time(f func()) { + ts := time.Now() + f() + t.Update(time.Since(ts)) +} + +// Record the duration of an event. +func (t *StandardTimer) Update(d time.Duration) { + t.mutex.Lock() + defer t.mutex.Unlock() + t.histogram.Update(int64(d)) + t.meter.Mark(1) +} + +// Record the duration of an event that started at a time and ends now. +func (t *StandardTimer) UpdateSince(ts time.Time) { + t.mutex.Lock() + defer t.mutex.Unlock() + t.histogram.Update(int64(time.Since(ts))) + t.meter.Mark(1) +} + +// Variance returns the variance of the values in the sample. +func (t *StandardTimer) Variance() float64 { + return t.histogram.Variance() +} + +// TimerSnapshot is a read-only copy of another Timer. +type TimerSnapshot struct { + histogram *HistogramSnapshot + meter *MeterSnapshot +} + +// Count returns the number of events recorded at the time the snapshot was +// taken. +func (t *TimerSnapshot) Count() int64 { return t.histogram.Count() } + +// Max returns the maximum value at the time the snapshot was taken. +func (t *TimerSnapshot) Max() int64 { return t.histogram.Max() } + +// Mean returns the mean value at the time the snapshot was taken. +func (t *TimerSnapshot) Mean() float64 { return t.histogram.Mean() } + +// Min returns the minimum value at the time the snapshot was taken. +func (t *TimerSnapshot) Min() int64 { return t.histogram.Min() } + +// Percentile returns an arbitrary percentile of sampled values at the time the +// snapshot was taken. +func (t *TimerSnapshot) Percentile(p float64) float64 { + return t.histogram.Percentile(p) +} + +// Percentiles returns a slice of arbitrary percentiles of sampled values at +// the time the snapshot was taken. +func (t *TimerSnapshot) Percentiles(ps []float64) []float64 { + return t.histogram.Percentiles(ps) +} + +// Rate1 returns the one-minute moving average rate of events per second at the +// time the snapshot was taken. +func (t *TimerSnapshot) Rate1() float64 { return t.meter.Rate1() } + +// Rate5 returns the five-minute moving average rate of events per second at +// the time the snapshot was taken. +func (t *TimerSnapshot) Rate5() float64 { return t.meter.Rate5() } + +// Rate15 returns the fifteen-minute moving average rate of events per second +// at the time the snapshot was taken. +func (t *TimerSnapshot) Rate15() float64 { return t.meter.Rate15() } + +// RateMean returns the meter's mean rate of events per second at the time the +// snapshot was taken. +func (t *TimerSnapshot) RateMean() float64 { return t.meter.RateMean() } + +// Snapshot returns the snapshot. +func (t *TimerSnapshot) Snapshot() Timer { return t } + +// StdDev returns the standard deviation of the values at the time the snapshot +// was taken. +func (t *TimerSnapshot) StdDev() float64 { return t.histogram.StdDev() } + +// Stop is a no-op. +func (t *TimerSnapshot) Stop() {} + +// Sum returns the sum at the time the snapshot was taken. +func (t *TimerSnapshot) Sum() int64 { return t.histogram.Sum() } + +// Time panics. +func (*TimerSnapshot) Time(func()) { + panic("Time called on a TimerSnapshot") +} + +// Update panics. +func (*TimerSnapshot) Update(time.Duration) { + panic("Update called on a TimerSnapshot") +} + +// UpdateSince panics. +func (*TimerSnapshot) UpdateSince(time.Time) { + panic("UpdateSince called on a TimerSnapshot") +} + +// Variance returns the variance of the values at the time the snapshot was +// taken. +func (t *TimerSnapshot) Variance() float64 { return t.histogram.Variance() } diff --git a/vendor/github.com/rcrowley/go-metrics/validate.sh b/vendor/github.com/rcrowley/go-metrics/validate.sh new file mode 100644 index 000000000..c4ae91e64 --- /dev/null +++ b/vendor/github.com/rcrowley/go-metrics/validate.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +set -e + +# check there are no formatting issues +GOFMT_LINES=`gofmt -l . | wc -l | xargs` +test $GOFMT_LINES -eq 0 || echo "gofmt needs to be run, ${GOFMT_LINES} files have issues" + +# run the tests for the root package +go test -race . diff --git a/vendor/github.com/rcrowley/go-metrics/writer.go b/vendor/github.com/rcrowley/go-metrics/writer.go new file mode 100644 index 000000000..091e971d2 --- /dev/null +++ b/vendor/github.com/rcrowley/go-metrics/writer.go @@ -0,0 +1,100 @@ +package metrics + +import ( + "fmt" + "io" + "sort" + "time" +) + +// Write sorts writes each metric in the given registry periodically to the +// given io.Writer. +func Write(r Registry, d time.Duration, w io.Writer) { + for _ = range time.Tick(d) { + WriteOnce(r, w) + } +} + +// WriteOnce sorts and writes metrics in the given registry to the given +// io.Writer. +func WriteOnce(r Registry, w io.Writer) { + var namedMetrics namedMetricSlice + r.Each(func(name string, i interface{}) { + namedMetrics = append(namedMetrics, namedMetric{name, i}) + }) + + sort.Sort(namedMetrics) + for _, namedMetric := range namedMetrics { + switch metric := namedMetric.m.(type) { + case Counter: + fmt.Fprintf(w, "counter %s\n", namedMetric.name) + fmt.Fprintf(w, " count: %9d\n", metric.Count()) + case Gauge: + fmt.Fprintf(w, "gauge %s\n", namedMetric.name) + fmt.Fprintf(w, " value: %9d\n", metric.Value()) + case GaugeFloat64: + fmt.Fprintf(w, "gauge %s\n", namedMetric.name) + fmt.Fprintf(w, " value: %f\n", metric.Value()) + case Healthcheck: + metric.Check() + fmt.Fprintf(w, "healthcheck %s\n", namedMetric.name) + fmt.Fprintf(w, " error: %v\n", metric.Error()) + case Histogram: + h := metric.Snapshot() + ps := h.Percentiles([]float64{0.5, 0.75, 0.95, 0.99, 0.999}) + fmt.Fprintf(w, "histogram %s\n", namedMetric.name) + fmt.Fprintf(w, " count: %9d\n", h.Count()) + fmt.Fprintf(w, " min: %9d\n", h.Min()) + fmt.Fprintf(w, " max: %9d\n", h.Max()) + fmt.Fprintf(w, " mean: %12.2f\n", h.Mean()) + fmt.Fprintf(w, " stddev: %12.2f\n", h.StdDev()) + fmt.Fprintf(w, " median: %12.2f\n", ps[0]) + fmt.Fprintf(w, " 75%%: %12.2f\n", ps[1]) + fmt.Fprintf(w, " 95%%: %12.2f\n", ps[2]) + fmt.Fprintf(w, " 99%%: %12.2f\n", ps[3]) + fmt.Fprintf(w, " 99.9%%: %12.2f\n", ps[4]) + case Meter: + m := metric.Snapshot() + fmt.Fprintf(w, "meter %s\n", namedMetric.name) + fmt.Fprintf(w, " count: %9d\n", m.Count()) + fmt.Fprintf(w, " 1-min rate: %12.2f\n", m.Rate1()) + fmt.Fprintf(w, " 5-min rate: %12.2f\n", m.Rate5()) + fmt.Fprintf(w, " 15-min rate: %12.2f\n", m.Rate15()) + fmt.Fprintf(w, " mean rate: %12.2f\n", m.RateMean()) + case Timer: + t := metric.Snapshot() + ps := t.Percentiles([]float64{0.5, 0.75, 0.95, 0.99, 0.999}) + fmt.Fprintf(w, "timer %s\n", namedMetric.name) + fmt.Fprintf(w, " count: %9d\n", t.Count()) + fmt.Fprintf(w, " min: %9d\n", t.Min()) + fmt.Fprintf(w, " max: %9d\n", t.Max()) + fmt.Fprintf(w, " mean: %12.2f\n", t.Mean()) + fmt.Fprintf(w, " stddev: %12.2f\n", t.StdDev()) + fmt.Fprintf(w, " median: %12.2f\n", ps[0]) + fmt.Fprintf(w, " 75%%: %12.2f\n", ps[1]) + fmt.Fprintf(w, " 95%%: %12.2f\n", ps[2]) + fmt.Fprintf(w, " 99%%: %12.2f\n", ps[3]) + fmt.Fprintf(w, " 99.9%%: %12.2f\n", ps[4]) + fmt.Fprintf(w, " 1-min rate: %12.2f\n", t.Rate1()) + fmt.Fprintf(w, " 5-min rate: %12.2f\n", t.Rate5()) + fmt.Fprintf(w, " 15-min rate: %12.2f\n", t.Rate15()) + fmt.Fprintf(w, " mean rate: %12.2f\n", t.RateMean()) + } + } +} + +type namedMetric struct { + name string + m interface{} +} + +// namedMetricSlice is a slice of namedMetrics that implements sort.Interface. +type namedMetricSlice []namedMetric + +func (nms namedMetricSlice) Len() int { return len(nms) } + +func (nms namedMetricSlice) Swap(i, j int) { nms[i], nms[j] = nms[j], nms[i] } + +func (nms namedMetricSlice) Less(i, j int) bool { + return nms[i].name < nms[j].name +} diff --git a/vendor/github.com/yashtewari/glob-intersection/LICENSE b/vendor/github.com/yashtewari/glob-intersection/LICENSE new file mode 100644 index 000000000..8dada3eda --- /dev/null +++ b/vendor/github.com/yashtewari/glob-intersection/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/vendor/github.com/yashtewari/glob-intersection/README.md b/vendor/github.com/yashtewari/glob-intersection/README.md new file mode 100644 index 000000000..618a85066 --- /dev/null +++ b/vendor/github.com/yashtewari/glob-intersection/README.md @@ -0,0 +1,26 @@ +# glob-intersection +Go package to check if the set of non-empty strings matched by the intersection of two regexp-style globs is non-empty. + +### Examples +- `gintersect.NonEmpty("a.a.", ".b.b")` is `true` because both globs match the string `abab`. +- `gintersect.NonEmpty("[a-z]+", "[0-9]*)` is `false` because there are no non-empty strings that both globs match. + +### Limitations + +- It is assumed that all input is rooted at the beginning and the end, i.e, starts and ends with the regexp symbols `^` and `$` respectively. This is done because any non-rooted expressions will always match a non-empty set of non-empty strings. +- The only special symbols are: + - `.` for any character. + - `+` for 1 or more of the preceding expression. + - `*` for 0 or more of the preceding expression. + - `[` and `]` to define regexp-style character classes. + - `-` to specify Unicode ranges inside character class definitions. + - `\` escapes any special symbol, including itself. + +### Complexity + +Complexity is exponential in the number of flags (`+` or `*`) present in the glob with the smaller flag count. +Benchmarks (see [`non_empty_bench_test.go`](/non_empty_bench_test.go)) reveal that inputs where one of the globs has <= 10 flags, and both globs have 100s of characters, will run in less than a nanosecond. This should be ok for most use cases. + +### Acknowledgements + +[This StackOverflow discussion](https://stackoverflow.com/questions/18695727/algorithm-to-find-out-whether-the-matches-for-two-glob-patterns-or-regular-expr) for fleshing out the logic. diff --git a/vendor/github.com/yashtewari/glob-intersection/glob.go b/vendor/github.com/yashtewari/glob-intersection/glob.go new file mode 100644 index 000000000..54d4729be --- /dev/null +++ b/vendor/github.com/yashtewari/glob-intersection/glob.go @@ -0,0 +1,182 @@ +// Package gintersect provides methods to check whether the intersection of several globs matches a non-empty set of strings. +package gintersect + +import ( + "fmt" + "strings" +) + +// Glob represents a glob. +type Glob []Token + +// NewGlob constructs a Glob from the given string by tokenizing and then simplifying it, or reports errors if any. +func NewGlob(input string) (Glob, error) { + tokens, err := Tokenize([]rune(input)) + if err != nil { + return nil, err + } + + tokens = Simplify(tokens) + + return Glob(tokens), nil +} + +// TokenType is the type of a Token. +type TokenType uint + +const ( + TTCharacter TokenType = iota + TTDot + TTSet +) + +// Flag applies to a token. +type Flag uint + +func (f Flag) String() (s string) { + for r, flag := range flagRunes { + if f == flag { + s = string(r) + break + } + } + return +} + +const ( + FlagNone = iota + FlagPlus + FlagStar +) + +// Token is the element that makes up a Glob. +type Token interface { + Type() TokenType + Flag() Flag + SetFlag(Flag) + // Equal describes whether the given Token is exactly equal to this one, barring differences in flags. + Equal(Token) bool + String() string +} + +// token is the base for all structs implementing Token. +type token struct { + ttype TokenType + flag Flag +} + +func (t token) Type() TokenType { + return t.ttype +} + +func (t token) Flag() Flag { + return t.flag +} + +func (t *token) SetFlag(f Flag) { + t.flag = f +} + +// character is a specific rune. It implements Token. +type character struct { + token + r rune +} + +func NewCharacter(r rune) Token { + return &character{ + token: token{ttype: TTCharacter}, + r: r, + } +} + +func (c character) Equal(other Token) bool { + if c.Type() != other.Type() { + return false + } + + o := other.(*character) + return c.Rune() == o.Rune() +} + +func (c character) String() string { + return fmt.Sprintf("{character: %s flag: %s}", string(c.Rune()), c.Flag().String()) +} + +func (c character) Rune() rune { + return c.r +} + +// dot is any character. It implements Token. +type dot struct { + token +} + +func NewDot() Token { + return &dot{ + token: token{ttype: TTDot}, + } +} + +func (d dot) Equal(other Token) bool { + if d.Type() != other.Type() { + return false + } + + return true +} + +func (d dot) String() string { + return fmt.Sprintf("{dot flag: %s}", d.Flag().String()) +} + +// set is a set of characters (similar to regexp character class). +// It implements Token. +type set struct { + token + runes map[rune]bool +} + +func NewSet(runes []rune) Token { + m := map[rune]bool{} + for _, r := range runes { + m[r] = true + } + return &set{ + token: token{ttype: TTSet}, + runes: m, + } +} + +func (s set) Equal(other Token) bool { + if s.Type() != other.Type() { + return false + } + + o := other.(*set) + r1, r2 := s.Runes(), o.Runes() + + if len(r1) != len(r2) { + return false + } + + for k, _ := range r1 { + if _, ok := r2[k]; !ok { + return false + } + } + + return true +} + +func (s set) String() string { + rs := make([]string, 0, 30) + for r, _ := range s.Runes() { + rs = append(rs, string(r)) + } + return fmt.Sprintf("{set: %s flag: %s}", strings.Join(rs, ""), s.Flag().String()) +} + +func (s set) Runes() map[rune]bool { + return s.runes +} diff --git a/vendor/github.com/yashtewari/glob-intersection/match.go b/vendor/github.com/yashtewari/glob-intersection/match.go new file mode 100644 index 000000000..45a988a85 --- /dev/null +++ b/vendor/github.com/yashtewari/glob-intersection/match.go @@ -0,0 +1,91 @@ +package gintersect + +import ( + "github.com/pkg/errors" +) + +var ( + errBadImplementation = errors.New("this logical path is invalid") +) + +// Match implements single-Token matching, ignoring flags. +// Example: [a-d] and [b-e] match, while [a-z] and [0-9] do not. +func Match(t1 Token, t2 Token) bool { + var temp Token + if t1.Type() > t2.Type() { + temp = t1 + t1 = t2 + t2 = temp + } + + switch t1.Type() { + case TTCharacter: + ch := t1.(*character) + + switch t2.Type() { + case TTCharacter: + return matchCharacters(ch, t2.(*character)) + case TTDot: + return matchCharacterDot(ch, t2.(*dot)) + case TTSet: + return matchCharacterSet(ch, t2.(*set)) + default: + panic(errBadImplementation) + } + + case TTDot: + d := t1.(*dot) + + switch t2.Type() { + case TTDot: + return matchDots(d, t2.(*dot)) + case TTSet: + return matchDotSet(d, t2.(*set)) + default: + panic(errBadImplementation) + } + + case TTSet: + switch t2.Type() { + case TTSet: + return matchSets(t1.(*set), t2.(*set)) + default: + panic(errBadImplementation) + } + + default: + panic(errBadImplementation) + + } +} + +func matchCharacters(a *character, b *character) bool { + return a.Rune() == b.Rune() +} + +func matchCharacterDot(a *character, b *dot) bool { + return true +} + +func matchCharacterSet(a *character, b *set) bool { + _, ok := b.Runes()[a.Rune()] + return ok +} + +func matchDots(a *dot, b *dot) bool { + return true +} + +func matchDotSet(a *dot, b *set) bool { + return true +} + +func matchSets(a *set, b *set) bool { + for k, _ := range a.Runes() { + if _, ok := b.Runes()[k]; ok { + return true + } + } + + return false +} diff --git a/vendor/github.com/yashtewari/glob-intersection/non_empty.go b/vendor/github.com/yashtewari/glob-intersection/non_empty.go new file mode 100644 index 000000000..91cbdbde5 --- /dev/null +++ b/vendor/github.com/yashtewari/glob-intersection/non_empty.go @@ -0,0 +1,154 @@ +package gintersect + +// NonEmpty is true if the intersection of lhs and rhs matches a non-empty set of non-empty str1ngs. +func NonEmpty(lhs string, rhs string) (bool, error) { + g1, err := NewGlob(lhs) + if err != nil { + return false, err + } + + g2, err := NewGlob(rhs) + if err != nil { + return false, err + } + + var match bool + g1, g2, match = trimGlobs(g1, g2) + if !match { + return false, nil + } + + return intersectNormal(g1, g2), nil +} + +// trimGlobs removes matching prefixes and suffixes from g1, g2, or returns false if prefixes/suffixes don't match. +func trimGlobs(g1, g2 Glob) (Glob, Glob, bool) { + var l, r1, r2 int + + // Trim from the beginning until a flagged Token or a mismatch is found. + for l = 0; l < len(g1) && l < len(g2) && g1[l].Flag() == FlagNone && g2[l].Flag() == FlagNone; l++ { + if !Match(g1[l], g2[l]) { + return nil, nil, false + } + } + + // Leave one prefix Token untrimmed to avoid empty Globs because those will break the algorithm. + if l > 0 { + l-- + } + + // Trim from the end until a flagged Token or a mismatch is found. + for r1, r2 = len(g1)-1, len(g2)-1; r1 >= 0 && r1 >= l && r2 >= 0 && r2 >= l && g1[r1].Flag() == FlagNone && g2[r2].Flag() == FlagNone; r1, r2 = r1-1, r2-1 { + if !Match(g1[r1], g2[r2]) { + return nil, nil, false + } + } + + // Leave one suffix Token untrimmed to avoid empty Globs because those will break the algorithm. + if r1 < len(g1)-1 { + r1++ + r2++ + } + + return g1[l : r1+1], g2[l : r2+1], true +} + +// All uses of `intersection exists` below mean that the intersection of the globs matches a non-empty set of non-empty strings. + +// intersectNormal accepts two globs and returns a boolean describing whether their intersection exists. +// It traverses g1, g2 while ensuring that their Tokens match. +// If a flagged Token is encountered, flow of control is handed off to intersectSpecial. +func intersectNormal(g1, g2 Glob) bool { + var i, j int + for i, j = 0, 0; i < len(g1) && j < len(g2); i, j = i+1, j+1 { + if g1[i].Flag() == FlagNone && g2[j].Flag() == FlagNone { + if !Match(g1[i], g2[j]) { + return false + } + } else { + return intersectSpecial(g1[i:], g2[j:]) + } + } + + if i == len(g1) && j == len(g2) { + return true + } + + return false +} + +// intersectSpecial accepts two globs such that at least one starts with a flagged Token. +// It returns a boolean describing whether their intersection exists. +// It hands flow of control to intersectPlus or intersectStar correctly. +func intersectSpecial(g1, g2 Glob) bool { + if g1[0].Flag() != FlagNone { // If g1 starts with a Token having a Flag. + switch g1[0].Flag() { + case FlagPlus: + return intersectPlus(g1, g2) + case FlagStar: + return intersectStar(g1, g2) + } + } else { // If g2 starts with a Token having a Flag. + switch g2[0].Flag() { + case FlagPlus: + return intersectPlus(g2, g1) + case FlagStar: + return intersectStar(g2, g1) + } + } + + return false +} + +// intersectPlus accepts two globs such that plussed[0].Flag() == FlagPlus. +// It returns a boolean describing whether their intersection exists. +// It ensures that at least one token in other maches plussed[0], before handing flow of control to intersectSpecial. +func intersectPlus(plussed, other Glob) bool { + if !Match(plussed[0], other[0]) { + return false + } + return intersectStar(plussed, other[1:]) +} + +// intersectStar accepts two globs such that starred[0].Flag() == FlagStar. +// It returns a boolean describing whether their intersection exists. +// It gobbles up Tokens from other until the Tokens remaining in other intersect with starred[1:] +func intersectStar(starred, other Glob) bool { + // starToken, nextToken are the token having FlagStar and the one that follows immediately after, respectively. + var starToken, nextToken Token + + starToken = starred[0] + if len(starred) > 1 { + nextToken = starred[1] + } + + for i, t := range other { + // Start gobbl1ng up tokens in other while they match starToken. + if nextToken != nil && Match(t, nextToken) { + // When a token in other matches the token after starToken, stop gobbl1ng and try to match the two all the way. + allTheWay := intersectNormal(starred[1:], other[i:]) + // If they match all the way, the Globs intersect. + if allTheWay { + return true + } else { + // If they don't match all the way, then the current token from other should still match starToken. + if !Match(t, starToken) { + return false + } + } + } else { + // Only move forward if this token can be gobbled up by starToken. + if !Match(t, starToken) { + return false + } + } + } + + // If there was no token following starToken, and everything from other was gobbled, the Globs intersect. + if nextToken == nil { + return true + } + + // If everything from other was gobbles but there was a nextToken to match, they don't intersect. + return false +} diff --git a/vendor/github.com/yashtewari/glob-intersection/simplify.go b/vendor/github.com/yashtewari/glob-intersection/simplify.go new file mode 100644 index 000000000..7704fb10e --- /dev/null +++ b/vendor/github.com/yashtewari/glob-intersection/simplify.go @@ -0,0 +1,43 @@ +package gintersect + +// Simplify accepts a Token slice and returns a equivalient Token slice that is shorter/simpler. +// The only simplification currently applied is removing redundant flagged Tokens. +// TODO: Remove unflagged Tokens next to equivalen Tokens with FlagPlus. Example: tt+t == t+ +func Simplify(tokens []Token) []Token { + if len(tokens) == 0 { + return tokens + } + simple := make([]Token, 1, len(tokens)) + simple[0] = tokens[0] + + latest := simple[0] + + for i := 1; i < len(tokens); i++ { + handled := false + // Possible simplifications to apply if there is a flag. + if tokens[i].Flag() != FlagNone && latest.Flag() != FlagNone { + // If the token contents are the same, then apply simplification. + if tokens[i].Equal(latest) { + var flag Flag + // FlagPlus takes precedence, because: + // t+t* == t+ + // t*t+ == t+ + if tokens[i].Flag() == FlagPlus || latest.Flag() == FlagPlus { + flag = FlagPlus + } else { + flag = FlagStar + } + + simple[len(simple)-1].SetFlag(flag) + handled = true + } + } + + if !handled { + latest = tokens[i] + simple = append(simple, tokens[i]) + } + } + + return simple +} diff --git a/vendor/github.com/yashtewari/glob-intersection/test_samples.go b/vendor/github.com/yashtewari/glob-intersection/test_samples.go new file mode 100644 index 000000000..5d2922c7a --- /dev/null +++ b/vendor/github.com/yashtewari/glob-intersection/test_samples.go @@ -0,0 +1,84 @@ +package gintersect + +var ( + samplesInitialized = false + + testCharacters map[rune]Token + testCharactersPlus map[rune]Token + testCharactersStar map[rune]Token + + testDot, testDotPlus, testDotStar Token + + testLowerAlphaSet, testLowerAlphaSetPlus, lowerAplhaSetStar Token + testUpperAlphaSet, testUpperAlphaSetPlus, testUpperAlphaSetStar Token + testNumSet, testNumSetPlus, testNumSetStar Token + testSymbolSet, testSymbolSetPlus, testSymbolSetStar Token + + testEmptySet Token +) + +func initializeTestSamples() { + if samplesInitialized { + return + } + + testCharacters, testCharactersPlus, testCharactersStar = make(map[rune]Token), make(map[rune]Token), make(map[rune]Token) + + testDot, testDotPlus, testDotStar = NewDot(), NewDot(), NewDot() + testDotPlus.SetFlag(FlagPlus) + testDotStar.SetFlag(FlagStar) + + var runes []rune + runes = makeRunes('a', 'z') + + testLowerAlphaSet, testLowerAlphaSetPlus, lowerAplhaSetStar = NewSet(runes), NewSet(runes), NewSet(runes) + testLowerAlphaSetPlus.SetFlag(FlagPlus) + lowerAplhaSetStar.SetFlag(FlagStar) + + runes = makeRunes('A', 'Z') + + testUpperAlphaSet, testUpperAlphaSetPlus, testUpperAlphaSetStar = NewSet(runes), NewSet(runes), NewSet(runes) + testUpperAlphaSetPlus.SetFlag(FlagPlus) + testUpperAlphaSetStar.SetFlag(FlagStar) + + runes = makeRunes('0', '9') + + testNumSet, testNumSetPlus, testNumSetStar = NewSet(runes), NewSet(runes), NewSet(runes) + testNumSetPlus.SetFlag(FlagPlus) + testNumSetStar.SetFlag(FlagStar) + + runes = makeRunes('!', '/') + + testSymbolSet, testSymbolSetPlus, testSymbolSetStar = NewSet(runes), NewSet(runes), NewSet(runes) + testSymbolSetPlus.SetFlag(FlagPlus) + testSymbolSetStar.SetFlag(FlagStar) + + testEmptySet = NewSet([]rune{}) + + samplesInitialized = true +} + +func makeRunes(from rune, to rune) []rune { + runes := make([]rune, 0, 30) + for r := from; r <= to; r++ { + runes = append(runes, r) + addToCharacters(r) + } + + return runes +} + +func addToCharacters(r rune) { + var t Token + + t = NewCharacter(r) + testCharacters[r] = t + + t = NewCharacter(r) + t.SetFlag(FlagPlus) + testCharactersPlus[r] = t + + t = NewCharacter(r) + t.SetFlag(FlagStar) + testCharactersStar[r] = t +} diff --git a/vendor/github.com/yashtewari/glob-intersection/tokenize.go b/vendor/github.com/yashtewari/glob-intersection/tokenize.go new file mode 100644 index 000000000..0b6743165 --- /dev/null +++ b/vendor/github.com/yashtewari/glob-intersection/tokenize.go @@ -0,0 +1,251 @@ +package gintersect + +import ( + "fmt" + + "github.com/pkg/errors" +) + +// Modifier is a special character that affects lexical analysis. +type Modifier uint + +const ( + ModifierBackslash Modifier = iota +) + +var ( + // Special runes. + tokenTypeRunes = map[rune]TokenType{ + '.': TTDot, + '[': TTSet, + ']': TTSet, + } + flagRunes = map[rune]Flag{ + '+': FlagPlus, + '*': FlagStar, + } + modifierRunes = map[rune]Modifier{ + '\\': ModifierBackslash, + } + + // Errors. + ErrInvalidInput = errors.New("the input provided is invalid") + errEndOfInput = errors.New("reached end of input") +) + +// Tokenize converts a rune slice into a Token slice. +func Tokenize(input []rune) ([]Token, error) { + tokens := []Token{} + for i, t, err := nextToken(0, input); err != errEndOfInput; i, t, err = nextToken(i, input) { + if err != nil { + return nil, err + } + + tokens = append(tokens, t) + } + + return tokens, nil +} + +// nextToken yields the Token starting at the given index of input, and newIndex at which the next Token should start. +func nextToken(index int, input []rune) (newIndex int, token Token, err error) { + var r rune + var escaped bool + + newIndex, r, escaped, err = nextRune(index, input) + if err != nil { + return + } + + if !escaped { + if ttype, ok := tokenTypeRunes[r]; ok { + switch ttype { + case TTDot: + token = NewDot() + + case TTSet: + if r == ']' { + err = errors.Wrap(ErrInvalidInput, invalidInputMessage(input, newIndex, "set-close ']' with no preceding '['")) + return + } + + newIndex, token, err = nextTokenSet(newIndex, input) + if err != nil { + return + } + + default: + panic(errors.Wrapf(errBadImplementation, "encountered unhandled token type: %v", ttype)) + } + + } else if _, ok := flagRunes[r]; ok { + err = errors.Wrap(ErrInvalidInput, invalidInputMessage(input, newIndex, "flag '%s' must be preceded by a non-flag", string(r))) + return + + } else if m, ok := modifierRunes[r]; ok { + panic(errors.Wrapf(errBadImplementation, "encountered unhandled modifier: %v", m)) + } else { + // Nothing special to do. + token = NewCharacter(r) + } + } else { + // Nothing special to do. + token = NewCharacter(r) + } + + var f Flag + newIndex, f, err = nextFlag(newIndex, input) + if err == errEndOfInput { + // Let this err be passed in the next cycle, after the current token is consumed. + err = nil + } else if err != nil { + return + } + + token.SetFlag(f) + + return +} + +// nextTokenSet yields a Token having type TokenSet and starting at the given index of input. +// The next Token/Flag should start at newIndex. +func nextTokenSet(index int, input []rune) (newIndex int, t Token, err error) { + var r, prev rune + var escaped bool + + runes := make([]rune, 0, 30) + complete, prevExists := false, false + + newIndex, r, escaped, err = nextRune(index, input) + // If errEndOfInput is encountered, flow of control proceeds to the end of the function, + // where the error is handled. + if err != nil && err != errEndOfInput { + return + } + + for ; err != errEndOfInput; newIndex, r, escaped, err = nextRune(newIndex, input) { + if err != nil { + return + } + + if !escaped { + // Handle symbols. + switch r { + case '-': + if !prevExists { + err = errors.Wrap(ErrInvalidInput, invalidInputMessage(input, newIndex, "range character '-' must be preceded by a Unicode character")) + return + } + if newIndex >= len(input)-1 { + err = errors.Wrap(ErrInvalidInput, invalidInputMessage(input, newIndex, "range character '-' must be followed by a Unicode character")) + return + } + + // Get the next rune to know the extent of the range. + newIndex, r, escaped, err = nextRune(newIndex, input) + + if !escaped { + if r == ']' || r == '-' { + err = errors.Wrap(ErrInvalidInput, invalidInputMessage(input, newIndex, "range character '-' cannot be followed by a special symbol")) + return + } + } + if r < prev { + err = errors.Wrap(ErrInvalidInput, invalidInputMessage(input, newIndex, "range is out of order: '%s' comes before '%s' in Unicode", string(r), string(prev))) + return + } + + for x := prev; x <= r; x++ { + runes = append(runes, x) + } + + prevExists = false + + case ']': + complete = true + + // Nothing special to do. + default: + runes = append(runes, r) + prev, prevExists = r, true + } + } else { + // Nothing special to do. + runes = append(runes, r) + prev, prevExists = r, true + } + + // Don't move the index forward if the set is complete. + if complete { + break + } + } + + // End of input is reached before the set completes. + if !complete { + err = errors.Wrap(ErrInvalidInput, invalidInputMessage(input, newIndex, "found [ without matching ]")) + } else { + t = NewSet(runes) + } + + return +} + +// nextFlag yields the Flag starting at the given index of input, if any. +// The next Token should start at newIndex. +func nextFlag(index int, input []rune) (newIndex int, f Flag, err error) { + var escaped, ok bool + var r rune + + f = FlagNone + + newIndex, r, escaped, err = nextRune(index, input) + if err != nil { + return + } + + if !escaped { + // Revert back to index for later consumption. + if f, ok = flagRunes[r]; !ok { + newIndex = index + } + } else { + // Revert back to index for later consumption. + newIndex = index + } + + return +} + +// nextRune yields the rune starting (with modifiers) at the given index of input, with boolean escaped describing whether the rune is escaped. +// The next rune should start at newIndex. +func nextRune(index int, input []rune) (newIndex int, r rune, escaped bool, err error) { + if index >= len(input) { + newIndex = index + err = errEndOfInput + return + } + + if m, ok := modifierRunes[input[index]]; ok { + switch m { + + case ModifierBackslash: + if index < len(input)-1 { + newIndex, r, escaped = index+2, input[index+1], true + } else if index == len(input)-1 { + err = errors.Wrap(ErrInvalidInput, invalidInputMessage(input, index, "input ends with a \\ (escape) character")) + } + default: + panic(errors.Wrapf(errBadImplementation, "encountered unhandled modifier: %v", m)) + } + } else { + newIndex, r, escaped = index+1, input[index], false + } + + return +} + +// invalidInputMessage wraps the message describing invalid input with the input itself and index at which it is invalid. +func invalidInputMessage(input []rune, index int, message string, args ...interface{}) string { + return fmt.Sprintf("input:%s, pos:%d, %s", string(input), index, fmt.Sprintf(message, args...)) +} diff --git a/vendor/golang.org/x/net/html/atom/gen.go b/vendor/golang.org/x/net/html/atom/gen.go new file mode 100644 index 000000000..5d052781b --- /dev/null +++ b/vendor/golang.org/x/net/html/atom/gen.go @@ -0,0 +1,712 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +//go:generate go run gen.go +//go:generate go run gen.go -test + +package main + +import ( + "bytes" + "flag" + "fmt" + "go/format" + "io/ioutil" + "math/rand" + "os" + "sort" + "strings" +) + +// identifier converts s to a Go exported identifier. +// It converts "div" to "Div" and "accept-charset" to "AcceptCharset". +func identifier(s string) string { + b := make([]byte, 0, len(s)) + cap := true + for _, c := range s { + if c == '-' { + cap = true + continue + } + if cap && 'a' <= c && c <= 'z' { + c -= 'a' - 'A' + } + cap = false + b = append(b, byte(c)) + } + return string(b) +} + +var test = flag.Bool("test", false, "generate table_test.go") + +func genFile(name string, buf *bytes.Buffer) { + b, err := format.Source(buf.Bytes()) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + if err := ioutil.WriteFile(name, b, 0644); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} + +func main() { + flag.Parse() + + var all []string + all = append(all, elements...) + all = append(all, attributes...) + all = append(all, eventHandlers...) + all = append(all, extra...) + sort.Strings(all) + + // uniq - lists have dups + w := 0 + for _, s := range all { + if w == 0 || all[w-1] != s { + all[w] = s + w++ + } + } + all = all[:w] + + if *test { + var buf bytes.Buffer + fmt.Fprintln(&buf, "// Code generated by go generate gen.go; DO NOT EDIT.\n") + fmt.Fprintln(&buf, "//go:generate go run gen.go -test\n") + fmt.Fprintln(&buf, "package atom\n") + fmt.Fprintln(&buf, "var testAtomList = []string{") + for _, s := range all { + fmt.Fprintf(&buf, "\t%q,\n", s) + } + fmt.Fprintln(&buf, "}") + + genFile("table_test.go", &buf) + return + } + + // Find hash that minimizes table size. + var best *table + for i := 0; i < 1000000; i++ { + if best != nil && 1<<(best.k-1) < len(all) { + break + } + h := rand.Uint32() + for k := uint(0); k <= 16; k++ { + if best != nil && k >= best.k { + break + } + var t table + if t.init(h, k, all) { + best = &t + break + } + } + } + if best == nil { + fmt.Fprintf(os.Stderr, "failed to construct string table\n") + os.Exit(1) + } + + // Lay out strings, using overlaps when possible. + layout := append([]string{}, all...) + + // Remove strings that are substrings of other strings + for changed := true; changed; { + changed = false + for i, s := range layout { + if s == "" { + continue + } + for j, t := range layout { + if i != j && t != "" && strings.Contains(s, t) { + changed = true + layout[j] = "" + } + } + } + } + + // Join strings where one suffix matches another prefix. + for { + // Find best i, j, k such that layout[i][len-k:] == layout[j][:k], + // maximizing overlap length k. + besti := -1 + bestj := -1 + bestk := 0 + for i, s := range layout { + if s == "" { + continue + } + for j, t := range layout { + if i == j { + continue + } + for k := bestk + 1; k <= len(s) && k <= len(t); k++ { + if s[len(s)-k:] == t[:k] { + besti = i + bestj = j + bestk = k + } + } + } + } + if bestk > 0 { + layout[besti] += layout[bestj][bestk:] + layout[bestj] = "" + continue + } + break + } + + text := strings.Join(layout, "") + + atom := map[string]uint32{} + for _, s := range all { + off := strings.Index(text, s) + if off < 0 { + panic("lost string " + s) + } + atom[s] = uint32(off<<8 | len(s)) + } + + var buf bytes.Buffer + // Generate the Go code. + fmt.Fprintln(&buf, "// Code generated by go generate gen.go; DO NOT EDIT.\n") + fmt.Fprintln(&buf, "//go:generate go run gen.go\n") + fmt.Fprintln(&buf, "package atom\n\nconst (") + + // compute max len + maxLen := 0 + for _, s := range all { + if maxLen < len(s) { + maxLen = len(s) + } + fmt.Fprintf(&buf, "\t%s Atom = %#x\n", identifier(s), atom[s]) + } + fmt.Fprintln(&buf, ")\n") + + fmt.Fprintf(&buf, "const hash0 = %#x\n\n", best.h0) + fmt.Fprintf(&buf, "const maxAtomLen = %d\n\n", maxLen) + + fmt.Fprintf(&buf, "var table = [1<<%d]Atom{\n", best.k) + for i, s := range best.tab { + if s == "" { + continue + } + fmt.Fprintf(&buf, "\t%#x: %#x, // %s\n", i, atom[s], s) + } + fmt.Fprintf(&buf, "}\n") + datasize := (1 << best.k) * 4 + + fmt.Fprintln(&buf, "const atomText =") + textsize := len(text) + for len(text) > 60 { + fmt.Fprintf(&buf, "\t%q +\n", text[:60]) + text = text[60:] + } + fmt.Fprintf(&buf, "\t%q\n\n", text) + + genFile("table.go", &buf) + + fmt.Fprintf(os.Stdout, "%d atoms; %d string bytes + %d tables = %d total data\n", len(all), textsize, datasize, textsize+datasize) +} + +type byLen []string + +func (x byLen) Less(i, j int) bool { return len(x[i]) > len(x[j]) } +func (x byLen) Swap(i, j int) { x[i], x[j] = x[j], x[i] } +func (x byLen) Len() int { return len(x) } + +// fnv computes the FNV hash with an arbitrary starting value h. +func fnv(h uint32, s string) uint32 { + for i := 0; i < len(s); i++ { + h ^= uint32(s[i]) + h *= 16777619 + } + return h +} + +// A table represents an attempt at constructing the lookup table. +// The lookup table uses cuckoo hashing, meaning that each string +// can be found in one of two positions. +type table struct { + h0 uint32 + k uint + mask uint32 + tab []string +} + +// hash returns the two hashes for s. +func (t *table) hash(s string) (h1, h2 uint32) { + h := fnv(t.h0, s) + h1 = h & t.mask + h2 = (h >> 16) & t.mask + return +} + +// init initializes the table with the given parameters. +// h0 is the initial hash value, +// k is the number of bits of hash value to use, and +// x is the list of strings to store in the table. +// init returns false if the table cannot be constructed. +func (t *table) init(h0 uint32, k uint, x []string) bool { + t.h0 = h0 + t.k = k + t.tab = make([]string, 1< len(t.tab) { + return false + } + s := t.tab[i] + h1, h2 := t.hash(s) + j := h1 + h2 - i + if t.tab[j] != "" && !t.push(j, depth+1) { + return false + } + t.tab[j] = s + return true +} + +// The lists of element names and attribute keys were taken from +// https://html.spec.whatwg.org/multipage/indices.html#index +// as of the "HTML Living Standard - Last Updated 16 April 2018" version. + +// "command", "keygen" and "menuitem" have been removed from the spec, +// but are kept here for backwards compatibility. +var elements = []string{ + "a", + "abbr", + "address", + "area", + "article", + "aside", + "audio", + "b", + "base", + "bdi", + "bdo", + "blockquote", + "body", + "br", + "button", + "canvas", + "caption", + "cite", + "code", + "col", + "colgroup", + "command", + "data", + "datalist", + "dd", + "del", + "details", + "dfn", + "dialog", + "div", + "dl", + "dt", + "em", + "embed", + "fieldset", + "figcaption", + "figure", + "footer", + "form", + "h1", + "h2", + "h3", + "h4", + "h5", + "h6", + "head", + "header", + "hgroup", + "hr", + "html", + "i", + "iframe", + "img", + "input", + "ins", + "kbd", + "keygen", + "label", + "legend", + "li", + "link", + "main", + "map", + "mark", + "menu", + "menuitem", + "meta", + "meter", + "nav", + "noscript", + "object", + "ol", + "optgroup", + "option", + "output", + "p", + "param", + "picture", + "pre", + "progress", + "q", + "rp", + "rt", + "ruby", + "s", + "samp", + "script", + "section", + "select", + "slot", + "small", + "source", + "span", + "strong", + "style", + "sub", + "summary", + "sup", + "table", + "tbody", + "td", + "template", + "textarea", + "tfoot", + "th", + "thead", + "time", + "title", + "tr", + "track", + "u", + "ul", + "var", + "video", + "wbr", +} + +// https://html.spec.whatwg.org/multipage/indices.html#attributes-3 +// +// "challenge", "command", "contextmenu", "dropzone", "icon", "keytype", "mediagroup", +// "radiogroup", "spellcheck", "scoped", "seamless", "sortable" and "sorted" have been removed from the spec, +// but are kept here for backwards compatibility. +var attributes = []string{ + "abbr", + "accept", + "accept-charset", + "accesskey", + "action", + "allowfullscreen", + "allowpaymentrequest", + "allowusermedia", + "alt", + "as", + "async", + "autocomplete", + "autofocus", + "autoplay", + "challenge", + "charset", + "checked", + "cite", + "class", + "color", + "cols", + "colspan", + "command", + "content", + "contenteditable", + "contextmenu", + "controls", + "coords", + "crossorigin", + "data", + "datetime", + "default", + "defer", + "dir", + "dirname", + "disabled", + "download", + "draggable", + "dropzone", + "enctype", + "for", + "form", + "formaction", + "formenctype", + "formmethod", + "formnovalidate", + "formtarget", + "headers", + "height", + "hidden", + "high", + "href", + "hreflang", + "http-equiv", + "icon", + "id", + "inputmode", + "integrity", + "is", + "ismap", + "itemid", + "itemprop", + "itemref", + "itemscope", + "itemtype", + "keytype", + "kind", + "label", + "lang", + "list", + "loop", + "low", + "manifest", + "max", + "maxlength", + "media", + "mediagroup", + "method", + "min", + "minlength", + "multiple", + "muted", + "name", + "nomodule", + "nonce", + "novalidate", + "open", + "optimum", + "pattern", + "ping", + "placeholder", + "playsinline", + "poster", + "preload", + "radiogroup", + "readonly", + "referrerpolicy", + "rel", + "required", + "reversed", + "rows", + "rowspan", + "sandbox", + "spellcheck", + "scope", + "scoped", + "seamless", + "selected", + "shape", + "size", + "sizes", + "sortable", + "sorted", + "slot", + "span", + "spellcheck", + "src", + "srcdoc", + "srclang", + "srcset", + "start", + "step", + "style", + "tabindex", + "target", + "title", + "translate", + "type", + "typemustmatch", + "updateviacache", + "usemap", + "value", + "width", + "workertype", + "wrap", +} + +// "onautocomplete", "onautocompleteerror", "onmousewheel", +// "onshow" and "onsort" have been removed from the spec, +// but are kept here for backwards compatibility. +var eventHandlers = []string{ + "onabort", + "onautocomplete", + "onautocompleteerror", + "onauxclick", + "onafterprint", + "onbeforeprint", + "onbeforeunload", + "onblur", + "oncancel", + "oncanplay", + "oncanplaythrough", + "onchange", + "onclick", + "onclose", + "oncontextmenu", + "oncopy", + "oncuechange", + "oncut", + "ondblclick", + "ondrag", + "ondragend", + "ondragenter", + "ondragexit", + "ondragleave", + "ondragover", + "ondragstart", + "ondrop", + "ondurationchange", + "onemptied", + "onended", + "onerror", + "onfocus", + "onhashchange", + "oninput", + "oninvalid", + "onkeydown", + "onkeypress", + "onkeyup", + "onlanguagechange", + "onload", + "onloadeddata", + "onloadedmetadata", + "onloadend", + "onloadstart", + "onmessage", + "onmessageerror", + "onmousedown", + "onmouseenter", + "onmouseleave", + "onmousemove", + "onmouseout", + "onmouseover", + "onmouseup", + "onmousewheel", + "onwheel", + "onoffline", + "ononline", + "onpagehide", + "onpageshow", + "onpaste", + "onpause", + "onplay", + "onplaying", + "onpopstate", + "onprogress", + "onratechange", + "onreset", + "onresize", + "onrejectionhandled", + "onscroll", + "onsecuritypolicyviolation", + "onseeked", + "onseeking", + "onselect", + "onshow", + "onsort", + "onstalled", + "onstorage", + "onsubmit", + "onsuspend", + "ontimeupdate", + "ontoggle", + "onunhandledrejection", + "onunload", + "onvolumechange", + "onwaiting", +} + +// extra are ad-hoc values not covered by any of the lists above. +var extra = []string{ + "acronym", + "align", + "annotation", + "annotation-xml", + "applet", + "basefont", + "bgsound", + "big", + "blink", + "center", + "color", + "desc", + "face", + "font", + "foreignObject", // HTML is case-insensitive, but SVG-embedded-in-HTML is case-sensitive. + "foreignobject", + "frame", + "frameset", + "image", + "isindex", + "listing", + "malignmark", + "marquee", + "math", + "mglyph", + "mi", + "mn", + "mo", + "ms", + "mtext", + "nobr", + "noembed", + "noframes", + "plaintext", + "prompt", + "public", + "rb", + "rtc", + "spacer", + "strike", + "svg", + "system", + "tt", + "xmp", +} diff --git a/vendor/golang.org/x/net/internal/iana/gen.go b/vendor/golang.org/x/net/internal/iana/gen.go new file mode 100644 index 000000000..2a7661c27 --- /dev/null +++ b/vendor/golang.org/x/net/internal/iana/gen.go @@ -0,0 +1,383 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +//go:generate go run gen.go + +// This program generates internet protocol constants and tables by +// reading IANA protocol registries. +package main + +import ( + "bytes" + "encoding/xml" + "fmt" + "go/format" + "io" + "io/ioutil" + "net/http" + "os" + "strconv" + "strings" +) + +var registries = []struct { + url string + parse func(io.Writer, io.Reader) error +}{ + { + "https://www.iana.org/assignments/dscp-registry/dscp-registry.xml", + parseDSCPRegistry, + }, + { + "https://www.iana.org/assignments/protocol-numbers/protocol-numbers.xml", + parseProtocolNumbers, + }, + { + "https://www.iana.org/assignments/address-family-numbers/address-family-numbers.xml", + parseAddrFamilyNumbers, + }, +} + +func main() { + var bb bytes.Buffer + fmt.Fprintf(&bb, "// go generate gen.go\n") + fmt.Fprintf(&bb, "// Code generated by the command above; DO NOT EDIT.\n\n") + fmt.Fprintf(&bb, "// Package iana provides protocol number resources managed by the Internet Assigned Numbers Authority (IANA).\n") + fmt.Fprintf(&bb, `package iana // import "golang.org/x/net/internal/iana"`+"\n\n") + for _, r := range registries { + resp, err := http.Get(r.url) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + fmt.Fprintf(os.Stderr, "got HTTP status code %v for %v\n", resp.StatusCode, r.url) + os.Exit(1) + } + if err := r.parse(&bb, resp.Body); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + fmt.Fprintf(&bb, "\n") + } + b, err := format.Source(bb.Bytes()) + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + if err := ioutil.WriteFile("const.go", b, 0644); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} + +func parseDSCPRegistry(w io.Writer, r io.Reader) error { + dec := xml.NewDecoder(r) + var dr dscpRegistry + if err := dec.Decode(&dr); err != nil { + return err + } + fmt.Fprintf(w, "// %s, Updated: %s\n", dr.Title, dr.Updated) + fmt.Fprintf(w, "const (\n") + for _, dr := range dr.escapeDSCP() { + fmt.Fprintf(w, "DiffServ%s = %#02x", dr.Name, dr.Value) + fmt.Fprintf(w, "// %s\n", dr.OrigName) + } + for _, er := range dr.escapeECN() { + fmt.Fprintf(w, "%s = %#02x", er.Descr, er.Value) + fmt.Fprintf(w, "// %s\n", er.OrigDescr) + } + fmt.Fprintf(w, ")\n") + return nil +} + +type dscpRegistry struct { + XMLName xml.Name `xml:"registry"` + Title string `xml:"title"` + Updated string `xml:"updated"` + Note string `xml:"note"` + Registries []struct { + Title string `xml:"title"` + Registries []struct { + Title string `xml:"title"` + Records []struct { + Name string `xml:"name"` + Space string `xml:"space"` + } `xml:"record"` + } `xml:"registry"` + Records []struct { + Value string `xml:"value"` + Descr string `xml:"description"` + } `xml:"record"` + } `xml:"registry"` +} + +type canonDSCPRecord struct { + OrigName string + Name string + Value int +} + +func (drr *dscpRegistry) escapeDSCP() []canonDSCPRecord { + var drs []canonDSCPRecord + for _, preg := range drr.Registries { + if !strings.Contains(preg.Title, "Differentiated Services Field Codepoints") { + continue + } + for _, reg := range preg.Registries { + if !strings.Contains(reg.Title, "Pool 1 Codepoints") { + continue + } + drs = make([]canonDSCPRecord, len(reg.Records)) + sr := strings.NewReplacer( + "+", "", + "-", "", + "/", "", + ".", "", + " ", "", + ) + for i, dr := range reg.Records { + s := strings.TrimSpace(dr.Name) + drs[i].OrigName = s + drs[i].Name = sr.Replace(s) + n, err := strconv.ParseUint(dr.Space, 2, 8) + if err != nil { + continue + } + drs[i].Value = int(n) << 2 + } + } + } + return drs +} + +type canonECNRecord struct { + OrigDescr string + Descr string + Value int +} + +func (drr *dscpRegistry) escapeECN() []canonECNRecord { + var ers []canonECNRecord + for _, reg := range drr.Registries { + if !strings.Contains(reg.Title, "ECN Field") { + continue + } + ers = make([]canonECNRecord, len(reg.Records)) + sr := strings.NewReplacer( + "Capable", "", + "Not-ECT", "", + "ECT(1)", "", + "ECT(0)", "", + "CE", "", + "(", "", + ")", "", + "+", "", + "-", "", + "/", "", + ".", "", + " ", "", + ) + for i, er := range reg.Records { + s := strings.TrimSpace(er.Descr) + ers[i].OrigDescr = s + ss := strings.Split(s, " ") + if len(ss) > 1 { + ers[i].Descr = strings.Join(ss[1:], " ") + } else { + ers[i].Descr = ss[0] + } + ers[i].Descr = sr.Replace(er.Descr) + n, err := strconv.ParseUint(er.Value, 2, 8) + if err != nil { + continue + } + ers[i].Value = int(n) + } + } + return ers +} + +func parseProtocolNumbers(w io.Writer, r io.Reader) error { + dec := xml.NewDecoder(r) + var pn protocolNumbers + if err := dec.Decode(&pn); err != nil { + return err + } + prs := pn.escape() + prs = append([]canonProtocolRecord{{ + Name: "IP", + Descr: "IPv4 encapsulation, pseudo protocol number", + Value: 0, + }}, prs...) + fmt.Fprintf(w, "// %s, Updated: %s\n", pn.Title, pn.Updated) + fmt.Fprintf(w, "const (\n") + for _, pr := range prs { + if pr.Name == "" { + continue + } + fmt.Fprintf(w, "Protocol%s = %d", pr.Name, pr.Value) + s := pr.Descr + if s == "" { + s = pr.OrigName + } + fmt.Fprintf(w, "// %s\n", s) + } + fmt.Fprintf(w, ")\n") + return nil +} + +type protocolNumbers struct { + XMLName xml.Name `xml:"registry"` + Title string `xml:"title"` + Updated string `xml:"updated"` + RegTitle string `xml:"registry>title"` + Note string `xml:"registry>note"` + Records []struct { + Value string `xml:"value"` + Name string `xml:"name"` + Descr string `xml:"description"` + } `xml:"registry>record"` +} + +type canonProtocolRecord struct { + OrigName string + Name string + Descr string + Value int +} + +func (pn *protocolNumbers) escape() []canonProtocolRecord { + prs := make([]canonProtocolRecord, len(pn.Records)) + sr := strings.NewReplacer( + "-in-", "in", + "-within-", "within", + "-over-", "over", + "+", "P", + "-", "", + "/", "", + ".", "", + " ", "", + ) + for i, pr := range pn.Records { + if strings.Contains(pr.Name, "Deprecated") || + strings.Contains(pr.Name, "deprecated") { + continue + } + prs[i].OrigName = pr.Name + s := strings.TrimSpace(pr.Name) + switch pr.Name { + case "ISIS over IPv4": + prs[i].Name = "ISIS" + case "manet": + prs[i].Name = "MANET" + default: + prs[i].Name = sr.Replace(s) + } + ss := strings.Split(pr.Descr, "\n") + for i := range ss { + ss[i] = strings.TrimSpace(ss[i]) + } + if len(ss) > 1 { + prs[i].Descr = strings.Join(ss, " ") + } else { + prs[i].Descr = ss[0] + } + prs[i].Value, _ = strconv.Atoi(pr.Value) + } + return prs +} + +func parseAddrFamilyNumbers(w io.Writer, r io.Reader) error { + dec := xml.NewDecoder(r) + var afn addrFamilylNumbers + if err := dec.Decode(&afn); err != nil { + return err + } + afrs := afn.escape() + fmt.Fprintf(w, "// %s, Updated: %s\n", afn.Title, afn.Updated) + fmt.Fprintf(w, "const (\n") + for _, afr := range afrs { + if afr.Name == "" { + continue + } + fmt.Fprintf(w, "AddrFamily%s = %d", afr.Name, afr.Value) + fmt.Fprintf(w, "// %s\n", afr.Descr) + } + fmt.Fprintf(w, ")\n") + return nil +} + +type addrFamilylNumbers struct { + XMLName xml.Name `xml:"registry"` + Title string `xml:"title"` + Updated string `xml:"updated"` + RegTitle string `xml:"registry>title"` + Note string `xml:"registry>note"` + Records []struct { + Value string `xml:"value"` + Descr string `xml:"description"` + } `xml:"registry>record"` +} + +type canonAddrFamilyRecord struct { + Name string + Descr string + Value int +} + +func (afn *addrFamilylNumbers) escape() []canonAddrFamilyRecord { + afrs := make([]canonAddrFamilyRecord, len(afn.Records)) + sr := strings.NewReplacer( + "IP version 4", "IPv4", + "IP version 6", "IPv6", + "Identifier", "ID", + "-", "", + "-", "", + "/", "", + ".", "", + " ", "", + ) + for i, afr := range afn.Records { + if strings.Contains(afr.Descr, "Unassigned") || + strings.Contains(afr.Descr, "Reserved") { + continue + } + afrs[i].Descr = afr.Descr + s := strings.TrimSpace(afr.Descr) + switch s { + case "IP (IP version 4)": + afrs[i].Name = "IPv4" + case "IP6 (IP version 6)": + afrs[i].Name = "IPv6" + case "AFI for L2VPN information": + afrs[i].Name = "L2VPN" + case "E.164 with NSAP format subaddress": + afrs[i].Name = "E164withSubaddress" + case "MT IP: Multi-Topology IP version 4": + afrs[i].Name = "MTIPv4" + case "MAC/24": + afrs[i].Name = "MACFinal24bits" + case "MAC/40": + afrs[i].Name = "MACFinal40bits" + case "IPv6/64": + afrs[i].Name = "IPv6Initial64bits" + default: + n := strings.Index(s, "(") + if n > 0 { + s = s[:n] + } + n = strings.Index(s, ":") + if n > 0 { + s = s[:n] + } + afrs[i].Name = sr.Replace(s) + } + afrs[i].Value, _ = strconv.Atoi(afr.Value) + } + return afrs +} diff --git a/vendor/golang.org/x/net/internal/socket/defs_aix.go b/vendor/golang.org/x/net/internal/socket/defs_aix.go new file mode 100644 index 000000000..c9d05b261 --- /dev/null +++ b/vendor/golang.org/x/net/internal/socket/defs_aix.go @@ -0,0 +1,39 @@ +// Copyright 2019 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +// +godefs map struct_in_addr [4]byte /* in_addr */ +// +godefs map struct_in6_addr [16]byte /* in6_addr */ + +package socket + +/* +#include + +#include +*/ +import "C" + +type iovec C.struct_iovec + +type msghdr C.struct_msghdr + +type mmsghdr C.struct_mmsghdr + +type cmsghdr C.struct_cmsghdr + +type sockaddrInet C.struct_sockaddr_in + +type sockaddrInet6 C.struct_sockaddr_in6 + +const ( + sizeofIovec = C.sizeof_struct_iovec + sizeofMsghdr = C.sizeof_struct_msghdr + sizeofMmsghdr = C.sizeof_struct_mmsghdr + sizeofCmsghdr = C.sizeof_struct_cmsghdr + + sizeofSockaddrInet = C.sizeof_struct_sockaddr_in + sizeofSockaddrInet6 = C.sizeof_struct_sockaddr_in6 +) diff --git a/vendor/golang.org/x/net/internal/socket/defs_darwin.go b/vendor/golang.org/x/net/internal/socket/defs_darwin.go new file mode 100644 index 000000000..b780bc67a --- /dev/null +++ b/vendor/golang.org/x/net/internal/socket/defs_darwin.go @@ -0,0 +1,36 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +// +godefs map struct_in_addr [4]byte /* in_addr */ +// +godefs map struct_in6_addr [16]byte /* in6_addr */ + +package socket + +/* +#include + +#include +*/ +import "C" + +type iovec C.struct_iovec + +type msghdr C.struct_msghdr + +type cmsghdr C.struct_cmsghdr + +type sockaddrInet C.struct_sockaddr_in + +type sockaddrInet6 C.struct_sockaddr_in6 + +const ( + sizeofIovec = C.sizeof_struct_iovec + sizeofMsghdr = C.sizeof_struct_msghdr + sizeofCmsghdr = C.sizeof_struct_cmsghdr + + sizeofSockaddrInet = C.sizeof_struct_sockaddr_in + sizeofSockaddrInet6 = C.sizeof_struct_sockaddr_in6 +) diff --git a/vendor/golang.org/x/net/internal/socket/defs_dragonfly.go b/vendor/golang.org/x/net/internal/socket/defs_dragonfly.go new file mode 100644 index 000000000..b780bc67a --- /dev/null +++ b/vendor/golang.org/x/net/internal/socket/defs_dragonfly.go @@ -0,0 +1,36 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +// +godefs map struct_in_addr [4]byte /* in_addr */ +// +godefs map struct_in6_addr [16]byte /* in6_addr */ + +package socket + +/* +#include + +#include +*/ +import "C" + +type iovec C.struct_iovec + +type msghdr C.struct_msghdr + +type cmsghdr C.struct_cmsghdr + +type sockaddrInet C.struct_sockaddr_in + +type sockaddrInet6 C.struct_sockaddr_in6 + +const ( + sizeofIovec = C.sizeof_struct_iovec + sizeofMsghdr = C.sizeof_struct_msghdr + sizeofCmsghdr = C.sizeof_struct_cmsghdr + + sizeofSockaddrInet = C.sizeof_struct_sockaddr_in + sizeofSockaddrInet6 = C.sizeof_struct_sockaddr_in6 +) diff --git a/vendor/golang.org/x/net/internal/socket/defs_freebsd.go b/vendor/golang.org/x/net/internal/socket/defs_freebsd.go new file mode 100644 index 000000000..b780bc67a --- /dev/null +++ b/vendor/golang.org/x/net/internal/socket/defs_freebsd.go @@ -0,0 +1,36 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +// +godefs map struct_in_addr [4]byte /* in_addr */ +// +godefs map struct_in6_addr [16]byte /* in6_addr */ + +package socket + +/* +#include + +#include +*/ +import "C" + +type iovec C.struct_iovec + +type msghdr C.struct_msghdr + +type cmsghdr C.struct_cmsghdr + +type sockaddrInet C.struct_sockaddr_in + +type sockaddrInet6 C.struct_sockaddr_in6 + +const ( + sizeofIovec = C.sizeof_struct_iovec + sizeofMsghdr = C.sizeof_struct_msghdr + sizeofCmsghdr = C.sizeof_struct_cmsghdr + + sizeofSockaddrInet = C.sizeof_struct_sockaddr_in + sizeofSockaddrInet6 = C.sizeof_struct_sockaddr_in6 +) diff --git a/vendor/golang.org/x/net/internal/socket/defs_linux.go b/vendor/golang.org/x/net/internal/socket/defs_linux.go new file mode 100644 index 000000000..6c5c11dcc --- /dev/null +++ b/vendor/golang.org/x/net/internal/socket/defs_linux.go @@ -0,0 +1,41 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +// +godefs map struct_in_addr [4]byte /* in_addr */ +// +godefs map struct_in6_addr [16]byte /* in6_addr */ + +package socket + +/* +#include +#include + +#define _GNU_SOURCE +#include +*/ +import "C" + +type iovec C.struct_iovec + +type msghdr C.struct_msghdr + +type mmsghdr C.struct_mmsghdr + +type cmsghdr C.struct_cmsghdr + +type sockaddrInet C.struct_sockaddr_in + +type sockaddrInet6 C.struct_sockaddr_in6 + +const ( + sizeofIovec = C.sizeof_struct_iovec + sizeofMsghdr = C.sizeof_struct_msghdr + sizeofMmsghdr = C.sizeof_struct_mmsghdr + sizeofCmsghdr = C.sizeof_struct_cmsghdr + + sizeofSockaddrInet = C.sizeof_struct_sockaddr_in + sizeofSockaddrInet6 = C.sizeof_struct_sockaddr_in6 +) diff --git a/vendor/golang.org/x/net/internal/socket/defs_netbsd.go b/vendor/golang.org/x/net/internal/socket/defs_netbsd.go new file mode 100644 index 000000000..3d3b77639 --- /dev/null +++ b/vendor/golang.org/x/net/internal/socket/defs_netbsd.go @@ -0,0 +1,39 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +// +godefs map struct_in_addr [4]byte /* in_addr */ +// +godefs map struct_in6_addr [16]byte /* in6_addr */ + +package socket + +/* +#include + +#include +*/ +import "C" + +type iovec C.struct_iovec + +type msghdr C.struct_msghdr + +type mmsghdr C.struct_mmsghdr + +type cmsghdr C.struct_cmsghdr + +type sockaddrInet C.struct_sockaddr_in + +type sockaddrInet6 C.struct_sockaddr_in6 + +const ( + sizeofIovec = C.sizeof_struct_iovec + sizeofMsghdr = C.sizeof_struct_msghdr + sizeofMmsghdr = C.sizeof_struct_mmsghdr + sizeofCmsghdr = C.sizeof_struct_cmsghdr + + sizeofSockaddrInet = C.sizeof_struct_sockaddr_in + sizeofSockaddrInet6 = C.sizeof_struct_sockaddr_in6 +) diff --git a/vendor/golang.org/x/net/internal/socket/defs_openbsd.go b/vendor/golang.org/x/net/internal/socket/defs_openbsd.go new file mode 100644 index 000000000..b780bc67a --- /dev/null +++ b/vendor/golang.org/x/net/internal/socket/defs_openbsd.go @@ -0,0 +1,36 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +// +godefs map struct_in_addr [4]byte /* in_addr */ +// +godefs map struct_in6_addr [16]byte /* in6_addr */ + +package socket + +/* +#include + +#include +*/ +import "C" + +type iovec C.struct_iovec + +type msghdr C.struct_msghdr + +type cmsghdr C.struct_cmsghdr + +type sockaddrInet C.struct_sockaddr_in + +type sockaddrInet6 C.struct_sockaddr_in6 + +const ( + sizeofIovec = C.sizeof_struct_iovec + sizeofMsghdr = C.sizeof_struct_msghdr + sizeofCmsghdr = C.sizeof_struct_cmsghdr + + sizeofSockaddrInet = C.sizeof_struct_sockaddr_in + sizeofSockaddrInet6 = C.sizeof_struct_sockaddr_in6 +) diff --git a/vendor/golang.org/x/net/internal/socket/defs_solaris.go b/vendor/golang.org/x/net/internal/socket/defs_solaris.go new file mode 100644 index 000000000..b780bc67a --- /dev/null +++ b/vendor/golang.org/x/net/internal/socket/defs_solaris.go @@ -0,0 +1,36 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +// +godefs map struct_in_addr [4]byte /* in_addr */ +// +godefs map struct_in6_addr [16]byte /* in6_addr */ + +package socket + +/* +#include + +#include +*/ +import "C" + +type iovec C.struct_iovec + +type msghdr C.struct_msghdr + +type cmsghdr C.struct_cmsghdr + +type sockaddrInet C.struct_sockaddr_in + +type sockaddrInet6 C.struct_sockaddr_in6 + +const ( + sizeofIovec = C.sizeof_struct_iovec + sizeofMsghdr = C.sizeof_struct_msghdr + sizeofCmsghdr = C.sizeof_struct_cmsghdr + + sizeofSockaddrInet = C.sizeof_struct_sockaddr_in + sizeofSockaddrInet6 = C.sizeof_struct_sockaddr_in6 +) diff --git a/vendor/golang.org/x/net/ipv4/defs_aix.go b/vendor/golang.org/x/net/ipv4/defs_aix.go new file mode 100644 index 000000000..0f37211c6 --- /dev/null +++ b/vendor/golang.org/x/net/ipv4/defs_aix.go @@ -0,0 +1,39 @@ +// Copyright 2019 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +// +godefs map struct_in_addr [4]byte /* in_addr */ + +package ipv4 + +/* +#include +*/ +import "C" + +const ( + sysIP_OPTIONS = C.IP_OPTIONS + sysIP_HDRINCL = C.IP_HDRINCL + sysIP_TOS = C.IP_TOS + sysIP_TTL = C.IP_TTL + sysIP_RECVOPTS = C.IP_RECVOPTS + sysIP_RECVRETOPTS = C.IP_RECVRETOPTS + sysIP_RECVDSTADDR = C.IP_RECVDSTADDR + sysIP_RETOPTS = C.IP_RETOPTS + // IP_RECVIF is defined on AIX but doesn't work. + // IP_RECVINTERFACE must be used instead. + sysIP_RECVIF = C.IP_RECVINTERFACE + sysIP_RECVTTL = C.IP_RECVTTL + + sysIP_MULTICAST_IF = C.IP_MULTICAST_IF + sysIP_MULTICAST_TTL = C.IP_MULTICAST_TTL + sysIP_MULTICAST_LOOP = C.IP_MULTICAST_LOOP + sysIP_ADD_MEMBERSHIP = C.IP_ADD_MEMBERSHIP + sysIP_DROP_MEMBERSHIP = C.IP_DROP_MEMBERSHIP + + sizeofIPMreq = C.sizeof_struct_ip_mreq +) + +type ipMreq C.struct_ip_mreq diff --git a/vendor/golang.org/x/net/ipv4/defs_darwin.go b/vendor/golang.org/x/net/ipv4/defs_darwin.go new file mode 100644 index 000000000..c8f2e05b8 --- /dev/null +++ b/vendor/golang.org/x/net/ipv4/defs_darwin.go @@ -0,0 +1,77 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +// +godefs map struct_in_addr [4]byte /* in_addr */ + +package ipv4 + +/* +#include + +#include +*/ +import "C" + +const ( + sysIP_OPTIONS = C.IP_OPTIONS + sysIP_HDRINCL = C.IP_HDRINCL + sysIP_TOS = C.IP_TOS + sysIP_TTL = C.IP_TTL + sysIP_RECVOPTS = C.IP_RECVOPTS + sysIP_RECVRETOPTS = C.IP_RECVRETOPTS + sysIP_RECVDSTADDR = C.IP_RECVDSTADDR + sysIP_RETOPTS = C.IP_RETOPTS + sysIP_RECVIF = C.IP_RECVIF + sysIP_STRIPHDR = C.IP_STRIPHDR + sysIP_RECVTTL = C.IP_RECVTTL + sysIP_BOUND_IF = C.IP_BOUND_IF + sysIP_PKTINFO = C.IP_PKTINFO + sysIP_RECVPKTINFO = C.IP_RECVPKTINFO + + sysIP_MULTICAST_IF = C.IP_MULTICAST_IF + sysIP_MULTICAST_TTL = C.IP_MULTICAST_TTL + sysIP_MULTICAST_LOOP = C.IP_MULTICAST_LOOP + sysIP_ADD_MEMBERSHIP = C.IP_ADD_MEMBERSHIP + sysIP_DROP_MEMBERSHIP = C.IP_DROP_MEMBERSHIP + sysIP_MULTICAST_VIF = C.IP_MULTICAST_VIF + sysIP_MULTICAST_IFINDEX = C.IP_MULTICAST_IFINDEX + sysIP_ADD_SOURCE_MEMBERSHIP = C.IP_ADD_SOURCE_MEMBERSHIP + sysIP_DROP_SOURCE_MEMBERSHIP = C.IP_DROP_SOURCE_MEMBERSHIP + sysIP_BLOCK_SOURCE = C.IP_BLOCK_SOURCE + sysIP_UNBLOCK_SOURCE = C.IP_UNBLOCK_SOURCE + sysMCAST_JOIN_GROUP = C.MCAST_JOIN_GROUP + sysMCAST_LEAVE_GROUP = C.MCAST_LEAVE_GROUP + sysMCAST_JOIN_SOURCE_GROUP = C.MCAST_JOIN_SOURCE_GROUP + sysMCAST_LEAVE_SOURCE_GROUP = C.MCAST_LEAVE_SOURCE_GROUP + sysMCAST_BLOCK_SOURCE = C.MCAST_BLOCK_SOURCE + sysMCAST_UNBLOCK_SOURCE = C.MCAST_UNBLOCK_SOURCE + + sizeofSockaddrStorage = C.sizeof_struct_sockaddr_storage + sizeofSockaddrInet = C.sizeof_struct_sockaddr_in + sizeofInetPktinfo = C.sizeof_struct_in_pktinfo + + sizeofIPMreq = C.sizeof_struct_ip_mreq + sizeofIPMreqn = C.sizeof_struct_ip_mreqn + sizeofIPMreqSource = C.sizeof_struct_ip_mreq_source + sizeofGroupReq = C.sizeof_struct_group_req + sizeofGroupSourceReq = C.sizeof_struct_group_source_req +) + +type sockaddrStorage C.struct_sockaddr_storage + +type sockaddrInet C.struct_sockaddr_in + +type inetPktinfo C.struct_in_pktinfo + +type ipMreq C.struct_ip_mreq + +type ipMreqn C.struct_ip_mreqn + +type ipMreqSource C.struct_ip_mreq_source + +type groupReq C.struct_group_req + +type groupSourceReq C.struct_group_source_req diff --git a/vendor/golang.org/x/net/ipv4/defs_dragonfly.go b/vendor/golang.org/x/net/ipv4/defs_dragonfly.go new file mode 100644 index 000000000..f30544ea2 --- /dev/null +++ b/vendor/golang.org/x/net/ipv4/defs_dragonfly.go @@ -0,0 +1,38 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +// +godefs map struct_in_addr [4]byte /* in_addr */ + +package ipv4 + +/* +#include +*/ +import "C" + +const ( + sysIP_OPTIONS = C.IP_OPTIONS + sysIP_HDRINCL = C.IP_HDRINCL + sysIP_TOS = C.IP_TOS + sysIP_TTL = C.IP_TTL + sysIP_RECVOPTS = C.IP_RECVOPTS + sysIP_RECVRETOPTS = C.IP_RECVRETOPTS + sysIP_RECVDSTADDR = C.IP_RECVDSTADDR + sysIP_RETOPTS = C.IP_RETOPTS + sysIP_RECVIF = C.IP_RECVIF + sysIP_RECVTTL = C.IP_RECVTTL + + sysIP_MULTICAST_IF = C.IP_MULTICAST_IF + sysIP_MULTICAST_TTL = C.IP_MULTICAST_TTL + sysIP_MULTICAST_LOOP = C.IP_MULTICAST_LOOP + sysIP_MULTICAST_VIF = C.IP_MULTICAST_VIF + sysIP_ADD_MEMBERSHIP = C.IP_ADD_MEMBERSHIP + sysIP_DROP_MEMBERSHIP = C.IP_DROP_MEMBERSHIP + + sizeofIPMreq = C.sizeof_struct_ip_mreq +) + +type ipMreq C.struct_ip_mreq diff --git a/vendor/golang.org/x/net/ipv4/defs_freebsd.go b/vendor/golang.org/x/net/ipv4/defs_freebsd.go new file mode 100644 index 000000000..4dd57d865 --- /dev/null +++ b/vendor/golang.org/x/net/ipv4/defs_freebsd.go @@ -0,0 +1,75 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +// +godefs map struct_in_addr [4]byte /* in_addr */ + +package ipv4 + +/* +#include + +#include +*/ +import "C" + +const ( + sysIP_OPTIONS = C.IP_OPTIONS + sysIP_HDRINCL = C.IP_HDRINCL + sysIP_TOS = C.IP_TOS + sysIP_TTL = C.IP_TTL + sysIP_RECVOPTS = C.IP_RECVOPTS + sysIP_RECVRETOPTS = C.IP_RECVRETOPTS + sysIP_RECVDSTADDR = C.IP_RECVDSTADDR + sysIP_SENDSRCADDR = C.IP_SENDSRCADDR + sysIP_RETOPTS = C.IP_RETOPTS + sysIP_RECVIF = C.IP_RECVIF + sysIP_ONESBCAST = C.IP_ONESBCAST + sysIP_BINDANY = C.IP_BINDANY + sysIP_RECVTTL = C.IP_RECVTTL + sysIP_MINTTL = C.IP_MINTTL + sysIP_DONTFRAG = C.IP_DONTFRAG + sysIP_RECVTOS = C.IP_RECVTOS + + sysIP_MULTICAST_IF = C.IP_MULTICAST_IF + sysIP_MULTICAST_TTL = C.IP_MULTICAST_TTL + sysIP_MULTICAST_LOOP = C.IP_MULTICAST_LOOP + sysIP_ADD_MEMBERSHIP = C.IP_ADD_MEMBERSHIP + sysIP_DROP_MEMBERSHIP = C.IP_DROP_MEMBERSHIP + sysIP_MULTICAST_VIF = C.IP_MULTICAST_VIF + sysIP_ADD_SOURCE_MEMBERSHIP = C.IP_ADD_SOURCE_MEMBERSHIP + sysIP_DROP_SOURCE_MEMBERSHIP = C.IP_DROP_SOURCE_MEMBERSHIP + sysIP_BLOCK_SOURCE = C.IP_BLOCK_SOURCE + sysIP_UNBLOCK_SOURCE = C.IP_UNBLOCK_SOURCE + sysMCAST_JOIN_GROUP = C.MCAST_JOIN_GROUP + sysMCAST_LEAVE_GROUP = C.MCAST_LEAVE_GROUP + sysMCAST_JOIN_SOURCE_GROUP = C.MCAST_JOIN_SOURCE_GROUP + sysMCAST_LEAVE_SOURCE_GROUP = C.MCAST_LEAVE_SOURCE_GROUP + sysMCAST_BLOCK_SOURCE = C.MCAST_BLOCK_SOURCE + sysMCAST_UNBLOCK_SOURCE = C.MCAST_UNBLOCK_SOURCE + + sizeofSockaddrStorage = C.sizeof_struct_sockaddr_storage + sizeofSockaddrInet = C.sizeof_struct_sockaddr_in + + sizeofIPMreq = C.sizeof_struct_ip_mreq + sizeofIPMreqn = C.sizeof_struct_ip_mreqn + sizeofIPMreqSource = C.sizeof_struct_ip_mreq_source + sizeofGroupReq = C.sizeof_struct_group_req + sizeofGroupSourceReq = C.sizeof_struct_group_source_req +) + +type sockaddrStorage C.struct_sockaddr_storage + +type sockaddrInet C.struct_sockaddr_in + +type ipMreq C.struct_ip_mreq + +type ipMreqn C.struct_ip_mreqn + +type ipMreqSource C.struct_ip_mreq_source + +type groupReq C.struct_group_req + +type groupSourceReq C.struct_group_source_req diff --git a/vendor/golang.org/x/net/ipv4/defs_linux.go b/vendor/golang.org/x/net/ipv4/defs_linux.go new file mode 100644 index 000000000..beb11071a --- /dev/null +++ b/vendor/golang.org/x/net/ipv4/defs_linux.go @@ -0,0 +1,122 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +// +godefs map struct_in_addr [4]byte /* in_addr */ + +package ipv4 + +/* +#include + +#include +#include +#include +#include +#include +*/ +import "C" + +const ( + sysIP_TOS = C.IP_TOS + sysIP_TTL = C.IP_TTL + sysIP_HDRINCL = C.IP_HDRINCL + sysIP_OPTIONS = C.IP_OPTIONS + sysIP_ROUTER_ALERT = C.IP_ROUTER_ALERT + sysIP_RECVOPTS = C.IP_RECVOPTS + sysIP_RETOPTS = C.IP_RETOPTS + sysIP_PKTINFO = C.IP_PKTINFO + sysIP_PKTOPTIONS = C.IP_PKTOPTIONS + sysIP_MTU_DISCOVER = C.IP_MTU_DISCOVER + sysIP_RECVERR = C.IP_RECVERR + sysIP_RECVTTL = C.IP_RECVTTL + sysIP_RECVTOS = C.IP_RECVTOS + sysIP_MTU = C.IP_MTU + sysIP_FREEBIND = C.IP_FREEBIND + sysIP_TRANSPARENT = C.IP_TRANSPARENT + sysIP_RECVRETOPTS = C.IP_RECVRETOPTS + sysIP_ORIGDSTADDR = C.IP_ORIGDSTADDR + sysIP_RECVORIGDSTADDR = C.IP_RECVORIGDSTADDR + sysIP_MINTTL = C.IP_MINTTL + sysIP_NODEFRAG = C.IP_NODEFRAG + sysIP_UNICAST_IF = C.IP_UNICAST_IF + + sysIP_MULTICAST_IF = C.IP_MULTICAST_IF + sysIP_MULTICAST_TTL = C.IP_MULTICAST_TTL + sysIP_MULTICAST_LOOP = C.IP_MULTICAST_LOOP + sysIP_ADD_MEMBERSHIP = C.IP_ADD_MEMBERSHIP + sysIP_DROP_MEMBERSHIP = C.IP_DROP_MEMBERSHIP + sysIP_UNBLOCK_SOURCE = C.IP_UNBLOCK_SOURCE + sysIP_BLOCK_SOURCE = C.IP_BLOCK_SOURCE + sysIP_ADD_SOURCE_MEMBERSHIP = C.IP_ADD_SOURCE_MEMBERSHIP + sysIP_DROP_SOURCE_MEMBERSHIP = C.IP_DROP_SOURCE_MEMBERSHIP + sysIP_MSFILTER = C.IP_MSFILTER + sysMCAST_JOIN_GROUP = C.MCAST_JOIN_GROUP + sysMCAST_LEAVE_GROUP = C.MCAST_LEAVE_GROUP + sysMCAST_JOIN_SOURCE_GROUP = C.MCAST_JOIN_SOURCE_GROUP + sysMCAST_LEAVE_SOURCE_GROUP = C.MCAST_LEAVE_SOURCE_GROUP + sysMCAST_BLOCK_SOURCE = C.MCAST_BLOCK_SOURCE + sysMCAST_UNBLOCK_SOURCE = C.MCAST_UNBLOCK_SOURCE + sysMCAST_MSFILTER = C.MCAST_MSFILTER + sysIP_MULTICAST_ALL = C.IP_MULTICAST_ALL + + //sysIP_PMTUDISC_DONT = C.IP_PMTUDISC_DONT + //sysIP_PMTUDISC_WANT = C.IP_PMTUDISC_WANT + //sysIP_PMTUDISC_DO = C.IP_PMTUDISC_DO + //sysIP_PMTUDISC_PROBE = C.IP_PMTUDISC_PROBE + //sysIP_PMTUDISC_INTERFACE = C.IP_PMTUDISC_INTERFACE + //sysIP_PMTUDISC_OMIT = C.IP_PMTUDISC_OMIT + + sysICMP_FILTER = C.ICMP_FILTER + + sysSO_EE_ORIGIN_NONE = C.SO_EE_ORIGIN_NONE + sysSO_EE_ORIGIN_LOCAL = C.SO_EE_ORIGIN_LOCAL + sysSO_EE_ORIGIN_ICMP = C.SO_EE_ORIGIN_ICMP + sysSO_EE_ORIGIN_ICMP6 = C.SO_EE_ORIGIN_ICMP6 + sysSO_EE_ORIGIN_TXSTATUS = C.SO_EE_ORIGIN_TXSTATUS + sysSO_EE_ORIGIN_TIMESTAMPING = C.SO_EE_ORIGIN_TIMESTAMPING + + sysSOL_SOCKET = C.SOL_SOCKET + sysSO_ATTACH_FILTER = C.SO_ATTACH_FILTER + + sizeofKernelSockaddrStorage = C.sizeof_struct___kernel_sockaddr_storage + sizeofSockaddrInet = C.sizeof_struct_sockaddr_in + sizeofInetPktinfo = C.sizeof_struct_in_pktinfo + sizeofSockExtendedErr = C.sizeof_struct_sock_extended_err + + sizeofIPMreq = C.sizeof_struct_ip_mreq + sizeofIPMreqn = C.sizeof_struct_ip_mreqn + sizeofIPMreqSource = C.sizeof_struct_ip_mreq_source + sizeofGroupReq = C.sizeof_struct_group_req + sizeofGroupSourceReq = C.sizeof_struct_group_source_req + + sizeofICMPFilter = C.sizeof_struct_icmp_filter + + sizeofSockFprog = C.sizeof_struct_sock_fprog +) + +type kernelSockaddrStorage C.struct___kernel_sockaddr_storage + +type sockaddrInet C.struct_sockaddr_in + +type inetPktinfo C.struct_in_pktinfo + +type sockExtendedErr C.struct_sock_extended_err + +type ipMreq C.struct_ip_mreq + +type ipMreqn C.struct_ip_mreqn + +type ipMreqSource C.struct_ip_mreq_source + +type groupReq C.struct_group_req + +type groupSourceReq C.struct_group_source_req + +type icmpFilter C.struct_icmp_filter + +type sockFProg C.struct_sock_fprog + +type sockFilter C.struct_sock_filter diff --git a/vendor/golang.org/x/net/ipv4/defs_netbsd.go b/vendor/golang.org/x/net/ipv4/defs_netbsd.go new file mode 100644 index 000000000..8f8af1b89 --- /dev/null +++ b/vendor/golang.org/x/net/ipv4/defs_netbsd.go @@ -0,0 +1,37 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +// +godefs map struct_in_addr [4]byte /* in_addr */ + +package ipv4 + +/* +#include +*/ +import "C" + +const ( + sysIP_OPTIONS = C.IP_OPTIONS + sysIP_HDRINCL = C.IP_HDRINCL + sysIP_TOS = C.IP_TOS + sysIP_TTL = C.IP_TTL + sysIP_RECVOPTS = C.IP_RECVOPTS + sysIP_RECVRETOPTS = C.IP_RECVRETOPTS + sysIP_RECVDSTADDR = C.IP_RECVDSTADDR + sysIP_RETOPTS = C.IP_RETOPTS + sysIP_RECVIF = C.IP_RECVIF + sysIP_RECVTTL = C.IP_RECVTTL + + sysIP_MULTICAST_IF = C.IP_MULTICAST_IF + sysIP_MULTICAST_TTL = C.IP_MULTICAST_TTL + sysIP_MULTICAST_LOOP = C.IP_MULTICAST_LOOP + sysIP_ADD_MEMBERSHIP = C.IP_ADD_MEMBERSHIP + sysIP_DROP_MEMBERSHIP = C.IP_DROP_MEMBERSHIP + + sizeofIPMreq = C.sizeof_struct_ip_mreq +) + +type ipMreq C.struct_ip_mreq diff --git a/vendor/golang.org/x/net/ipv4/defs_openbsd.go b/vendor/golang.org/x/net/ipv4/defs_openbsd.go new file mode 100644 index 000000000..8f8af1b89 --- /dev/null +++ b/vendor/golang.org/x/net/ipv4/defs_openbsd.go @@ -0,0 +1,37 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +// +godefs map struct_in_addr [4]byte /* in_addr */ + +package ipv4 + +/* +#include +*/ +import "C" + +const ( + sysIP_OPTIONS = C.IP_OPTIONS + sysIP_HDRINCL = C.IP_HDRINCL + sysIP_TOS = C.IP_TOS + sysIP_TTL = C.IP_TTL + sysIP_RECVOPTS = C.IP_RECVOPTS + sysIP_RECVRETOPTS = C.IP_RECVRETOPTS + sysIP_RECVDSTADDR = C.IP_RECVDSTADDR + sysIP_RETOPTS = C.IP_RETOPTS + sysIP_RECVIF = C.IP_RECVIF + sysIP_RECVTTL = C.IP_RECVTTL + + sysIP_MULTICAST_IF = C.IP_MULTICAST_IF + sysIP_MULTICAST_TTL = C.IP_MULTICAST_TTL + sysIP_MULTICAST_LOOP = C.IP_MULTICAST_LOOP + sysIP_ADD_MEMBERSHIP = C.IP_ADD_MEMBERSHIP + sysIP_DROP_MEMBERSHIP = C.IP_DROP_MEMBERSHIP + + sizeofIPMreq = C.sizeof_struct_ip_mreq +) + +type ipMreq C.struct_ip_mreq diff --git a/vendor/golang.org/x/net/ipv4/defs_solaris.go b/vendor/golang.org/x/net/ipv4/defs_solaris.go new file mode 100644 index 000000000..aeb33e9c8 --- /dev/null +++ b/vendor/golang.org/x/net/ipv4/defs_solaris.go @@ -0,0 +1,84 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +// +godefs map struct_in_addr [4]byte /* in_addr */ + +package ipv4 + +/* +#include + +#include +*/ +import "C" + +const ( + sysIP_OPTIONS = C.IP_OPTIONS + sysIP_HDRINCL = C.IP_HDRINCL + sysIP_TOS = C.IP_TOS + sysIP_TTL = C.IP_TTL + sysIP_RECVOPTS = C.IP_RECVOPTS + sysIP_RECVRETOPTS = C.IP_RECVRETOPTS + sysIP_RECVDSTADDR = C.IP_RECVDSTADDR + sysIP_RETOPTS = C.IP_RETOPTS + sysIP_RECVIF = C.IP_RECVIF + sysIP_RECVSLLA = C.IP_RECVSLLA + sysIP_RECVTTL = C.IP_RECVTTL + + sysIP_MULTICAST_IF = C.IP_MULTICAST_IF + sysIP_MULTICAST_TTL = C.IP_MULTICAST_TTL + sysIP_MULTICAST_LOOP = C.IP_MULTICAST_LOOP + sysIP_ADD_MEMBERSHIP = C.IP_ADD_MEMBERSHIP + sysIP_DROP_MEMBERSHIP = C.IP_DROP_MEMBERSHIP + sysIP_BLOCK_SOURCE = C.IP_BLOCK_SOURCE + sysIP_UNBLOCK_SOURCE = C.IP_UNBLOCK_SOURCE + sysIP_ADD_SOURCE_MEMBERSHIP = C.IP_ADD_SOURCE_MEMBERSHIP + sysIP_DROP_SOURCE_MEMBERSHIP = C.IP_DROP_SOURCE_MEMBERSHIP + sysIP_NEXTHOP = C.IP_NEXTHOP + + sysIP_PKTINFO = C.IP_PKTINFO + sysIP_RECVPKTINFO = C.IP_RECVPKTINFO + sysIP_DONTFRAG = C.IP_DONTFRAG + + sysIP_BOUND_IF = C.IP_BOUND_IF + sysIP_UNSPEC_SRC = C.IP_UNSPEC_SRC + sysIP_BROADCAST_TTL = C.IP_BROADCAST_TTL + sysIP_DHCPINIT_IF = C.IP_DHCPINIT_IF + + sysIP_REUSEADDR = C.IP_REUSEADDR + sysIP_DONTROUTE = C.IP_DONTROUTE + sysIP_BROADCAST = C.IP_BROADCAST + + sysMCAST_JOIN_GROUP = C.MCAST_JOIN_GROUP + sysMCAST_LEAVE_GROUP = C.MCAST_LEAVE_GROUP + sysMCAST_BLOCK_SOURCE = C.MCAST_BLOCK_SOURCE + sysMCAST_UNBLOCK_SOURCE = C.MCAST_UNBLOCK_SOURCE + sysMCAST_JOIN_SOURCE_GROUP = C.MCAST_JOIN_SOURCE_GROUP + sysMCAST_LEAVE_SOURCE_GROUP = C.MCAST_LEAVE_SOURCE_GROUP + + sizeofSockaddrStorage = C.sizeof_struct_sockaddr_storage + sizeofSockaddrInet = C.sizeof_struct_sockaddr_in + sizeofInetPktinfo = C.sizeof_struct_in_pktinfo + + sizeofIPMreq = C.sizeof_struct_ip_mreq + sizeofIPMreqSource = C.sizeof_struct_ip_mreq_source + sizeofGroupReq = C.sizeof_struct_group_req + sizeofGroupSourceReq = C.sizeof_struct_group_source_req +) + +type sockaddrStorage C.struct_sockaddr_storage + +type sockaddrInet C.struct_sockaddr_in + +type inetPktinfo C.struct_in_pktinfo + +type ipMreq C.struct_ip_mreq + +type ipMreqSource C.struct_ip_mreq_source + +type groupReq C.struct_group_req + +type groupSourceReq C.struct_group_source_req diff --git a/vendor/golang.org/x/net/ipv4/gen.go b/vendor/golang.org/x/net/ipv4/gen.go new file mode 100644 index 000000000..1bb1737f6 --- /dev/null +++ b/vendor/golang.org/x/net/ipv4/gen.go @@ -0,0 +1,199 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +//go:generate go run gen.go + +// This program generates system adaptation constants and types, +// internet protocol constants and tables by reading template files +// and IANA protocol registries. +package main + +import ( + "bytes" + "encoding/xml" + "fmt" + "go/format" + "io" + "io/ioutil" + "net/http" + "os" + "os/exec" + "runtime" + "strconv" + "strings" +) + +func main() { + if err := genzsys(); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + if err := geniana(); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} + +func genzsys() error { + defs := "defs_" + runtime.GOOS + ".go" + f, err := os.Open(defs) + if err != nil { + if os.IsNotExist(err) { + return nil + } + return err + } + f.Close() + cmd := exec.Command("go", "tool", "cgo", "-godefs", defs) + b, err := cmd.Output() + if err != nil { + return err + } + b, err = format.Source(b) + if err != nil { + return err + } + zsys := "zsys_" + runtime.GOOS + ".go" + switch runtime.GOOS { + case "freebsd", "linux": + zsys = "zsys_" + runtime.GOOS + "_" + runtime.GOARCH + ".go" + } + if err := ioutil.WriteFile(zsys, b, 0644); err != nil { + return err + } + return nil +} + +var registries = []struct { + url string + parse func(io.Writer, io.Reader) error +}{ + { + "https://www.iana.org/assignments/icmp-parameters/icmp-parameters.xml", + parseICMPv4Parameters, + }, +} + +func geniana() error { + var bb bytes.Buffer + fmt.Fprintf(&bb, "// go generate gen.go\n") + fmt.Fprintf(&bb, "// Code generated by the command above; DO NOT EDIT.\n\n") + fmt.Fprintf(&bb, "package ipv4\n\n") + for _, r := range registries { + resp, err := http.Get(r.url) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("got HTTP status code %v for %v\n", resp.StatusCode, r.url) + } + if err := r.parse(&bb, resp.Body); err != nil { + return err + } + fmt.Fprintf(&bb, "\n") + } + b, err := format.Source(bb.Bytes()) + if err != nil { + return err + } + if err := ioutil.WriteFile("iana.go", b, 0644); err != nil { + return err + } + return nil +} + +func parseICMPv4Parameters(w io.Writer, r io.Reader) error { + dec := xml.NewDecoder(r) + var icp icmpv4Parameters + if err := dec.Decode(&icp); err != nil { + return err + } + prs := icp.escape() + fmt.Fprintf(w, "// %s, Updated: %s\n", icp.Title, icp.Updated) + fmt.Fprintf(w, "const (\n") + for _, pr := range prs { + if pr.Descr == "" { + continue + } + fmt.Fprintf(w, "ICMPType%s ICMPType = %d", pr.Descr, pr.Value) + fmt.Fprintf(w, "// %s\n", pr.OrigDescr) + } + fmt.Fprintf(w, ")\n\n") + fmt.Fprintf(w, "// %s, Updated: %s\n", icp.Title, icp.Updated) + fmt.Fprintf(w, "var icmpTypes = map[ICMPType]string{\n") + for _, pr := range prs { + if pr.Descr == "" { + continue + } + fmt.Fprintf(w, "%d: %q,\n", pr.Value, strings.ToLower(pr.OrigDescr)) + } + fmt.Fprintf(w, "}\n") + return nil +} + +type icmpv4Parameters struct { + XMLName xml.Name `xml:"registry"` + Title string `xml:"title"` + Updated string `xml:"updated"` + Registries []struct { + Title string `xml:"title"` + Records []struct { + Value string `xml:"value"` + Descr string `xml:"description"` + } `xml:"record"` + } `xml:"registry"` +} + +type canonICMPv4ParamRecord struct { + OrigDescr string + Descr string + Value int +} + +func (icp *icmpv4Parameters) escape() []canonICMPv4ParamRecord { + id := -1 + for i, r := range icp.Registries { + if strings.Contains(r.Title, "Type") || strings.Contains(r.Title, "type") { + id = i + break + } + } + if id < 0 { + return nil + } + prs := make([]canonICMPv4ParamRecord, len(icp.Registries[id].Records)) + sr := strings.NewReplacer( + "Messages", "", + "Message", "", + "ICMP", "", + "+", "P", + "-", "", + "/", "", + ".", "", + " ", "", + ) + for i, pr := range icp.Registries[id].Records { + if strings.Contains(pr.Descr, "Reserved") || + strings.Contains(pr.Descr, "Unassigned") || + strings.Contains(pr.Descr, "Deprecated") || + strings.Contains(pr.Descr, "Experiment") || + strings.Contains(pr.Descr, "experiment") { + continue + } + ss := strings.Split(pr.Descr, "\n") + if len(ss) > 1 { + prs[i].Descr = strings.Join(ss, " ") + } else { + prs[i].Descr = ss[0] + } + s := strings.TrimSpace(prs[i].Descr) + prs[i].OrigDescr = s + prs[i].Descr = sr.Replace(s) + prs[i].Value, _ = strconv.Atoi(pr.Value) + } + return prs +} diff --git a/vendor/golang.org/x/net/ipv6/defs_aix.go b/vendor/golang.org/x/net/ipv6/defs_aix.go new file mode 100644 index 000000000..ea396a3cb --- /dev/null +++ b/vendor/golang.org/x/net/ipv6/defs_aix.go @@ -0,0 +1,82 @@ +// Copyright 2019 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +// +godefs map struct_in6_addr [16]byte /* in6_addr */ + +package ipv6 + +/* +#include +#include + +#include +#include +*/ +import "C" + +const ( + sysIPV6_UNICAST_HOPS = C.IPV6_UNICAST_HOPS + sysIPV6_MULTICAST_IF = C.IPV6_MULTICAST_IF + sysIPV6_MULTICAST_HOPS = C.IPV6_MULTICAST_HOPS + sysIPV6_MULTICAST_LOOP = C.IPV6_MULTICAST_LOOP + sysIPV6_JOIN_GROUP = C.IPV6_JOIN_GROUP + sysIPV6_LEAVE_GROUP = C.IPV6_LEAVE_GROUP + sysICMP6_FILTER = C.ICMP6_FILTER + + sysIPV6_CHECKSUM = C.IPV6_CHECKSUM + sysIPV6_V6ONLY = C.IPV6_V6ONLY + + sysIPV6_RTHDRDSTOPTS = C.IPV6_RTHDRDSTOPTS + + sysIPV6_RECVPKTINFO = C.IPV6_RECVPKTINFO + sysIPV6_RECVHOPLIMIT = C.IPV6_RECVHOPLIMIT + sysIPV6_RECVRTHDR = C.IPV6_RECVRTHDR + sysIPV6_RECVHOPOPTS = C.IPV6_RECVHOPOPTS + sysIPV6_RECVDSTOPTS = C.IPV6_RECVDSTOPTS + + sysIPV6_USE_MIN_MTU = C.IPV6_USE_MIN_MTU + sysIPV6_RECVPATHMTU = C.IPV6_RECVPATHMTU + sysIPV6_PATHMTU = C.IPV6_PATHMTU + + sysIPV6_PKTINFO = C.IPV6_PKTINFO + sysIPV6_HOPLIMIT = C.IPV6_HOPLIMIT + sysIPV6_NEXTHOP = C.IPV6_NEXTHOP + sysIPV6_HOPOPTS = C.IPV6_HOPOPTS + sysIPV6_DSTOPTS = C.IPV6_DSTOPTS + sysIPV6_RTHDR = C.IPV6_RTHDR + + sysIPV6_RECVTCLASS = C.IPV6_RECVTCLASS + + sysIPV6_TCLASS = C.IPV6_TCLASS + sysIPV6_DONTFRAG = C.IPV6_DONTFRAG + + sizeofSockaddrStorage = C.sizeof_struct_sockaddr_storage + sizeofSockaddrInet6 = C.sizeof_struct_sockaddr_in6 + sizeofInet6Pktinfo = C.sizeof_struct_in6_pktinfo + sizeofIPv6Mtuinfo = C.sizeof_struct_ip6_mtuinfo + + sizeofIPv6Mreq = C.sizeof_struct_ipv6_mreq + sizeofGroupReq = C.sizeof_struct_group_req + sizeofGroupSourceReq = C.sizeof_struct_group_source_req + + sizeofICMPv6Filter = C.sizeof_struct_icmp6_filter +) + +type sockaddrStorage C.struct_sockaddr_storage + +type sockaddrInet6 C.struct_sockaddr_in6 + +type inet6Pktinfo C.struct_in6_pktinfo + +type ipv6Mtuinfo C.struct_ip6_mtuinfo + +type ipv6Mreq C.struct_ipv6_mreq + +type icmpv6Filter C.struct_icmp6_filter + +type groupReq C.struct_group_req + +type groupSourceReq C.struct_group_source_req diff --git a/vendor/golang.org/x/net/ipv6/defs_darwin.go b/vendor/golang.org/x/net/ipv6/defs_darwin.go new file mode 100644 index 000000000..55ddc116f --- /dev/null +++ b/vendor/golang.org/x/net/ipv6/defs_darwin.go @@ -0,0 +1,112 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +// +godefs map struct_in6_addr [16]byte /* in6_addr */ + +package ipv6 + +/* +#define __APPLE_USE_RFC_3542 +#include +#include +*/ +import "C" + +const ( + sysIPV6_UNICAST_HOPS = C.IPV6_UNICAST_HOPS + sysIPV6_MULTICAST_IF = C.IPV6_MULTICAST_IF + sysIPV6_MULTICAST_HOPS = C.IPV6_MULTICAST_HOPS + sysIPV6_MULTICAST_LOOP = C.IPV6_MULTICAST_LOOP + sysIPV6_JOIN_GROUP = C.IPV6_JOIN_GROUP + sysIPV6_LEAVE_GROUP = C.IPV6_LEAVE_GROUP + + sysIPV6_PORTRANGE = C.IPV6_PORTRANGE + sysICMP6_FILTER = C.ICMP6_FILTER + sysIPV6_2292PKTINFO = C.IPV6_2292PKTINFO + sysIPV6_2292HOPLIMIT = C.IPV6_2292HOPLIMIT + sysIPV6_2292NEXTHOP = C.IPV6_2292NEXTHOP + sysIPV6_2292HOPOPTS = C.IPV6_2292HOPOPTS + sysIPV6_2292DSTOPTS = C.IPV6_2292DSTOPTS + sysIPV6_2292RTHDR = C.IPV6_2292RTHDR + + sysIPV6_2292PKTOPTIONS = C.IPV6_2292PKTOPTIONS + + sysIPV6_CHECKSUM = C.IPV6_CHECKSUM + sysIPV6_V6ONLY = C.IPV6_V6ONLY + + sysIPV6_IPSEC_POLICY = C.IPV6_IPSEC_POLICY + + sysIPV6_RECVTCLASS = C.IPV6_RECVTCLASS + sysIPV6_TCLASS = C.IPV6_TCLASS + + sysIPV6_RTHDRDSTOPTS = C.IPV6_RTHDRDSTOPTS + + sysIPV6_RECVPKTINFO = C.IPV6_RECVPKTINFO + + sysIPV6_RECVHOPLIMIT = C.IPV6_RECVHOPLIMIT + sysIPV6_RECVRTHDR = C.IPV6_RECVRTHDR + sysIPV6_RECVHOPOPTS = C.IPV6_RECVHOPOPTS + sysIPV6_RECVDSTOPTS = C.IPV6_RECVDSTOPTS + + sysIPV6_USE_MIN_MTU = C.IPV6_USE_MIN_MTU + sysIPV6_RECVPATHMTU = C.IPV6_RECVPATHMTU + + sysIPV6_PATHMTU = C.IPV6_PATHMTU + + sysIPV6_PKTINFO = C.IPV6_PKTINFO + sysIPV6_HOPLIMIT = C.IPV6_HOPLIMIT + sysIPV6_NEXTHOP = C.IPV6_NEXTHOP + sysIPV6_HOPOPTS = C.IPV6_HOPOPTS + sysIPV6_DSTOPTS = C.IPV6_DSTOPTS + sysIPV6_RTHDR = C.IPV6_RTHDR + + sysIPV6_AUTOFLOWLABEL = C.IPV6_AUTOFLOWLABEL + + sysIPV6_DONTFRAG = C.IPV6_DONTFRAG + + sysIPV6_PREFER_TEMPADDR = C.IPV6_PREFER_TEMPADDR + + sysIPV6_MSFILTER = C.IPV6_MSFILTER + sysMCAST_JOIN_GROUP = C.MCAST_JOIN_GROUP + sysMCAST_LEAVE_GROUP = C.MCAST_LEAVE_GROUP + sysMCAST_JOIN_SOURCE_GROUP = C.MCAST_JOIN_SOURCE_GROUP + sysMCAST_LEAVE_SOURCE_GROUP = C.MCAST_LEAVE_SOURCE_GROUP + sysMCAST_BLOCK_SOURCE = C.MCAST_BLOCK_SOURCE + sysMCAST_UNBLOCK_SOURCE = C.MCAST_UNBLOCK_SOURCE + + sysIPV6_BOUND_IF = C.IPV6_BOUND_IF + + sysIPV6_PORTRANGE_DEFAULT = C.IPV6_PORTRANGE_DEFAULT + sysIPV6_PORTRANGE_HIGH = C.IPV6_PORTRANGE_HIGH + sysIPV6_PORTRANGE_LOW = C.IPV6_PORTRANGE_LOW + + sizeofSockaddrStorage = C.sizeof_struct_sockaddr_storage + sizeofSockaddrInet6 = C.sizeof_struct_sockaddr_in6 + sizeofInet6Pktinfo = C.sizeof_struct_in6_pktinfo + sizeofIPv6Mtuinfo = C.sizeof_struct_ip6_mtuinfo + + sizeofIPv6Mreq = C.sizeof_struct_ipv6_mreq + sizeofGroupReq = C.sizeof_struct_group_req + sizeofGroupSourceReq = C.sizeof_struct_group_source_req + + sizeofICMPv6Filter = C.sizeof_struct_icmp6_filter +) + +type sockaddrStorage C.struct_sockaddr_storage + +type sockaddrInet6 C.struct_sockaddr_in6 + +type inet6Pktinfo C.struct_in6_pktinfo + +type ipv6Mtuinfo C.struct_ip6_mtuinfo + +type ipv6Mreq C.struct_ipv6_mreq + +type icmpv6Filter C.struct_icmp6_filter + +type groupReq C.struct_group_req + +type groupSourceReq C.struct_group_source_req diff --git a/vendor/golang.org/x/net/ipv6/defs_dragonfly.go b/vendor/golang.org/x/net/ipv6/defs_dragonfly.go new file mode 100644 index 000000000..a4c383a51 --- /dev/null +++ b/vendor/golang.org/x/net/ipv6/defs_dragonfly.go @@ -0,0 +1,84 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +// +godefs map struct_in6_addr [16]byte /* in6_addr */ + +package ipv6 + +/* +#include +#include + +#include +#include +*/ +import "C" + +const ( + sysIPV6_UNICAST_HOPS = C.IPV6_UNICAST_HOPS + sysIPV6_MULTICAST_IF = C.IPV6_MULTICAST_IF + sysIPV6_MULTICAST_HOPS = C.IPV6_MULTICAST_HOPS + sysIPV6_MULTICAST_LOOP = C.IPV6_MULTICAST_LOOP + sysIPV6_JOIN_GROUP = C.IPV6_JOIN_GROUP + sysIPV6_LEAVE_GROUP = C.IPV6_LEAVE_GROUP + sysIPV6_PORTRANGE = C.IPV6_PORTRANGE + sysICMP6_FILTER = C.ICMP6_FILTER + + sysIPV6_CHECKSUM = C.IPV6_CHECKSUM + sysIPV6_V6ONLY = C.IPV6_V6ONLY + + sysIPV6_IPSEC_POLICY = C.IPV6_IPSEC_POLICY + + sysIPV6_RTHDRDSTOPTS = C.IPV6_RTHDRDSTOPTS + sysIPV6_RECVPKTINFO = C.IPV6_RECVPKTINFO + sysIPV6_RECVHOPLIMIT = C.IPV6_RECVHOPLIMIT + sysIPV6_RECVRTHDR = C.IPV6_RECVRTHDR + sysIPV6_RECVHOPOPTS = C.IPV6_RECVHOPOPTS + sysIPV6_RECVDSTOPTS = C.IPV6_RECVDSTOPTS + + sysIPV6_USE_MIN_MTU = C.IPV6_USE_MIN_MTU + sysIPV6_RECVPATHMTU = C.IPV6_RECVPATHMTU + + sysIPV6_PATHMTU = C.IPV6_PATHMTU + + sysIPV6_PKTINFO = C.IPV6_PKTINFO + sysIPV6_HOPLIMIT = C.IPV6_HOPLIMIT + sysIPV6_NEXTHOP = C.IPV6_NEXTHOP + sysIPV6_HOPOPTS = C.IPV6_HOPOPTS + sysIPV6_DSTOPTS = C.IPV6_DSTOPTS + sysIPV6_RTHDR = C.IPV6_RTHDR + + sysIPV6_RECVTCLASS = C.IPV6_RECVTCLASS + + sysIPV6_AUTOFLOWLABEL = C.IPV6_AUTOFLOWLABEL + + sysIPV6_TCLASS = C.IPV6_TCLASS + sysIPV6_DONTFRAG = C.IPV6_DONTFRAG + + sysIPV6_PREFER_TEMPADDR = C.IPV6_PREFER_TEMPADDR + + sysIPV6_PORTRANGE_DEFAULT = C.IPV6_PORTRANGE_DEFAULT + sysIPV6_PORTRANGE_HIGH = C.IPV6_PORTRANGE_HIGH + sysIPV6_PORTRANGE_LOW = C.IPV6_PORTRANGE_LOW + + sizeofSockaddrInet6 = C.sizeof_struct_sockaddr_in6 + sizeofInet6Pktinfo = C.sizeof_struct_in6_pktinfo + sizeofIPv6Mtuinfo = C.sizeof_struct_ip6_mtuinfo + + sizeofIPv6Mreq = C.sizeof_struct_ipv6_mreq + + sizeofICMPv6Filter = C.sizeof_struct_icmp6_filter +) + +type sockaddrInet6 C.struct_sockaddr_in6 + +type inet6Pktinfo C.struct_in6_pktinfo + +type ipv6Mtuinfo C.struct_ip6_mtuinfo + +type ipv6Mreq C.struct_ipv6_mreq + +type icmpv6Filter C.struct_icmp6_filter diff --git a/vendor/golang.org/x/net/ipv6/defs_freebsd.go b/vendor/golang.org/x/net/ipv6/defs_freebsd.go new file mode 100644 index 000000000..53e625389 --- /dev/null +++ b/vendor/golang.org/x/net/ipv6/defs_freebsd.go @@ -0,0 +1,105 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +// +godefs map struct_in6_addr [16]byte /* in6_addr */ + +package ipv6 + +/* +#include +#include + +#include +#include +*/ +import "C" + +const ( + sysIPV6_UNICAST_HOPS = C.IPV6_UNICAST_HOPS + sysIPV6_MULTICAST_IF = C.IPV6_MULTICAST_IF + sysIPV6_MULTICAST_HOPS = C.IPV6_MULTICAST_HOPS + sysIPV6_MULTICAST_LOOP = C.IPV6_MULTICAST_LOOP + sysIPV6_JOIN_GROUP = C.IPV6_JOIN_GROUP + sysIPV6_LEAVE_GROUP = C.IPV6_LEAVE_GROUP + sysIPV6_PORTRANGE = C.IPV6_PORTRANGE + sysICMP6_FILTER = C.ICMP6_FILTER + + sysIPV6_CHECKSUM = C.IPV6_CHECKSUM + sysIPV6_V6ONLY = C.IPV6_V6ONLY + + sysIPV6_IPSEC_POLICY = C.IPV6_IPSEC_POLICY + + sysIPV6_RTHDRDSTOPTS = C.IPV6_RTHDRDSTOPTS + + sysIPV6_RECVPKTINFO = C.IPV6_RECVPKTINFO + sysIPV6_RECVHOPLIMIT = C.IPV6_RECVHOPLIMIT + sysIPV6_RECVRTHDR = C.IPV6_RECVRTHDR + sysIPV6_RECVHOPOPTS = C.IPV6_RECVHOPOPTS + sysIPV6_RECVDSTOPTS = C.IPV6_RECVDSTOPTS + + sysIPV6_USE_MIN_MTU = C.IPV6_USE_MIN_MTU + sysIPV6_RECVPATHMTU = C.IPV6_RECVPATHMTU + + sysIPV6_PATHMTU = C.IPV6_PATHMTU + + sysIPV6_PKTINFO = C.IPV6_PKTINFO + sysIPV6_HOPLIMIT = C.IPV6_HOPLIMIT + sysIPV6_NEXTHOP = C.IPV6_NEXTHOP + sysIPV6_HOPOPTS = C.IPV6_HOPOPTS + sysIPV6_DSTOPTS = C.IPV6_DSTOPTS + sysIPV6_RTHDR = C.IPV6_RTHDR + + sysIPV6_RECVTCLASS = C.IPV6_RECVTCLASS + + sysIPV6_AUTOFLOWLABEL = C.IPV6_AUTOFLOWLABEL + + sysIPV6_TCLASS = C.IPV6_TCLASS + sysIPV6_DONTFRAG = C.IPV6_DONTFRAG + + sysIPV6_PREFER_TEMPADDR = C.IPV6_PREFER_TEMPADDR + + sysIPV6_BINDANY = C.IPV6_BINDANY + + sysIPV6_MSFILTER = C.IPV6_MSFILTER + + sysMCAST_JOIN_GROUP = C.MCAST_JOIN_GROUP + sysMCAST_LEAVE_GROUP = C.MCAST_LEAVE_GROUP + sysMCAST_JOIN_SOURCE_GROUP = C.MCAST_JOIN_SOURCE_GROUP + sysMCAST_LEAVE_SOURCE_GROUP = C.MCAST_LEAVE_SOURCE_GROUP + sysMCAST_BLOCK_SOURCE = C.MCAST_BLOCK_SOURCE + sysMCAST_UNBLOCK_SOURCE = C.MCAST_UNBLOCK_SOURCE + + sysIPV6_PORTRANGE_DEFAULT = C.IPV6_PORTRANGE_DEFAULT + sysIPV6_PORTRANGE_HIGH = C.IPV6_PORTRANGE_HIGH + sysIPV6_PORTRANGE_LOW = C.IPV6_PORTRANGE_LOW + + sizeofSockaddrStorage = C.sizeof_struct_sockaddr_storage + sizeofSockaddrInet6 = C.sizeof_struct_sockaddr_in6 + sizeofInet6Pktinfo = C.sizeof_struct_in6_pktinfo + sizeofIPv6Mtuinfo = C.sizeof_struct_ip6_mtuinfo + + sizeofIPv6Mreq = C.sizeof_struct_ipv6_mreq + sizeofGroupReq = C.sizeof_struct_group_req + sizeofGroupSourceReq = C.sizeof_struct_group_source_req + + sizeofICMPv6Filter = C.sizeof_struct_icmp6_filter +) + +type sockaddrStorage C.struct_sockaddr_storage + +type sockaddrInet6 C.struct_sockaddr_in6 + +type inet6Pktinfo C.struct_in6_pktinfo + +type ipv6Mtuinfo C.struct_ip6_mtuinfo + +type ipv6Mreq C.struct_ipv6_mreq + +type groupReq C.struct_group_req + +type groupSourceReq C.struct_group_source_req + +type icmpv6Filter C.struct_icmp6_filter diff --git a/vendor/golang.org/x/net/ipv6/defs_linux.go b/vendor/golang.org/x/net/ipv6/defs_linux.go new file mode 100644 index 000000000..3308cb2c3 --- /dev/null +++ b/vendor/golang.org/x/net/ipv6/defs_linux.go @@ -0,0 +1,147 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +// +godefs map struct_in6_addr [16]byte /* in6_addr */ + +package ipv6 + +/* +#include +#include +#include +#include +#include +#include +*/ +import "C" + +const ( + sysIPV6_ADDRFORM = C.IPV6_ADDRFORM + sysIPV6_2292PKTINFO = C.IPV6_2292PKTINFO + sysIPV6_2292HOPOPTS = C.IPV6_2292HOPOPTS + sysIPV6_2292DSTOPTS = C.IPV6_2292DSTOPTS + sysIPV6_2292RTHDR = C.IPV6_2292RTHDR + sysIPV6_2292PKTOPTIONS = C.IPV6_2292PKTOPTIONS + sysIPV6_CHECKSUM = C.IPV6_CHECKSUM + sysIPV6_2292HOPLIMIT = C.IPV6_2292HOPLIMIT + sysIPV6_NEXTHOP = C.IPV6_NEXTHOP + sysIPV6_FLOWINFO = C.IPV6_FLOWINFO + + sysIPV6_UNICAST_HOPS = C.IPV6_UNICAST_HOPS + sysIPV6_MULTICAST_IF = C.IPV6_MULTICAST_IF + sysIPV6_MULTICAST_HOPS = C.IPV6_MULTICAST_HOPS + sysIPV6_MULTICAST_LOOP = C.IPV6_MULTICAST_LOOP + sysIPV6_ADD_MEMBERSHIP = C.IPV6_ADD_MEMBERSHIP + sysIPV6_DROP_MEMBERSHIP = C.IPV6_DROP_MEMBERSHIP + sysMCAST_JOIN_GROUP = C.MCAST_JOIN_GROUP + sysMCAST_LEAVE_GROUP = C.MCAST_LEAVE_GROUP + sysMCAST_JOIN_SOURCE_GROUP = C.MCAST_JOIN_SOURCE_GROUP + sysMCAST_LEAVE_SOURCE_GROUP = C.MCAST_LEAVE_SOURCE_GROUP + sysMCAST_BLOCK_SOURCE = C.MCAST_BLOCK_SOURCE + sysMCAST_UNBLOCK_SOURCE = C.MCAST_UNBLOCK_SOURCE + sysMCAST_MSFILTER = C.MCAST_MSFILTER + sysIPV6_ROUTER_ALERT = C.IPV6_ROUTER_ALERT + sysIPV6_MTU_DISCOVER = C.IPV6_MTU_DISCOVER + sysIPV6_MTU = C.IPV6_MTU + sysIPV6_RECVERR = C.IPV6_RECVERR + sysIPV6_V6ONLY = C.IPV6_V6ONLY + sysIPV6_JOIN_ANYCAST = C.IPV6_JOIN_ANYCAST + sysIPV6_LEAVE_ANYCAST = C.IPV6_LEAVE_ANYCAST + + //sysIPV6_PMTUDISC_DONT = C.IPV6_PMTUDISC_DONT + //sysIPV6_PMTUDISC_WANT = C.IPV6_PMTUDISC_WANT + //sysIPV6_PMTUDISC_DO = C.IPV6_PMTUDISC_DO + //sysIPV6_PMTUDISC_PROBE = C.IPV6_PMTUDISC_PROBE + //sysIPV6_PMTUDISC_INTERFACE = C.IPV6_PMTUDISC_INTERFACE + //sysIPV6_PMTUDISC_OMIT = C.IPV6_PMTUDISC_OMIT + + sysIPV6_FLOWLABEL_MGR = C.IPV6_FLOWLABEL_MGR + sysIPV6_FLOWINFO_SEND = C.IPV6_FLOWINFO_SEND + + sysIPV6_IPSEC_POLICY = C.IPV6_IPSEC_POLICY + sysIPV6_XFRM_POLICY = C.IPV6_XFRM_POLICY + + sysIPV6_RECVPKTINFO = C.IPV6_RECVPKTINFO + sysIPV6_PKTINFO = C.IPV6_PKTINFO + sysIPV6_RECVHOPLIMIT = C.IPV6_RECVHOPLIMIT + sysIPV6_HOPLIMIT = C.IPV6_HOPLIMIT + sysIPV6_RECVHOPOPTS = C.IPV6_RECVHOPOPTS + sysIPV6_HOPOPTS = C.IPV6_HOPOPTS + sysIPV6_RTHDRDSTOPTS = C.IPV6_RTHDRDSTOPTS + sysIPV6_RECVRTHDR = C.IPV6_RECVRTHDR + sysIPV6_RTHDR = C.IPV6_RTHDR + sysIPV6_RECVDSTOPTS = C.IPV6_RECVDSTOPTS + sysIPV6_DSTOPTS = C.IPV6_DSTOPTS + sysIPV6_RECVPATHMTU = C.IPV6_RECVPATHMTU + sysIPV6_PATHMTU = C.IPV6_PATHMTU + sysIPV6_DONTFRAG = C.IPV6_DONTFRAG + + sysIPV6_RECVTCLASS = C.IPV6_RECVTCLASS + sysIPV6_TCLASS = C.IPV6_TCLASS + + sysIPV6_ADDR_PREFERENCES = C.IPV6_ADDR_PREFERENCES + + sysIPV6_PREFER_SRC_TMP = C.IPV6_PREFER_SRC_TMP + sysIPV6_PREFER_SRC_PUBLIC = C.IPV6_PREFER_SRC_PUBLIC + sysIPV6_PREFER_SRC_PUBTMP_DEFAULT = C.IPV6_PREFER_SRC_PUBTMP_DEFAULT + sysIPV6_PREFER_SRC_COA = C.IPV6_PREFER_SRC_COA + sysIPV6_PREFER_SRC_HOME = C.IPV6_PREFER_SRC_HOME + sysIPV6_PREFER_SRC_CGA = C.IPV6_PREFER_SRC_CGA + sysIPV6_PREFER_SRC_NONCGA = C.IPV6_PREFER_SRC_NONCGA + + sysIPV6_MINHOPCOUNT = C.IPV6_MINHOPCOUNT + + sysIPV6_ORIGDSTADDR = C.IPV6_ORIGDSTADDR + sysIPV6_RECVORIGDSTADDR = C.IPV6_RECVORIGDSTADDR + sysIPV6_TRANSPARENT = C.IPV6_TRANSPARENT + sysIPV6_UNICAST_IF = C.IPV6_UNICAST_IF + + sysICMPV6_FILTER = C.ICMPV6_FILTER + + sysICMPV6_FILTER_BLOCK = C.ICMPV6_FILTER_BLOCK + sysICMPV6_FILTER_PASS = C.ICMPV6_FILTER_PASS + sysICMPV6_FILTER_BLOCKOTHERS = C.ICMPV6_FILTER_BLOCKOTHERS + sysICMPV6_FILTER_PASSONLY = C.ICMPV6_FILTER_PASSONLY + + sysSOL_SOCKET = C.SOL_SOCKET + sysSO_ATTACH_FILTER = C.SO_ATTACH_FILTER + + sizeofKernelSockaddrStorage = C.sizeof_struct___kernel_sockaddr_storage + sizeofSockaddrInet6 = C.sizeof_struct_sockaddr_in6 + sizeofInet6Pktinfo = C.sizeof_struct_in6_pktinfo + sizeofIPv6Mtuinfo = C.sizeof_struct_ip6_mtuinfo + sizeofIPv6FlowlabelReq = C.sizeof_struct_in6_flowlabel_req + + sizeofIPv6Mreq = C.sizeof_struct_ipv6_mreq + sizeofGroupReq = C.sizeof_struct_group_req + sizeofGroupSourceReq = C.sizeof_struct_group_source_req + + sizeofICMPv6Filter = C.sizeof_struct_icmp6_filter + + sizeofSockFprog = C.sizeof_struct_sock_fprog +) + +type kernelSockaddrStorage C.struct___kernel_sockaddr_storage + +type sockaddrInet6 C.struct_sockaddr_in6 + +type inet6Pktinfo C.struct_in6_pktinfo + +type ipv6Mtuinfo C.struct_ip6_mtuinfo + +type ipv6FlowlabelReq C.struct_in6_flowlabel_req + +type ipv6Mreq C.struct_ipv6_mreq + +type groupReq C.struct_group_req + +type groupSourceReq C.struct_group_source_req + +type icmpv6Filter C.struct_icmp6_filter + +type sockFProg C.struct_sock_fprog + +type sockFilter C.struct_sock_filter diff --git a/vendor/golang.org/x/net/ipv6/defs_netbsd.go b/vendor/golang.org/x/net/ipv6/defs_netbsd.go new file mode 100644 index 000000000..be9ceb9cc --- /dev/null +++ b/vendor/golang.org/x/net/ipv6/defs_netbsd.go @@ -0,0 +1,80 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +// +godefs map struct_in6_addr [16]byte /* in6_addr */ + +package ipv6 + +/* +#include +#include + +#include +#include +*/ +import "C" + +const ( + sysIPV6_UNICAST_HOPS = C.IPV6_UNICAST_HOPS + sysIPV6_MULTICAST_IF = C.IPV6_MULTICAST_IF + sysIPV6_MULTICAST_HOPS = C.IPV6_MULTICAST_HOPS + sysIPV6_MULTICAST_LOOP = C.IPV6_MULTICAST_LOOP + sysIPV6_JOIN_GROUP = C.IPV6_JOIN_GROUP + sysIPV6_LEAVE_GROUP = C.IPV6_LEAVE_GROUP + sysIPV6_PORTRANGE = C.IPV6_PORTRANGE + sysICMP6_FILTER = C.ICMP6_FILTER + + sysIPV6_CHECKSUM = C.IPV6_CHECKSUM + sysIPV6_V6ONLY = C.IPV6_V6ONLY + + sysIPV6_IPSEC_POLICY = C.IPV6_IPSEC_POLICY + + sysIPV6_RTHDRDSTOPTS = C.IPV6_RTHDRDSTOPTS + + sysIPV6_RECVPKTINFO = C.IPV6_RECVPKTINFO + sysIPV6_RECVHOPLIMIT = C.IPV6_RECVHOPLIMIT + sysIPV6_RECVRTHDR = C.IPV6_RECVRTHDR + sysIPV6_RECVHOPOPTS = C.IPV6_RECVHOPOPTS + sysIPV6_RECVDSTOPTS = C.IPV6_RECVDSTOPTS + + sysIPV6_USE_MIN_MTU = C.IPV6_USE_MIN_MTU + sysIPV6_RECVPATHMTU = C.IPV6_RECVPATHMTU + sysIPV6_PATHMTU = C.IPV6_PATHMTU + + sysIPV6_PKTINFO = C.IPV6_PKTINFO + sysIPV6_HOPLIMIT = C.IPV6_HOPLIMIT + sysIPV6_NEXTHOP = C.IPV6_NEXTHOP + sysIPV6_HOPOPTS = C.IPV6_HOPOPTS + sysIPV6_DSTOPTS = C.IPV6_DSTOPTS + sysIPV6_RTHDR = C.IPV6_RTHDR + + sysIPV6_RECVTCLASS = C.IPV6_RECVTCLASS + + sysIPV6_TCLASS = C.IPV6_TCLASS + sysIPV6_DONTFRAG = C.IPV6_DONTFRAG + + sysIPV6_PORTRANGE_DEFAULT = C.IPV6_PORTRANGE_DEFAULT + sysIPV6_PORTRANGE_HIGH = C.IPV6_PORTRANGE_HIGH + sysIPV6_PORTRANGE_LOW = C.IPV6_PORTRANGE_LOW + + sizeofSockaddrInet6 = C.sizeof_struct_sockaddr_in6 + sizeofInet6Pktinfo = C.sizeof_struct_in6_pktinfo + sizeofIPv6Mtuinfo = C.sizeof_struct_ip6_mtuinfo + + sizeofIPv6Mreq = C.sizeof_struct_ipv6_mreq + + sizeofICMPv6Filter = C.sizeof_struct_icmp6_filter +) + +type sockaddrInet6 C.struct_sockaddr_in6 + +type inet6Pktinfo C.struct_in6_pktinfo + +type ipv6Mtuinfo C.struct_ip6_mtuinfo + +type ipv6Mreq C.struct_ipv6_mreq + +type icmpv6Filter C.struct_icmp6_filter diff --git a/vendor/golang.org/x/net/ipv6/defs_openbsd.go b/vendor/golang.org/x/net/ipv6/defs_openbsd.go new file mode 100644 index 000000000..177ddf87d --- /dev/null +++ b/vendor/golang.org/x/net/ipv6/defs_openbsd.go @@ -0,0 +1,89 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +// +godefs map struct_in6_addr [16]byte /* in6_addr */ + +package ipv6 + +/* +#include +#include + +#include +#include +*/ +import "C" + +const ( + sysIPV6_UNICAST_HOPS = C.IPV6_UNICAST_HOPS + sysIPV6_MULTICAST_IF = C.IPV6_MULTICAST_IF + sysIPV6_MULTICAST_HOPS = C.IPV6_MULTICAST_HOPS + sysIPV6_MULTICAST_LOOP = C.IPV6_MULTICAST_LOOP + sysIPV6_JOIN_GROUP = C.IPV6_JOIN_GROUP + sysIPV6_LEAVE_GROUP = C.IPV6_LEAVE_GROUP + sysIPV6_PORTRANGE = C.IPV6_PORTRANGE + sysICMP6_FILTER = C.ICMP6_FILTER + + sysIPV6_CHECKSUM = C.IPV6_CHECKSUM + sysIPV6_V6ONLY = C.IPV6_V6ONLY + + sysIPV6_RTHDRDSTOPTS = C.IPV6_RTHDRDSTOPTS + + sysIPV6_RECVPKTINFO = C.IPV6_RECVPKTINFO + sysIPV6_RECVHOPLIMIT = C.IPV6_RECVHOPLIMIT + sysIPV6_RECVRTHDR = C.IPV6_RECVRTHDR + sysIPV6_RECVHOPOPTS = C.IPV6_RECVHOPOPTS + sysIPV6_RECVDSTOPTS = C.IPV6_RECVDSTOPTS + + sysIPV6_USE_MIN_MTU = C.IPV6_USE_MIN_MTU + sysIPV6_RECVPATHMTU = C.IPV6_RECVPATHMTU + + sysIPV6_PATHMTU = C.IPV6_PATHMTU + + sysIPV6_PKTINFO = C.IPV6_PKTINFO + sysIPV6_HOPLIMIT = C.IPV6_HOPLIMIT + sysIPV6_NEXTHOP = C.IPV6_NEXTHOP + sysIPV6_HOPOPTS = C.IPV6_HOPOPTS + sysIPV6_DSTOPTS = C.IPV6_DSTOPTS + sysIPV6_RTHDR = C.IPV6_RTHDR + + sysIPV6_AUTH_LEVEL = C.IPV6_AUTH_LEVEL + sysIPV6_ESP_TRANS_LEVEL = C.IPV6_ESP_TRANS_LEVEL + sysIPV6_ESP_NETWORK_LEVEL = C.IPV6_ESP_NETWORK_LEVEL + sysIPSEC6_OUTSA = C.IPSEC6_OUTSA + sysIPV6_RECVTCLASS = C.IPV6_RECVTCLASS + + sysIPV6_AUTOFLOWLABEL = C.IPV6_AUTOFLOWLABEL + sysIPV6_IPCOMP_LEVEL = C.IPV6_IPCOMP_LEVEL + + sysIPV6_TCLASS = C.IPV6_TCLASS + sysIPV6_DONTFRAG = C.IPV6_DONTFRAG + sysIPV6_PIPEX = C.IPV6_PIPEX + + sysIPV6_RTABLE = C.IPV6_RTABLE + + sysIPV6_PORTRANGE_DEFAULT = C.IPV6_PORTRANGE_DEFAULT + sysIPV6_PORTRANGE_HIGH = C.IPV6_PORTRANGE_HIGH + sysIPV6_PORTRANGE_LOW = C.IPV6_PORTRANGE_LOW + + sizeofSockaddrInet6 = C.sizeof_struct_sockaddr_in6 + sizeofInet6Pktinfo = C.sizeof_struct_in6_pktinfo + sizeofIPv6Mtuinfo = C.sizeof_struct_ip6_mtuinfo + + sizeofIPv6Mreq = C.sizeof_struct_ipv6_mreq + + sizeofICMPv6Filter = C.sizeof_struct_icmp6_filter +) + +type sockaddrInet6 C.struct_sockaddr_in6 + +type inet6Pktinfo C.struct_in6_pktinfo + +type ipv6Mtuinfo C.struct_ip6_mtuinfo + +type ipv6Mreq C.struct_ipv6_mreq + +type icmpv6Filter C.struct_icmp6_filter diff --git a/vendor/golang.org/x/net/ipv6/defs_solaris.go b/vendor/golang.org/x/net/ipv6/defs_solaris.go new file mode 100644 index 000000000..0f8ce2b46 --- /dev/null +++ b/vendor/golang.org/x/net/ipv6/defs_solaris.go @@ -0,0 +1,114 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +// +godefs map struct_in6_addr [16]byte /* in6_addr */ + +package ipv6 + +/* +#include + +#include +#include +*/ +import "C" + +const ( + sysIPV6_UNICAST_HOPS = C.IPV6_UNICAST_HOPS + sysIPV6_MULTICAST_IF = C.IPV6_MULTICAST_IF + sysIPV6_MULTICAST_HOPS = C.IPV6_MULTICAST_HOPS + sysIPV6_MULTICAST_LOOP = C.IPV6_MULTICAST_LOOP + sysIPV6_JOIN_GROUP = C.IPV6_JOIN_GROUP + sysIPV6_LEAVE_GROUP = C.IPV6_LEAVE_GROUP + + sysIPV6_PKTINFO = C.IPV6_PKTINFO + + sysIPV6_HOPLIMIT = C.IPV6_HOPLIMIT + sysIPV6_NEXTHOP = C.IPV6_NEXTHOP + sysIPV6_HOPOPTS = C.IPV6_HOPOPTS + sysIPV6_DSTOPTS = C.IPV6_DSTOPTS + + sysIPV6_RTHDR = C.IPV6_RTHDR + sysIPV6_RTHDRDSTOPTS = C.IPV6_RTHDRDSTOPTS + + sysIPV6_RECVPKTINFO = C.IPV6_RECVPKTINFO + sysIPV6_RECVHOPLIMIT = C.IPV6_RECVHOPLIMIT + sysIPV6_RECVHOPOPTS = C.IPV6_RECVHOPOPTS + + sysIPV6_RECVRTHDR = C.IPV6_RECVRTHDR + + sysIPV6_RECVRTHDRDSTOPTS = C.IPV6_RECVRTHDRDSTOPTS + + sysIPV6_CHECKSUM = C.IPV6_CHECKSUM + sysIPV6_RECVTCLASS = C.IPV6_RECVTCLASS + sysIPV6_USE_MIN_MTU = C.IPV6_USE_MIN_MTU + sysIPV6_DONTFRAG = C.IPV6_DONTFRAG + sysIPV6_SEC_OPT = C.IPV6_SEC_OPT + sysIPV6_SRC_PREFERENCES = C.IPV6_SRC_PREFERENCES + sysIPV6_RECVPATHMTU = C.IPV6_RECVPATHMTU + sysIPV6_PATHMTU = C.IPV6_PATHMTU + sysIPV6_TCLASS = C.IPV6_TCLASS + sysIPV6_V6ONLY = C.IPV6_V6ONLY + + sysIPV6_RECVDSTOPTS = C.IPV6_RECVDSTOPTS + + sysMCAST_JOIN_GROUP = C.MCAST_JOIN_GROUP + sysMCAST_LEAVE_GROUP = C.MCAST_LEAVE_GROUP + sysMCAST_BLOCK_SOURCE = C.MCAST_BLOCK_SOURCE + sysMCAST_UNBLOCK_SOURCE = C.MCAST_UNBLOCK_SOURCE + sysMCAST_JOIN_SOURCE_GROUP = C.MCAST_JOIN_SOURCE_GROUP + sysMCAST_LEAVE_SOURCE_GROUP = C.MCAST_LEAVE_SOURCE_GROUP + + sysIPV6_PREFER_SRC_HOME = C.IPV6_PREFER_SRC_HOME + sysIPV6_PREFER_SRC_COA = C.IPV6_PREFER_SRC_COA + sysIPV6_PREFER_SRC_PUBLIC = C.IPV6_PREFER_SRC_PUBLIC + sysIPV6_PREFER_SRC_TMP = C.IPV6_PREFER_SRC_TMP + sysIPV6_PREFER_SRC_NONCGA = C.IPV6_PREFER_SRC_NONCGA + sysIPV6_PREFER_SRC_CGA = C.IPV6_PREFER_SRC_CGA + + sysIPV6_PREFER_SRC_MIPMASK = C.IPV6_PREFER_SRC_MIPMASK + sysIPV6_PREFER_SRC_MIPDEFAULT = C.IPV6_PREFER_SRC_MIPDEFAULT + sysIPV6_PREFER_SRC_TMPMASK = C.IPV6_PREFER_SRC_TMPMASK + sysIPV6_PREFER_SRC_TMPDEFAULT = C.IPV6_PREFER_SRC_TMPDEFAULT + sysIPV6_PREFER_SRC_CGAMASK = C.IPV6_PREFER_SRC_CGAMASK + sysIPV6_PREFER_SRC_CGADEFAULT = C.IPV6_PREFER_SRC_CGADEFAULT + + sysIPV6_PREFER_SRC_MASK = C.IPV6_PREFER_SRC_MASK + + sysIPV6_PREFER_SRC_DEFAULT = C.IPV6_PREFER_SRC_DEFAULT + + sysIPV6_BOUND_IF = C.IPV6_BOUND_IF + sysIPV6_UNSPEC_SRC = C.IPV6_UNSPEC_SRC + + sysICMP6_FILTER = C.ICMP6_FILTER + + sizeofSockaddrStorage = C.sizeof_struct_sockaddr_storage + sizeofSockaddrInet6 = C.sizeof_struct_sockaddr_in6 + sizeofInet6Pktinfo = C.sizeof_struct_in6_pktinfo + sizeofIPv6Mtuinfo = C.sizeof_struct_ip6_mtuinfo + + sizeofIPv6Mreq = C.sizeof_struct_ipv6_mreq + sizeofGroupReq = C.sizeof_struct_group_req + sizeofGroupSourceReq = C.sizeof_struct_group_source_req + + sizeofICMPv6Filter = C.sizeof_struct_icmp6_filter +) + +type sockaddrStorage C.struct_sockaddr_storage + +type sockaddrInet6 C.struct_sockaddr_in6 + +type inet6Pktinfo C.struct_in6_pktinfo + +type ipv6Mtuinfo C.struct_ip6_mtuinfo + +type ipv6Mreq C.struct_ipv6_mreq + +type groupReq C.struct_group_req + +type groupSourceReq C.struct_group_source_req + +type icmpv6Filter C.struct_icmp6_filter diff --git a/vendor/golang.org/x/net/ipv6/gen.go b/vendor/golang.org/x/net/ipv6/gen.go new file mode 100644 index 000000000..5885664fb --- /dev/null +++ b/vendor/golang.org/x/net/ipv6/gen.go @@ -0,0 +1,199 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +//go:generate go run gen.go + +// This program generates system adaptation constants and types, +// internet protocol constants and tables by reading template files +// and IANA protocol registries. +package main + +import ( + "bytes" + "encoding/xml" + "fmt" + "go/format" + "io" + "io/ioutil" + "net/http" + "os" + "os/exec" + "runtime" + "strconv" + "strings" +) + +func main() { + if err := genzsys(); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + if err := geniana(); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} + +func genzsys() error { + defs := "defs_" + runtime.GOOS + ".go" + f, err := os.Open(defs) + if err != nil { + if os.IsNotExist(err) { + return nil + } + return err + } + f.Close() + cmd := exec.Command("go", "tool", "cgo", "-godefs", defs) + b, err := cmd.Output() + if err != nil { + return err + } + b, err = format.Source(b) + if err != nil { + return err + } + zsys := "zsys_" + runtime.GOOS + ".go" + switch runtime.GOOS { + case "freebsd", "linux": + zsys = "zsys_" + runtime.GOOS + "_" + runtime.GOARCH + ".go" + } + if err := ioutil.WriteFile(zsys, b, 0644); err != nil { + return err + } + return nil +} + +var registries = []struct { + url string + parse func(io.Writer, io.Reader) error +}{ + { + "https://www.iana.org/assignments/icmpv6-parameters/icmpv6-parameters.xml", + parseICMPv6Parameters, + }, +} + +func geniana() error { + var bb bytes.Buffer + fmt.Fprintf(&bb, "// go generate gen.go\n") + fmt.Fprintf(&bb, "// Code generated by the command above; DO NOT EDIT.\n\n") + fmt.Fprintf(&bb, "package ipv6\n\n") + for _, r := range registries { + resp, err := http.Get(r.url) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("got HTTP status code %v for %v\n", resp.StatusCode, r.url) + } + if err := r.parse(&bb, resp.Body); err != nil { + return err + } + fmt.Fprintf(&bb, "\n") + } + b, err := format.Source(bb.Bytes()) + if err != nil { + return err + } + if err := ioutil.WriteFile("iana.go", b, 0644); err != nil { + return err + } + return nil +} + +func parseICMPv6Parameters(w io.Writer, r io.Reader) error { + dec := xml.NewDecoder(r) + var icp icmpv6Parameters + if err := dec.Decode(&icp); err != nil { + return err + } + prs := icp.escape() + fmt.Fprintf(w, "// %s, Updated: %s\n", icp.Title, icp.Updated) + fmt.Fprintf(w, "const (\n") + for _, pr := range prs { + if pr.Name == "" { + continue + } + fmt.Fprintf(w, "ICMPType%s ICMPType = %d", pr.Name, pr.Value) + fmt.Fprintf(w, "// %s\n", pr.OrigName) + } + fmt.Fprintf(w, ")\n\n") + fmt.Fprintf(w, "// %s, Updated: %s\n", icp.Title, icp.Updated) + fmt.Fprintf(w, "var icmpTypes = map[ICMPType]string{\n") + for _, pr := range prs { + if pr.Name == "" { + continue + } + fmt.Fprintf(w, "%d: %q,\n", pr.Value, strings.ToLower(pr.OrigName)) + } + fmt.Fprintf(w, "}\n") + return nil +} + +type icmpv6Parameters struct { + XMLName xml.Name `xml:"registry"` + Title string `xml:"title"` + Updated string `xml:"updated"` + Registries []struct { + Title string `xml:"title"` + Records []struct { + Value string `xml:"value"` + Name string `xml:"name"` + } `xml:"record"` + } `xml:"registry"` +} + +type canonICMPv6ParamRecord struct { + OrigName string + Name string + Value int +} + +func (icp *icmpv6Parameters) escape() []canonICMPv6ParamRecord { + id := -1 + for i, r := range icp.Registries { + if strings.Contains(r.Title, "Type") || strings.Contains(r.Title, "type") { + id = i + break + } + } + if id < 0 { + return nil + } + prs := make([]canonICMPv6ParamRecord, len(icp.Registries[id].Records)) + sr := strings.NewReplacer( + "Messages", "", + "Message", "", + "ICMP", "", + "+", "P", + "-", "", + "/", "", + ".", "", + " ", "", + ) + for i, pr := range icp.Registries[id].Records { + if strings.Contains(pr.Name, "Reserved") || + strings.Contains(pr.Name, "Unassigned") || + strings.Contains(pr.Name, "Deprecated") || + strings.Contains(pr.Name, "Experiment") || + strings.Contains(pr.Name, "experiment") { + continue + } + ss := strings.Split(pr.Name, "\n") + if len(ss) > 1 { + prs[i].Name = strings.Join(ss, " ") + } else { + prs[i].Name = ss[0] + } + s := strings.TrimSpace(prs[i].Name) + prs[i].OrigName = s + prs[i].Name = sr.Replace(s) + prs[i].Value, _ = strconv.Atoi(pr.Value) + } + return prs +} diff --git a/vendor/golang.org/x/sys/unix/mkasm_darwin.go b/vendor/golang.org/x/sys/unix/mkasm_darwin.go new file mode 100644 index 000000000..4548b993d --- /dev/null +++ b/vendor/golang.org/x/sys/unix/mkasm_darwin.go @@ -0,0 +1,61 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +// mkasm_darwin.go generates assembly trampolines to call libSystem routines from Go. +//This program must be run after mksyscall.go. +package main + +import ( + "bytes" + "fmt" + "io/ioutil" + "log" + "os" + "strings" +) + +func main() { + in1, err := ioutil.ReadFile("syscall_darwin.go") + if err != nil { + log.Fatalf("can't open syscall_darwin.go: %s", err) + } + arch := os.Args[1] + in2, err := ioutil.ReadFile(fmt.Sprintf("syscall_darwin_%s.go", arch)) + if err != nil { + log.Fatalf("can't open syscall_darwin_%s.go: %s", arch, err) + } + in3, err := ioutil.ReadFile(fmt.Sprintf("zsyscall_darwin_%s.go", arch)) + if err != nil { + log.Fatalf("can't open zsyscall_darwin_%s.go: %s", arch, err) + } + in := string(in1) + string(in2) + string(in3) + + trampolines := map[string]bool{} + + var out bytes.Buffer + + fmt.Fprintf(&out, "// go run mkasm_darwin.go %s\n", strings.Join(os.Args[1:], " ")) + fmt.Fprintf(&out, "// Code generated by the command above; DO NOT EDIT.\n") + fmt.Fprintf(&out, "\n") + fmt.Fprintf(&out, "// +build go1.12\n") + fmt.Fprintf(&out, "\n") + fmt.Fprintf(&out, "#include \"textflag.h\"\n") + for _, line := range strings.Split(in, "\n") { + if !strings.HasPrefix(line, "func ") || !strings.HasSuffix(line, "_trampoline()") { + continue + } + fn := line[5 : len(line)-13] + if !trampolines[fn] { + trampolines[fn] = true + fmt.Fprintf(&out, "TEXT ·%s_trampoline(SB),NOSPLIT,$0-0\n", fn) + fmt.Fprintf(&out, "\tJMP\t%s(SB)\n", fn) + } + } + err = ioutil.WriteFile(fmt.Sprintf("zsyscall_darwin_%s.s", arch), out.Bytes(), 0644) + if err != nil { + log.Fatalf("can't write zsyscall_darwin_%s.s: %s", arch, err) + } +} diff --git a/vendor/golang.org/x/sys/unix/mkpost.go b/vendor/golang.org/x/sys/unix/mkpost.go new file mode 100644 index 000000000..9feddd00c --- /dev/null +++ b/vendor/golang.org/x/sys/unix/mkpost.go @@ -0,0 +1,106 @@ +// Copyright 2016 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +// mkpost processes the output of cgo -godefs to +// modify the generated types. It is used to clean up +// the sys API in an architecture specific manner. +// +// mkpost is run after cgo -godefs; see README.md. +package main + +import ( + "bytes" + "fmt" + "go/format" + "io/ioutil" + "log" + "os" + "regexp" +) + +func main() { + // Get the OS and architecture (using GOARCH_TARGET if it exists) + goos := os.Getenv("GOOS") + goarch := os.Getenv("GOARCH_TARGET") + if goarch == "" { + goarch = os.Getenv("GOARCH") + } + // Check that we are using the Docker-based build system if we should be. + if goos == "linux" { + if os.Getenv("GOLANG_SYS_BUILD") != "docker" { + os.Stderr.WriteString("In the Docker-based build system, mkpost should not be called directly.\n") + os.Stderr.WriteString("See README.md\n") + os.Exit(1) + } + } + + b, err := ioutil.ReadAll(os.Stdin) + if err != nil { + log.Fatal(err) + } + + // Intentionally export __val fields in Fsid and Sigset_t + valRegex := regexp.MustCompile(`type (Fsid|Sigset_t) struct {(\s+)X__val(\s+\S+\s+)}`) + b = valRegex.ReplaceAll(b, []byte("type $1 struct {${2}Val$3}")) + + // Intentionally export __fds_bits field in FdSet + fdSetRegex := regexp.MustCompile(`type (FdSet) struct {(\s+)X__fds_bits(\s+\S+\s+)}`) + b = fdSetRegex.ReplaceAll(b, []byte("type $1 struct {${2}Bits$3}")) + + // If we have empty Ptrace structs, we should delete them. Only s390x emits + // nonempty Ptrace structs. + ptraceRexexp := regexp.MustCompile(`type Ptrace((Psw|Fpregs|Per) struct {\s*})`) + b = ptraceRexexp.ReplaceAll(b, nil) + + // Replace the control_regs union with a blank identifier for now. + controlRegsRegex := regexp.MustCompile(`(Control_regs)\s+\[0\]uint64`) + b = controlRegsRegex.ReplaceAll(b, []byte("_ [0]uint64")) + + // Remove fields that are added by glibc + // Note that this is unstable as the identifers are private. + removeFieldsRegex := regexp.MustCompile(`X__glibc\S*`) + b = removeFieldsRegex.ReplaceAll(b, []byte("_")) + + // Convert [65]int8 to [65]byte in Utsname members to simplify + // conversion to string; see golang.org/issue/20753 + convertUtsnameRegex := regexp.MustCompile(`((Sys|Node|Domain)name|Release|Version|Machine)(\s+)\[(\d+)\]u?int8`) + b = convertUtsnameRegex.ReplaceAll(b, []byte("$1$3[$4]byte")) + + // Convert [1024]int8 to [1024]byte in Ptmget members + convertPtmget := regexp.MustCompile(`([SC]n)(\s+)\[(\d+)\]u?int8`) + b = convertPtmget.ReplaceAll(b, []byte("$1[$3]byte")) + + // Remove spare fields (e.g. in Statx_t) + spareFieldsRegex := regexp.MustCompile(`X__spare\S*`) + b = spareFieldsRegex.ReplaceAll(b, []byte("_")) + + // Remove cgo padding fields + removePaddingFieldsRegex := regexp.MustCompile(`Pad_cgo_\d+`) + b = removePaddingFieldsRegex.ReplaceAll(b, []byte("_")) + + // Remove padding, hidden, or unused fields + removeFieldsRegex = regexp.MustCompile(`\b(X_\S+|Padding)`) + b = removeFieldsRegex.ReplaceAll(b, []byte("_")) + + // Remove the first line of warning from cgo + b = b[bytes.IndexByte(b, '\n')+1:] + // Modify the command in the header to include: + // mkpost, our own warning, and a build tag. + replacement := fmt.Sprintf(`$1 | go run mkpost.go +// Code generated by the command above; see README.md. DO NOT EDIT. + +// +build %s,%s`, goarch, goos) + cgoCommandRegex := regexp.MustCompile(`(cgo -godefs .*)`) + b = cgoCommandRegex.ReplaceAll(b, []byte(replacement)) + + // gofmt + b, err = format.Source(b) + if err != nil { + log.Fatal(err) + } + + os.Stdout.Write(b) +} diff --git a/vendor/golang.org/x/sys/unix/mksyscall.go b/vendor/golang.org/x/sys/unix/mksyscall.go new file mode 100644 index 000000000..e06e4253e --- /dev/null +++ b/vendor/golang.org/x/sys/unix/mksyscall.go @@ -0,0 +1,402 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +/* +This program reads a file containing function prototypes +(like syscall_darwin.go) and generates system call bodies. +The prototypes are marked by lines beginning with "//sys" +and read like func declarations if //sys is replaced by func, but: + * The parameter lists must give a name for each argument. + This includes return parameters. + * The parameter lists must give a type for each argument: + the (x, y, z int) shorthand is not allowed. + * If the return parameter is an error number, it must be named errno. + +A line beginning with //sysnb is like //sys, except that the +goroutine will not be suspended during the execution of the system +call. This must only be used for system calls which can never +block, as otherwise the system call could cause all goroutines to +hang. +*/ +package main + +import ( + "bufio" + "flag" + "fmt" + "os" + "regexp" + "strings" +) + +var ( + b32 = flag.Bool("b32", false, "32bit big-endian") + l32 = flag.Bool("l32", false, "32bit little-endian") + plan9 = flag.Bool("plan9", false, "plan9") + openbsd = flag.Bool("openbsd", false, "openbsd") + netbsd = flag.Bool("netbsd", false, "netbsd") + dragonfly = flag.Bool("dragonfly", false, "dragonfly") + arm = flag.Bool("arm", false, "arm") // 64-bit value should use (even, odd)-pair + tags = flag.String("tags", "", "build tags") + filename = flag.String("output", "", "output file name (standard output if omitted)") +) + +// cmdLine returns this programs's commandline arguments +func cmdLine() string { + return "go run mksyscall.go " + strings.Join(os.Args[1:], " ") +} + +// buildTags returns build tags +func buildTags() string { + return *tags +} + +// Param is function parameter +type Param struct { + Name string + Type string +} + +// usage prints the program usage +func usage() { + fmt.Fprintf(os.Stderr, "usage: go run mksyscall.go [-b32 | -l32] [-tags x,y] [file ...]\n") + os.Exit(1) +} + +// parseParamList parses parameter list and returns a slice of parameters +func parseParamList(list string) []string { + list = strings.TrimSpace(list) + if list == "" { + return []string{} + } + return regexp.MustCompile(`\s*,\s*`).Split(list, -1) +} + +// parseParam splits a parameter into name and type +func parseParam(p string) Param { + ps := regexp.MustCompile(`^(\S*) (\S*)$`).FindStringSubmatch(p) + if ps == nil { + fmt.Fprintf(os.Stderr, "malformed parameter: %s\n", p) + os.Exit(1) + } + return Param{ps[1], ps[2]} +} + +func main() { + // Get the OS and architecture (using GOARCH_TARGET if it exists) + goos := os.Getenv("GOOS") + if goos == "" { + fmt.Fprintln(os.Stderr, "GOOS not defined in environment") + os.Exit(1) + } + goarch := os.Getenv("GOARCH_TARGET") + if goarch == "" { + goarch = os.Getenv("GOARCH") + } + + // Check that we are using the Docker-based build system if we should + if goos == "linux" { + if os.Getenv("GOLANG_SYS_BUILD") != "docker" { + fmt.Fprintf(os.Stderr, "In the Docker-based build system, mksyscall should not be called directly.\n") + fmt.Fprintf(os.Stderr, "See README.md\n") + os.Exit(1) + } + } + + flag.Usage = usage + flag.Parse() + if len(flag.Args()) <= 0 { + fmt.Fprintf(os.Stderr, "no files to parse provided\n") + usage() + } + + endianness := "" + if *b32 { + endianness = "big-endian" + } else if *l32 { + endianness = "little-endian" + } + + libc := false + if goos == "darwin" && strings.Contains(buildTags(), ",go1.12") { + libc = true + } + trampolines := map[string]bool{} + + text := "" + for _, path := range flag.Args() { + file, err := os.Open(path) + if err != nil { + fmt.Fprintf(os.Stderr, err.Error()) + os.Exit(1) + } + s := bufio.NewScanner(file) + for s.Scan() { + t := s.Text() + t = strings.TrimSpace(t) + t = regexp.MustCompile(`\s+`).ReplaceAllString(t, ` `) + nonblock := regexp.MustCompile(`^\/\/sysnb `).FindStringSubmatch(t) + if regexp.MustCompile(`^\/\/sys `).FindStringSubmatch(t) == nil && nonblock == nil { + continue + } + + // Line must be of the form + // func Open(path string, mode int, perm int) (fd int, errno error) + // Split into name, in params, out params. + f := regexp.MustCompile(`^\/\/sys(nb)? (\w+)\(([^()]*)\)\s*(?:\(([^()]+)\))?\s*(?:=\s*((?i)SYS_[A-Z0-9_]+))?$`).FindStringSubmatch(t) + if f == nil { + fmt.Fprintf(os.Stderr, "%s:%s\nmalformed //sys declaration\n", path, t) + os.Exit(1) + } + funct, inps, outps, sysname := f[2], f[3], f[4], f[5] + + // Split argument lists on comma. + in := parseParamList(inps) + out := parseParamList(outps) + + // Try in vain to keep people from editing this file. + // The theory is that they jump into the middle of the file + // without reading the header. + text += "// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT\n\n" + + // Go function header. + outDecl := "" + if len(out) > 0 { + outDecl = fmt.Sprintf(" (%s)", strings.Join(out, ", ")) + } + text += fmt.Sprintf("func %s(%s)%s {\n", funct, strings.Join(in, ", "), outDecl) + + // Check if err return available + errvar := "" + for _, param := range out { + p := parseParam(param) + if p.Type == "error" { + errvar = p.Name + break + } + } + + // Prepare arguments to Syscall. + var args []string + n := 0 + for _, param := range in { + p := parseParam(param) + if regexp.MustCompile(`^\*`).FindStringSubmatch(p.Type) != nil { + args = append(args, "uintptr(unsafe.Pointer("+p.Name+"))") + } else if p.Type == "string" && errvar != "" { + text += fmt.Sprintf("\tvar _p%d *byte\n", n) + text += fmt.Sprintf("\t_p%d, %s = BytePtrFromString(%s)\n", n, errvar, p.Name) + text += fmt.Sprintf("\tif %s != nil {\n\t\treturn\n\t}\n", errvar) + args = append(args, fmt.Sprintf("uintptr(unsafe.Pointer(_p%d))", n)) + n++ + } else if p.Type == "string" { + fmt.Fprintf(os.Stderr, path+":"+funct+" uses string arguments, but has no error return\n") + text += fmt.Sprintf("\tvar _p%d *byte\n", n) + text += fmt.Sprintf("\t_p%d, _ = BytePtrFromString(%s)\n", n, p.Name) + args = append(args, fmt.Sprintf("uintptr(unsafe.Pointer(_p%d))", n)) + n++ + } else if regexp.MustCompile(`^\[\](.*)`).FindStringSubmatch(p.Type) != nil { + // Convert slice into pointer, length. + // Have to be careful not to take address of &a[0] if len == 0: + // pass dummy pointer in that case. + // Used to pass nil, but some OSes or simulators reject write(fd, nil, 0). + text += fmt.Sprintf("\tvar _p%d unsafe.Pointer\n", n) + text += fmt.Sprintf("\tif len(%s) > 0 {\n\t\t_p%d = unsafe.Pointer(&%s[0])\n\t}", p.Name, n, p.Name) + text += fmt.Sprintf(" else {\n\t\t_p%d = unsafe.Pointer(&_zero)\n\t}\n", n) + args = append(args, fmt.Sprintf("uintptr(_p%d)", n), fmt.Sprintf("uintptr(len(%s))", p.Name)) + n++ + } else if p.Type == "int64" && (*openbsd || *netbsd) { + args = append(args, "0") + if endianness == "big-endian" { + args = append(args, fmt.Sprintf("uintptr(%s>>32)", p.Name), fmt.Sprintf("uintptr(%s)", p.Name)) + } else if endianness == "little-endian" { + args = append(args, fmt.Sprintf("uintptr(%s)", p.Name), fmt.Sprintf("uintptr(%s>>32)", p.Name)) + } else { + args = append(args, fmt.Sprintf("uintptr(%s)", p.Name)) + } + } else if p.Type == "int64" && *dragonfly { + if regexp.MustCompile(`^(?i)extp(read|write)`).FindStringSubmatch(funct) == nil { + args = append(args, "0") + } + if endianness == "big-endian" { + args = append(args, fmt.Sprintf("uintptr(%s>>32)", p.Name), fmt.Sprintf("uintptr(%s)", p.Name)) + } else if endianness == "little-endian" { + args = append(args, fmt.Sprintf("uintptr(%s)", p.Name), fmt.Sprintf("uintptr(%s>>32)", p.Name)) + } else { + args = append(args, fmt.Sprintf("uintptr(%s)", p.Name)) + } + } else if p.Type == "int64" && endianness != "" { + if len(args)%2 == 1 && *arm { + // arm abi specifies 64-bit argument uses + // (even, odd) pair + args = append(args, "0") + } + if endianness == "big-endian" { + args = append(args, fmt.Sprintf("uintptr(%s>>32)", p.Name), fmt.Sprintf("uintptr(%s)", p.Name)) + } else { + args = append(args, fmt.Sprintf("uintptr(%s)", p.Name), fmt.Sprintf("uintptr(%s>>32)", p.Name)) + } + } else { + args = append(args, fmt.Sprintf("uintptr(%s)", p.Name)) + } + } + + // Determine which form to use; pad args with zeros. + asm := "Syscall" + if nonblock != nil { + if errvar == "" && goos == "linux" { + asm = "RawSyscallNoError" + } else { + asm = "RawSyscall" + } + } else { + if errvar == "" && goos == "linux" { + asm = "SyscallNoError" + } + } + if len(args) <= 3 { + for len(args) < 3 { + args = append(args, "0") + } + } else if len(args) <= 6 { + asm += "6" + for len(args) < 6 { + args = append(args, "0") + } + } else if len(args) <= 9 { + asm += "9" + for len(args) < 9 { + args = append(args, "0") + } + } else { + fmt.Fprintf(os.Stderr, "%s:%s too many arguments to system call\n", path, funct) + } + + // System call number. + if sysname == "" { + sysname = "SYS_" + funct + sysname = regexp.MustCompile(`([a-z])([A-Z])`).ReplaceAllString(sysname, `${1}_$2`) + sysname = strings.ToUpper(sysname) + } + + var libcFn string + if libc { + asm = "syscall_" + strings.ToLower(asm[:1]) + asm[1:] // internal syscall call + sysname = strings.TrimPrefix(sysname, "SYS_") // remove SYS_ + sysname = strings.ToLower(sysname) // lowercase + if sysname == "getdirentries64" { + // Special case - libSystem name and + // raw syscall name don't match. + sysname = "__getdirentries64" + } + libcFn = sysname + sysname = "funcPC(libc_" + sysname + "_trampoline)" + } + + // Actual call. + arglist := strings.Join(args, ", ") + call := fmt.Sprintf("%s(%s, %s)", asm, sysname, arglist) + + // Assign return values. + body := "" + ret := []string{"_", "_", "_"} + doErrno := false + for i := 0; i < len(out); i++ { + p := parseParam(out[i]) + reg := "" + if p.Name == "err" && !*plan9 { + reg = "e1" + ret[2] = reg + doErrno = true + } else if p.Name == "err" && *plan9 { + ret[0] = "r0" + ret[2] = "e1" + break + } else { + reg = fmt.Sprintf("r%d", i) + ret[i] = reg + } + if p.Type == "bool" { + reg = fmt.Sprintf("%s != 0", reg) + } + if p.Type == "int64" && endianness != "" { + // 64-bit number in r1:r0 or r0:r1. + if i+2 > len(out) { + fmt.Fprintf(os.Stderr, "%s:%s not enough registers for int64 return\n", path, funct) + } + if endianness == "big-endian" { + reg = fmt.Sprintf("int64(r%d)<<32 | int64(r%d)", i, i+1) + } else { + reg = fmt.Sprintf("int64(r%d)<<32 | int64(r%d)", i+1, i) + } + ret[i] = fmt.Sprintf("r%d", i) + ret[i+1] = fmt.Sprintf("r%d", i+1) + } + if reg != "e1" || *plan9 { + body += fmt.Sprintf("\t%s = %s(%s)\n", p.Name, p.Type, reg) + } + } + if ret[0] == "_" && ret[1] == "_" && ret[2] == "_" { + text += fmt.Sprintf("\t%s\n", call) + } else { + if errvar == "" && goos == "linux" { + // raw syscall without error on Linux, see golang.org/issue/22924 + text += fmt.Sprintf("\t%s, %s := %s\n", ret[0], ret[1], call) + } else { + text += fmt.Sprintf("\t%s, %s, %s := %s\n", ret[0], ret[1], ret[2], call) + } + } + text += body + + if *plan9 && ret[2] == "e1" { + text += "\tif int32(r0) == -1 {\n" + text += "\t\terr = e1\n" + text += "\t}\n" + } else if doErrno { + text += "\tif e1 != 0 {\n" + text += "\t\terr = errnoErr(e1)\n" + text += "\t}\n" + } + text += "\treturn\n" + text += "}\n\n" + + if libc && !trampolines[libcFn] { + // some system calls share a trampoline, like read and readlen. + trampolines[libcFn] = true + // Declare assembly trampoline. + text += fmt.Sprintf("func libc_%s_trampoline()\n", libcFn) + // Assembly trampoline calls the libc_* function, which this magic + // redirects to use the function from libSystem. + text += fmt.Sprintf("//go:linkname libc_%s libc_%s\n", libcFn, libcFn) + text += fmt.Sprintf("//go:cgo_import_dynamic libc_%s %s \"/usr/lib/libSystem.B.dylib\"\n", libcFn, libcFn) + text += "\n" + } + } + if err := s.Err(); err != nil { + fmt.Fprintf(os.Stderr, err.Error()) + os.Exit(1) + } + file.Close() + } + fmt.Printf(srcTemplate, cmdLine(), buildTags(), text) +} + +const srcTemplate = `// %s +// Code generated by the command above; see README.md. DO NOT EDIT. + +// +build %s + +package unix + +import ( + "syscall" + "unsafe" +) + +var _ syscall.Errno + +%s +` diff --git a/vendor/golang.org/x/sys/unix/mksyscall_aix_ppc.go b/vendor/golang.org/x/sys/unix/mksyscall_aix_ppc.go new file mode 100644 index 000000000..f2c58fb7c --- /dev/null +++ b/vendor/golang.org/x/sys/unix/mksyscall_aix_ppc.go @@ -0,0 +1,404 @@ +// Copyright 2019 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +/* +This program reads a file containing function prototypes +(like syscall_aix.go) and generates system call bodies. +The prototypes are marked by lines beginning with "//sys" +and read like func declarations if //sys is replaced by func, but: + * The parameter lists must give a name for each argument. + This includes return parameters. + * The parameter lists must give a type for each argument: + the (x, y, z int) shorthand is not allowed. + * If the return parameter is an error number, it must be named err. + * If go func name needs to be different than its libc name, + * or the function is not in libc, name could be specified + * at the end, after "=" sign, like + //sys getsockopt(s int, level int, name int, val uintptr, vallen *_Socklen) (err error) = libsocket.getsockopt +*/ +package main + +import ( + "bufio" + "flag" + "fmt" + "os" + "regexp" + "strings" +) + +var ( + b32 = flag.Bool("b32", false, "32bit big-endian") + l32 = flag.Bool("l32", false, "32bit little-endian") + aix = flag.Bool("aix", false, "aix") + tags = flag.String("tags", "", "build tags") +) + +// cmdLine returns this programs's commandline arguments +func cmdLine() string { + return "go run mksyscall_aix_ppc.go " + strings.Join(os.Args[1:], " ") +} + +// buildTags returns build tags +func buildTags() string { + return *tags +} + +// Param is function parameter +type Param struct { + Name string + Type string +} + +// usage prints the program usage +func usage() { + fmt.Fprintf(os.Stderr, "usage: go run mksyscall_aix_ppc.go [-b32 | -l32] [-tags x,y] [file ...]\n") + os.Exit(1) +} + +// parseParamList parses parameter list and returns a slice of parameters +func parseParamList(list string) []string { + list = strings.TrimSpace(list) + if list == "" { + return []string{} + } + return regexp.MustCompile(`\s*,\s*`).Split(list, -1) +} + +// parseParam splits a parameter into name and type +func parseParam(p string) Param { + ps := regexp.MustCompile(`^(\S*) (\S*)$`).FindStringSubmatch(p) + if ps == nil { + fmt.Fprintf(os.Stderr, "malformed parameter: %s\n", p) + os.Exit(1) + } + return Param{ps[1], ps[2]} +} + +func main() { + flag.Usage = usage + flag.Parse() + if len(flag.Args()) <= 0 { + fmt.Fprintf(os.Stderr, "no files to parse provided\n") + usage() + } + + endianness := "" + if *b32 { + endianness = "big-endian" + } else if *l32 { + endianness = "little-endian" + } + + pack := "" + text := "" + cExtern := "/*\n#include \n#include \n" + for _, path := range flag.Args() { + file, err := os.Open(path) + if err != nil { + fmt.Fprintf(os.Stderr, err.Error()) + os.Exit(1) + } + s := bufio.NewScanner(file) + for s.Scan() { + t := s.Text() + t = strings.TrimSpace(t) + t = regexp.MustCompile(`\s+`).ReplaceAllString(t, ` `) + if p := regexp.MustCompile(`^package (\S+)$`).FindStringSubmatch(t); p != nil && pack == "" { + pack = p[1] + } + nonblock := regexp.MustCompile(`^\/\/sysnb `).FindStringSubmatch(t) + if regexp.MustCompile(`^\/\/sys `).FindStringSubmatch(t) == nil && nonblock == nil { + continue + } + + // Line must be of the form + // func Open(path string, mode int, perm int) (fd int, err error) + // Split into name, in params, out params. + f := regexp.MustCompile(`^\/\/sys(nb)? (\w+)\(([^()]*)\)\s*(?:\(([^()]+)\))?\s*(?:=\s*(?:(\w*)\.)?(\w*))?$`).FindStringSubmatch(t) + if f == nil { + fmt.Fprintf(os.Stderr, "%s:%s\nmalformed //sys declaration\n", path, t) + os.Exit(1) + } + funct, inps, outps, modname, sysname := f[2], f[3], f[4], f[5], f[6] + + // Split argument lists on comma. + in := parseParamList(inps) + out := parseParamList(outps) + + inps = strings.Join(in, ", ") + outps = strings.Join(out, ", ") + + // Try in vain to keep people from editing this file. + // The theory is that they jump into the middle of the file + // without reading the header. + text += "// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT\n\n" + + // Check if value return, err return available + errvar := "" + retvar := "" + rettype := "" + for _, param := range out { + p := parseParam(param) + if p.Type == "error" { + errvar = p.Name + } else { + retvar = p.Name + rettype = p.Type + } + } + + // System call name. + if sysname == "" { + sysname = funct + } + sysname = regexp.MustCompile(`([a-z])([A-Z])`).ReplaceAllString(sysname, `${1}_$2`) + sysname = strings.ToLower(sysname) // All libc functions are lowercase. + + cRettype := "" + if rettype == "unsafe.Pointer" { + cRettype = "uintptr_t" + } else if rettype == "uintptr" { + cRettype = "uintptr_t" + } else if regexp.MustCompile(`^_`).FindStringSubmatch(rettype) != nil { + cRettype = "uintptr_t" + } else if rettype == "int" { + cRettype = "int" + } else if rettype == "int32" { + cRettype = "int" + } else if rettype == "int64" { + cRettype = "long long" + } else if rettype == "uint32" { + cRettype = "unsigned int" + } else if rettype == "uint64" { + cRettype = "unsigned long long" + } else { + cRettype = "int" + } + if sysname == "exit" { + cRettype = "void" + } + + // Change p.Types to c + var cIn []string + for _, param := range in { + p := parseParam(param) + if regexp.MustCompile(`^\*`).FindStringSubmatch(p.Type) != nil { + cIn = append(cIn, "uintptr_t") + } else if p.Type == "string" { + cIn = append(cIn, "uintptr_t") + } else if regexp.MustCompile(`^\[\](.*)`).FindStringSubmatch(p.Type) != nil { + cIn = append(cIn, "uintptr_t", "size_t") + } else if p.Type == "unsafe.Pointer" { + cIn = append(cIn, "uintptr_t") + } else if p.Type == "uintptr" { + cIn = append(cIn, "uintptr_t") + } else if regexp.MustCompile(`^_`).FindStringSubmatch(p.Type) != nil { + cIn = append(cIn, "uintptr_t") + } else if p.Type == "int" { + cIn = append(cIn, "int") + } else if p.Type == "int32" { + cIn = append(cIn, "int") + } else if p.Type == "int64" { + cIn = append(cIn, "long long") + } else if p.Type == "uint32" { + cIn = append(cIn, "unsigned int") + } else if p.Type == "uint64" { + cIn = append(cIn, "unsigned long long") + } else { + cIn = append(cIn, "int") + } + } + + if funct != "fcntl" && funct != "FcntlInt" && funct != "readlen" && funct != "writelen" { + // Imports of system calls from libc + cExtern += fmt.Sprintf("%s %s", cRettype, sysname) + cIn := strings.Join(cIn, ", ") + cExtern += fmt.Sprintf("(%s);\n", cIn) + } + + // So file name. + if *aix { + if modname == "" { + modname = "libc.a/shr_64.o" + } else { + fmt.Fprintf(os.Stderr, "%s: only syscall using libc are available\n", funct) + os.Exit(1) + } + } + + strconvfunc := "C.CString" + + // Go function header. + if outps != "" { + outps = fmt.Sprintf(" (%s)", outps) + } + if text != "" { + text += "\n" + } + + text += fmt.Sprintf("func %s(%s)%s {\n", funct, strings.Join(in, ", "), outps) + + // Prepare arguments to Syscall. + var args []string + n := 0 + argN := 0 + for _, param := range in { + p := parseParam(param) + if regexp.MustCompile(`^\*`).FindStringSubmatch(p.Type) != nil { + args = append(args, "C.uintptr_t(uintptr(unsafe.Pointer("+p.Name+")))") + } else if p.Type == "string" && errvar != "" { + text += fmt.Sprintf("\t_p%d := uintptr(unsafe.Pointer(%s(%s)))\n", n, strconvfunc, p.Name) + args = append(args, fmt.Sprintf("C.uintptr_t(_p%d)", n)) + n++ + } else if p.Type == "string" { + fmt.Fprintf(os.Stderr, path+":"+funct+" uses string arguments, but has no error return\n") + text += fmt.Sprintf("\t_p%d := uintptr(unsafe.Pointer(%s(%s)))\n", n, strconvfunc, p.Name) + args = append(args, fmt.Sprintf("C.uintptr_t(_p%d)", n)) + n++ + } else if m := regexp.MustCompile(`^\[\](.*)`).FindStringSubmatch(p.Type); m != nil { + // Convert slice into pointer, length. + // Have to be careful not to take address of &a[0] if len == 0: + // pass nil in that case. + text += fmt.Sprintf("\tvar _p%d *%s\n", n, m[1]) + text += fmt.Sprintf("\tif len(%s) > 0 {\n\t\t_p%d = &%s[0]\n\t}\n", p.Name, n, p.Name) + args = append(args, fmt.Sprintf("C.uintptr_t(uintptr(unsafe.Pointer(_p%d)))", n)) + n++ + text += fmt.Sprintf("\tvar _p%d int\n", n) + text += fmt.Sprintf("\t_p%d = len(%s)\n", n, p.Name) + args = append(args, fmt.Sprintf("C.size_t(_p%d)", n)) + n++ + } else if p.Type == "int64" && endianness != "" { + if endianness == "big-endian" { + args = append(args, fmt.Sprintf("uintptr(%s>>32)", p.Name), fmt.Sprintf("uintptr(%s)", p.Name)) + } else { + args = append(args, fmt.Sprintf("uintptr(%s)", p.Name), fmt.Sprintf("uintptr(%s>>32)", p.Name)) + } + n++ + } else if p.Type == "bool" { + text += fmt.Sprintf("\tvar _p%d uint32\n", n) + text += fmt.Sprintf("\tif %s {\n\t\t_p%d = 1\n\t} else {\n\t\t_p%d = 0\n\t}\n", p.Name, n, n) + args = append(args, fmt.Sprintf("_p%d", n)) + } else if regexp.MustCompile(`^_`).FindStringSubmatch(p.Type) != nil { + args = append(args, fmt.Sprintf("C.uintptr_t(uintptr(%s))", p.Name)) + } else if p.Type == "unsafe.Pointer" { + args = append(args, fmt.Sprintf("C.uintptr_t(uintptr(%s))", p.Name)) + } else if p.Type == "int" { + if (argN == 2) && ((funct == "readlen") || (funct == "writelen")) { + args = append(args, fmt.Sprintf("C.size_t(%s)", p.Name)) + } else if argN == 0 && funct == "fcntl" { + args = append(args, fmt.Sprintf("C.uintptr_t(%s)", p.Name)) + } else if (argN == 2) && ((funct == "fcntl") || (funct == "FcntlInt")) { + args = append(args, fmt.Sprintf("C.uintptr_t(%s)", p.Name)) + } else { + args = append(args, fmt.Sprintf("C.int(%s)", p.Name)) + } + } else if p.Type == "int32" { + args = append(args, fmt.Sprintf("C.int(%s)", p.Name)) + } else if p.Type == "int64" { + args = append(args, fmt.Sprintf("C.longlong(%s)", p.Name)) + } else if p.Type == "uint32" { + args = append(args, fmt.Sprintf("C.uint(%s)", p.Name)) + } else if p.Type == "uint64" { + args = append(args, fmt.Sprintf("C.ulonglong(%s)", p.Name)) + } else if p.Type == "uintptr" { + args = append(args, fmt.Sprintf("C.uintptr_t(%s)", p.Name)) + } else { + args = append(args, fmt.Sprintf("C.int(%s)", p.Name)) + } + argN++ + } + + // Actual call. + arglist := strings.Join(args, ", ") + call := "" + if sysname == "exit" { + if errvar != "" { + call += "er :=" + } else { + call += "" + } + } else if errvar != "" { + call += "r0,er :=" + } else if retvar != "" { + call += "r0,_ :=" + } else { + call += "" + } + call += fmt.Sprintf("C.%s(%s)", sysname, arglist) + + // Assign return values. + body := "" + for i := 0; i < len(out); i++ { + p := parseParam(out[i]) + reg := "" + if p.Name == "err" { + reg = "e1" + } else { + reg = "r0" + } + if reg != "e1" { + body += fmt.Sprintf("\t%s = %s(%s)\n", p.Name, p.Type, reg) + } + } + + // verify return + if sysname != "exit" && errvar != "" { + if regexp.MustCompile(`^uintptr`).FindStringSubmatch(cRettype) != nil { + body += "\tif (uintptr(r0) ==^uintptr(0) && er != nil) {\n" + body += fmt.Sprintf("\t\t%s = er\n", errvar) + body += "\t}\n" + } else { + body += "\tif (r0 ==-1 && er != nil) {\n" + body += fmt.Sprintf("\t\t%s = er\n", errvar) + body += "\t}\n" + } + } else if errvar != "" { + body += "\tif (er != nil) {\n" + body += fmt.Sprintf("\t\t%s = er\n", errvar) + body += "\t}\n" + } + + text += fmt.Sprintf("\t%s\n", call) + text += body + + text += "\treturn\n" + text += "}\n" + } + if err := s.Err(); err != nil { + fmt.Fprintf(os.Stderr, err.Error()) + os.Exit(1) + } + file.Close() + } + imp := "" + if pack != "unix" { + imp = "import \"golang.org/x/sys/unix\"\n" + + } + fmt.Printf(srcTemplate, cmdLine(), buildTags(), pack, cExtern, imp, text) +} + +const srcTemplate = `// %s +// Code generated by the command above; see README.md. DO NOT EDIT. + +// +build %s + +package %s + + +%s +*/ +import "C" +import ( + "unsafe" +) + + +%s + +%s +` diff --git a/vendor/golang.org/x/sys/unix/mksyscall_aix_ppc64.go b/vendor/golang.org/x/sys/unix/mksyscall_aix_ppc64.go new file mode 100644 index 000000000..45b442908 --- /dev/null +++ b/vendor/golang.org/x/sys/unix/mksyscall_aix_ppc64.go @@ -0,0 +1,602 @@ +// Copyright 2019 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +/* +This program reads a file containing function prototypes +(like syscall_aix.go) and generates system call bodies. +The prototypes are marked by lines beginning with "//sys" +and read like func declarations if //sys is replaced by func, but: + * The parameter lists must give a name for each argument. + This includes return parameters. + * The parameter lists must give a type for each argument: + the (x, y, z int) shorthand is not allowed. + * If the return parameter is an error number, it must be named err. + * If go func name needs to be different than its libc name, + * or the function is not in libc, name could be specified + * at the end, after "=" sign, like + //sys getsockopt(s int, level int, name int, val uintptr, vallen *_Socklen) (err error) = libsocket.getsockopt + + +This program will generate three files and handle both gc and gccgo implementation: + - zsyscall_aix_ppc64.go: the common part of each implementation (error handler, pointer creation) + - zsyscall_aix_ppc64_gc.go: gc part with //go_cgo_import_dynamic and a call to syscall6 + - zsyscall_aix_ppc64_gccgo.go: gccgo part with C function and conversion to C type. + + The generated code looks like this + +zsyscall_aix_ppc64.go +func asyscall(...) (n int, err error) { + // Pointer Creation + r1, e1 := callasyscall(...) + // Type Conversion + // Error Handler + return +} + +zsyscall_aix_ppc64_gc.go +//go:cgo_import_dynamic libc_asyscall asyscall "libc.a/shr_64.o" +//go:linkname libc_asyscall libc_asyscall +var asyscall syscallFunc + +func callasyscall(...) (r1 uintptr, e1 Errno) { + r1, _, e1 = syscall6(uintptr(unsafe.Pointer(&libc_asyscall)), "nb_args", ... ) + return +} + +zsyscall_aix_ppc64_ggcgo.go + +// int asyscall(...) + +import "C" + +func callasyscall(...) (r1 uintptr, e1 Errno) { + r1 = uintptr(C.asyscall(...)) + e1 = syscall.GetErrno() + return +} +*/ + +package main + +import ( + "bufio" + "flag" + "fmt" + "io/ioutil" + "os" + "regexp" + "strings" +) + +var ( + b32 = flag.Bool("b32", false, "32bit big-endian") + l32 = flag.Bool("l32", false, "32bit little-endian") + aix = flag.Bool("aix", false, "aix") + tags = flag.String("tags", "", "build tags") +) + +// cmdLine returns this programs's commandline arguments +func cmdLine() string { + return "go run mksyscall_aix_ppc64.go " + strings.Join(os.Args[1:], " ") +} + +// buildTags returns build tags +func buildTags() string { + return *tags +} + +// Param is function parameter +type Param struct { + Name string + Type string +} + +// usage prints the program usage +func usage() { + fmt.Fprintf(os.Stderr, "usage: go run mksyscall_aix_ppc64.go [-b32 | -l32] [-tags x,y] [file ...]\n") + os.Exit(1) +} + +// parseParamList parses parameter list and returns a slice of parameters +func parseParamList(list string) []string { + list = strings.TrimSpace(list) + if list == "" { + return []string{} + } + return regexp.MustCompile(`\s*,\s*`).Split(list, -1) +} + +// parseParam splits a parameter into name and type +func parseParam(p string) Param { + ps := regexp.MustCompile(`^(\S*) (\S*)$`).FindStringSubmatch(p) + if ps == nil { + fmt.Fprintf(os.Stderr, "malformed parameter: %s\n", p) + os.Exit(1) + } + return Param{ps[1], ps[2]} +} + +func main() { + flag.Usage = usage + flag.Parse() + if len(flag.Args()) <= 0 { + fmt.Fprintf(os.Stderr, "no files to parse provided\n") + usage() + } + + endianness := "" + if *b32 { + endianness = "big-endian" + } else if *l32 { + endianness = "little-endian" + } + + pack := "" + // GCCGO + textgccgo := "" + cExtern := "/*\n#include \n" + // GC + textgc := "" + dynimports := "" + linknames := "" + var vars []string + // COMMON + textcommon := "" + for _, path := range flag.Args() { + file, err := os.Open(path) + if err != nil { + fmt.Fprintf(os.Stderr, err.Error()) + os.Exit(1) + } + s := bufio.NewScanner(file) + for s.Scan() { + t := s.Text() + t = strings.TrimSpace(t) + t = regexp.MustCompile(`\s+`).ReplaceAllString(t, ` `) + if p := regexp.MustCompile(`^package (\S+)$`).FindStringSubmatch(t); p != nil && pack == "" { + pack = p[1] + } + nonblock := regexp.MustCompile(`^\/\/sysnb `).FindStringSubmatch(t) + if regexp.MustCompile(`^\/\/sys `).FindStringSubmatch(t) == nil && nonblock == nil { + continue + } + + // Line must be of the form + // func Open(path string, mode int, perm int) (fd int, err error) + // Split into name, in params, out params. + f := regexp.MustCompile(`^\/\/sys(nb)? (\w+)\(([^()]*)\)\s*(?:\(([^()]+)\))?\s*(?:=\s*(?:(\w*)\.)?(\w*))?$`).FindStringSubmatch(t) + if f == nil { + fmt.Fprintf(os.Stderr, "%s:%s\nmalformed //sys declaration\n", path, t) + os.Exit(1) + } + funct, inps, outps, modname, sysname := f[2], f[3], f[4], f[5], f[6] + + // Split argument lists on comma. + in := parseParamList(inps) + out := parseParamList(outps) + + inps = strings.Join(in, ", ") + outps = strings.Join(out, ", ") + + if sysname == "" { + sysname = funct + } + + onlyCommon := false + if funct == "readlen" || funct == "writelen" || funct == "FcntlInt" || funct == "FcntlFlock" { + // This function call another syscall which is already implemented. + // Therefore, the gc and gccgo part must not be generated. + onlyCommon = true + } + + // Try in vain to keep people from editing this file. + // The theory is that they jump into the middle of the file + // without reading the header. + + textcommon += "// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT\n\n" + if !onlyCommon { + textgccgo += "// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT\n\n" + textgc += "// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT\n\n" + } + + // Check if value return, err return available + errvar := "" + rettype := "" + for _, param := range out { + p := parseParam(param) + if p.Type == "error" { + errvar = p.Name + } else { + rettype = p.Type + } + } + + sysname = regexp.MustCompile(`([a-z])([A-Z])`).ReplaceAllString(sysname, `${1}_$2`) + sysname = strings.ToLower(sysname) // All libc functions are lowercase. + + // GCCGO Prototype return type + cRettype := "" + if rettype == "unsafe.Pointer" { + cRettype = "uintptr_t" + } else if rettype == "uintptr" { + cRettype = "uintptr_t" + } else if regexp.MustCompile(`^_`).FindStringSubmatch(rettype) != nil { + cRettype = "uintptr_t" + } else if rettype == "int" { + cRettype = "int" + } else if rettype == "int32" { + cRettype = "int" + } else if rettype == "int64" { + cRettype = "long long" + } else if rettype == "uint32" { + cRettype = "unsigned int" + } else if rettype == "uint64" { + cRettype = "unsigned long long" + } else { + cRettype = "int" + } + if sysname == "exit" { + cRettype = "void" + } + + // GCCGO Prototype arguments type + var cIn []string + for i, param := range in { + p := parseParam(param) + if regexp.MustCompile(`^\*`).FindStringSubmatch(p.Type) != nil { + cIn = append(cIn, "uintptr_t") + } else if p.Type == "string" { + cIn = append(cIn, "uintptr_t") + } else if regexp.MustCompile(`^\[\](.*)`).FindStringSubmatch(p.Type) != nil { + cIn = append(cIn, "uintptr_t", "size_t") + } else if p.Type == "unsafe.Pointer" { + cIn = append(cIn, "uintptr_t") + } else if p.Type == "uintptr" { + cIn = append(cIn, "uintptr_t") + } else if regexp.MustCompile(`^_`).FindStringSubmatch(p.Type) != nil { + cIn = append(cIn, "uintptr_t") + } else if p.Type == "int" { + if (i == 0 || i == 2) && funct == "fcntl" { + // These fcntl arguments needs to be uintptr to be able to call FcntlInt and FcntlFlock + cIn = append(cIn, "uintptr_t") + } else { + cIn = append(cIn, "int") + } + + } else if p.Type == "int32" { + cIn = append(cIn, "int") + } else if p.Type == "int64" { + cIn = append(cIn, "long long") + } else if p.Type == "uint32" { + cIn = append(cIn, "unsigned int") + } else if p.Type == "uint64" { + cIn = append(cIn, "unsigned long long") + } else { + cIn = append(cIn, "int") + } + } + + if !onlyCommon { + // GCCGO Prototype Generation + // Imports of system calls from libc + cExtern += fmt.Sprintf("%s %s", cRettype, sysname) + cIn := strings.Join(cIn, ", ") + cExtern += fmt.Sprintf("(%s);\n", cIn) + } + // GC Library name + if modname == "" { + modname = "libc.a/shr_64.o" + } else { + fmt.Fprintf(os.Stderr, "%s: only syscall using libc are available\n", funct) + os.Exit(1) + } + sysvarname := fmt.Sprintf("libc_%s", sysname) + + if !onlyCommon { + // GC Runtime import of function to allow cross-platform builds. + dynimports += fmt.Sprintf("//go:cgo_import_dynamic %s %s \"%s\"\n", sysvarname, sysname, modname) + // GC Link symbol to proc address variable. + linknames += fmt.Sprintf("//go:linkname %s %s\n", sysvarname, sysvarname) + // GC Library proc address variable. + vars = append(vars, sysvarname) + } + + strconvfunc := "BytePtrFromString" + strconvtype := "*byte" + + // Go function header. + if outps != "" { + outps = fmt.Sprintf(" (%s)", outps) + } + if textcommon != "" { + textcommon += "\n" + } + + textcommon += fmt.Sprintf("func %s(%s)%s {\n", funct, strings.Join(in, ", "), outps) + + // Prepare arguments tocall. + var argscommon []string // Arguments in the common part + var argscall []string // Arguments for call prototype + var argsgc []string // Arguments for gc call (with syscall6) + var argsgccgo []string // Arguments for gccgo call (with C.name_of_syscall) + n := 0 + argN := 0 + for _, param := range in { + p := parseParam(param) + if regexp.MustCompile(`^\*`).FindStringSubmatch(p.Type) != nil { + argscommon = append(argscommon, fmt.Sprintf("uintptr(unsafe.Pointer(%s))", p.Name)) + argscall = append(argscall, fmt.Sprintf("%s uintptr", p.Name)) + argsgc = append(argsgc, p.Name) + argsgccgo = append(argsgccgo, fmt.Sprintf("C.uintptr_t(%s)", p.Name)) + } else if p.Type == "string" && errvar != "" { + textcommon += fmt.Sprintf("\tvar _p%d %s\n", n, strconvtype) + textcommon += fmt.Sprintf("\t_p%d, %s = %s(%s)\n", n, errvar, strconvfunc, p.Name) + textcommon += fmt.Sprintf("\tif %s != nil {\n\t\treturn\n\t}\n", errvar) + + argscommon = append(argscommon, fmt.Sprintf("uintptr(unsafe.Pointer(_p%d))", n)) + argscall = append(argscall, fmt.Sprintf("_p%d uintptr ", n)) + argsgc = append(argsgc, fmt.Sprintf("_p%d", n)) + argsgccgo = append(argsgccgo, fmt.Sprintf("C.uintptr_t(_p%d)", n)) + n++ + } else if p.Type == "string" { + fmt.Fprintf(os.Stderr, path+":"+funct+" uses string arguments, but has no error return\n") + textcommon += fmt.Sprintf("\tvar _p%d %s\n", n, strconvtype) + textcommon += fmt.Sprintf("\t_p%d, %s = %s(%s)\n", n, errvar, strconvfunc, p.Name) + textcommon += fmt.Sprintf("\tif %s != nil {\n\t\treturn\n\t}\n", errvar) + + argscommon = append(argscommon, fmt.Sprintf("uintptr(unsafe.Pointer(_p%d))", n)) + argscall = append(argscall, fmt.Sprintf("_p%d uintptr", n)) + argsgc = append(argsgc, fmt.Sprintf("_p%d", n)) + argsgccgo = append(argsgccgo, fmt.Sprintf("C.uintptr_t(_p%d)", n)) + n++ + } else if m := regexp.MustCompile(`^\[\](.*)`).FindStringSubmatch(p.Type); m != nil { + // Convert slice into pointer, length. + // Have to be careful not to take address of &a[0] if len == 0: + // pass nil in that case. + textcommon += fmt.Sprintf("\tvar _p%d *%s\n", n, m[1]) + textcommon += fmt.Sprintf("\tif len(%s) > 0 {\n\t\t_p%d = &%s[0]\n\t}\n", p.Name, n, p.Name) + argscommon = append(argscommon, fmt.Sprintf("uintptr(unsafe.Pointer(_p%d))", n), fmt.Sprintf("len(%s)", p.Name)) + argscall = append(argscall, fmt.Sprintf("_p%d uintptr", n), fmt.Sprintf("_lenp%d int", n)) + argsgc = append(argsgc, fmt.Sprintf("_p%d", n), fmt.Sprintf("uintptr(_lenp%d)", n)) + argsgccgo = append(argsgccgo, fmt.Sprintf("C.uintptr_t(_p%d)", n), fmt.Sprintf("C.size_t(_lenp%d)", n)) + n++ + } else if p.Type == "int64" && endianness != "" { + fmt.Fprintf(os.Stderr, path+":"+funct+" uses int64 with 32 bits mode. Case not yet implemented\n") + } else if p.Type == "bool" { + fmt.Fprintf(os.Stderr, path+":"+funct+" uses bool. Case not yet implemented\n") + } else if regexp.MustCompile(`^_`).FindStringSubmatch(p.Type) != nil || p.Type == "unsafe.Pointer" { + argscommon = append(argscommon, fmt.Sprintf("uintptr(%s)", p.Name)) + argscall = append(argscall, fmt.Sprintf("%s uintptr", p.Name)) + argsgc = append(argsgc, p.Name) + argsgccgo = append(argsgccgo, fmt.Sprintf("C.uintptr_t(%s)", p.Name)) + } else if p.Type == "int" { + if (argN == 0 || argN == 2) && ((funct == "fcntl") || (funct == "FcntlInt") || (funct == "FcntlFlock")) { + // These fcntl arguments need to be uintptr to be able to call FcntlInt and FcntlFlock + argscommon = append(argscommon, fmt.Sprintf("uintptr(%s)", p.Name)) + argscall = append(argscall, fmt.Sprintf("%s uintptr", p.Name)) + argsgc = append(argsgc, p.Name) + argsgccgo = append(argsgccgo, fmt.Sprintf("C.uintptr_t(%s)", p.Name)) + + } else { + argscommon = append(argscommon, p.Name) + argscall = append(argscall, fmt.Sprintf("%s int", p.Name)) + argsgc = append(argsgc, fmt.Sprintf("uintptr(%s)", p.Name)) + argsgccgo = append(argsgccgo, fmt.Sprintf("C.int(%s)", p.Name)) + } + } else if p.Type == "int32" { + argscommon = append(argscommon, p.Name) + argscall = append(argscall, fmt.Sprintf("%s int32", p.Name)) + argsgc = append(argsgc, fmt.Sprintf("uintptr(%s)", p.Name)) + argsgccgo = append(argsgccgo, fmt.Sprintf("C.int(%s)", p.Name)) + } else if p.Type == "int64" { + argscommon = append(argscommon, p.Name) + argscall = append(argscall, fmt.Sprintf("%s int64", p.Name)) + argsgc = append(argsgc, fmt.Sprintf("uintptr(%s)", p.Name)) + argsgccgo = append(argsgccgo, fmt.Sprintf("C.longlong(%s)", p.Name)) + } else if p.Type == "uint32" { + argscommon = append(argscommon, p.Name) + argscall = append(argscall, fmt.Sprintf("%s uint32", p.Name)) + argsgc = append(argsgc, fmt.Sprintf("uintptr(%s)", p.Name)) + argsgccgo = append(argsgccgo, fmt.Sprintf("C.uint(%s)", p.Name)) + } else if p.Type == "uint64" { + argscommon = append(argscommon, p.Name) + argscall = append(argscall, fmt.Sprintf("%s uint64", p.Name)) + argsgc = append(argsgc, fmt.Sprintf("uintptr(%s)", p.Name)) + argsgccgo = append(argsgccgo, fmt.Sprintf("C.ulonglong(%s)", p.Name)) + } else if p.Type == "uintptr" { + argscommon = append(argscommon, p.Name) + argscall = append(argscall, fmt.Sprintf("%s uintptr", p.Name)) + argsgc = append(argsgc, p.Name) + argsgccgo = append(argsgccgo, fmt.Sprintf("C.uintptr_t(%s)", p.Name)) + } else { + argscommon = append(argscommon, fmt.Sprintf("int(%s)", p.Name)) + argscall = append(argscall, fmt.Sprintf("%s int", p.Name)) + argsgc = append(argsgc, fmt.Sprintf("uintptr(%s)", p.Name)) + argsgccgo = append(argsgccgo, fmt.Sprintf("C.int(%s)", p.Name)) + } + argN++ + } + nargs := len(argsgc) + + // COMMON function generation + argscommonlist := strings.Join(argscommon, ", ") + callcommon := fmt.Sprintf("call%s(%s)", sysname, argscommonlist) + ret := []string{"_", "_"} + body := "" + doErrno := false + for i := 0; i < len(out); i++ { + p := parseParam(out[i]) + reg := "" + if p.Name == "err" { + reg = "e1" + ret[1] = reg + doErrno = true + } else { + reg = "r0" + ret[0] = reg + } + if p.Type == "bool" { + reg = fmt.Sprintf("%s != 0", reg) + } + if reg != "e1" { + body += fmt.Sprintf("\t%s = %s(%s)\n", p.Name, p.Type, reg) + } + } + if ret[0] == "_" && ret[1] == "_" { + textcommon += fmt.Sprintf("\t%s\n", callcommon) + } else { + textcommon += fmt.Sprintf("\t%s, %s := %s\n", ret[0], ret[1], callcommon) + } + textcommon += body + + if doErrno { + textcommon += "\tif e1 != 0 {\n" + textcommon += "\t\terr = errnoErr(e1)\n" + textcommon += "\t}\n" + } + textcommon += "\treturn\n" + textcommon += "}\n" + + if onlyCommon { + continue + } + + // CALL Prototype + callProto := fmt.Sprintf("func call%s(%s) (r1 uintptr, e1 Errno) {\n", sysname, strings.Join(argscall, ", ")) + + // GC function generation + asm := "syscall6" + if nonblock != nil { + asm = "rawSyscall6" + } + + if len(argsgc) <= 6 { + for len(argsgc) < 6 { + argsgc = append(argsgc, "0") + } + } else { + fmt.Fprintf(os.Stderr, "%s: too many arguments to system call", funct) + os.Exit(1) + } + argsgclist := strings.Join(argsgc, ", ") + callgc := fmt.Sprintf("%s(uintptr(unsafe.Pointer(&%s)), %d, %s)", asm, sysvarname, nargs, argsgclist) + + textgc += callProto + textgc += fmt.Sprintf("\tr1, _, e1 = %s\n", callgc) + textgc += "\treturn\n}\n" + + // GCCGO function generation + argsgccgolist := strings.Join(argsgccgo, ", ") + callgccgo := fmt.Sprintf("C.%s(%s)", sysname, argsgccgolist) + textgccgo += callProto + textgccgo += fmt.Sprintf("\tr1 = uintptr(%s)\n", callgccgo) + textgccgo += "\te1 = syscall.GetErrno()\n" + textgccgo += "\treturn\n}\n" + } + if err := s.Err(); err != nil { + fmt.Fprintf(os.Stderr, err.Error()) + os.Exit(1) + } + file.Close() + } + imp := "" + if pack != "unix" { + imp = "import \"golang.org/x/sys/unix\"\n" + + } + + // Print zsyscall_aix_ppc64.go + err := ioutil.WriteFile("zsyscall_aix_ppc64.go", + []byte(fmt.Sprintf(srcTemplate1, cmdLine(), buildTags(), pack, imp, textcommon)), + 0644) + if err != nil { + fmt.Fprintf(os.Stderr, err.Error()) + os.Exit(1) + } + + // Print zsyscall_aix_ppc64_gc.go + vardecls := "\t" + strings.Join(vars, ",\n\t") + vardecls += " syscallFunc" + err = ioutil.WriteFile("zsyscall_aix_ppc64_gc.go", + []byte(fmt.Sprintf(srcTemplate2, cmdLine(), buildTags(), pack, imp, dynimports, linknames, vardecls, textgc)), + 0644) + if err != nil { + fmt.Fprintf(os.Stderr, err.Error()) + os.Exit(1) + } + + // Print zsyscall_aix_ppc64_gccgo.go + err = ioutil.WriteFile("zsyscall_aix_ppc64_gccgo.go", + []byte(fmt.Sprintf(srcTemplate3, cmdLine(), buildTags(), pack, cExtern, imp, textgccgo)), + 0644) + if err != nil { + fmt.Fprintf(os.Stderr, err.Error()) + os.Exit(1) + } +} + +const srcTemplate1 = `// %s +// Code generated by the command above; see README.md. DO NOT EDIT. + +// +build %s + +package %s + +import ( + "unsafe" +) + + +%s + +%s +` +const srcTemplate2 = `// %s +// Code generated by the command above; see README.md. DO NOT EDIT. + +// +build %s +// +build !gccgo + +package %s + +import ( + "unsafe" +) +%s +%s +%s +type syscallFunc uintptr + +var ( +%s +) + +// Implemented in runtime/syscall_aix.go. +func rawSyscall6(trap, nargs, a1, a2, a3, a4, a5, a6 uintptr) (r1, r2 uintptr, err Errno) +func syscall6(trap, nargs, a1, a2, a3, a4, a5, a6 uintptr) (r1, r2 uintptr, err Errno) + +%s +` +const srcTemplate3 = `// %s +// Code generated by the command above; see README.md. DO NOT EDIT. + +// +build %s +// +build gccgo + +package %s + +%s +*/ +import "C" +import ( + "syscall" +) + + +%s + +%s +` diff --git a/vendor/golang.org/x/sys/unix/mksyscall_solaris.go b/vendor/golang.org/x/sys/unix/mksyscall_solaris.go new file mode 100644 index 000000000..3d864738b --- /dev/null +++ b/vendor/golang.org/x/sys/unix/mksyscall_solaris.go @@ -0,0 +1,335 @@ +// Copyright 2019 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +/* + This program reads a file containing function prototypes + (like syscall_solaris.go) and generates system call bodies. + The prototypes are marked by lines beginning with "//sys" + and read like func declarations if //sys is replaced by func, but: + * The parameter lists must give a name for each argument. + This includes return parameters. + * The parameter lists must give a type for each argument: + the (x, y, z int) shorthand is not allowed. + * If the return parameter is an error number, it must be named err. + * If go func name needs to be different than its libc name, + * or the function is not in libc, name could be specified + * at the end, after "=" sign, like + //sys getsockopt(s int, level int, name int, val uintptr, vallen *_Socklen) (err error) = libsocket.getsockopt +*/ + +package main + +import ( + "bufio" + "flag" + "fmt" + "os" + "regexp" + "strings" +) + +var ( + b32 = flag.Bool("b32", false, "32bit big-endian") + l32 = flag.Bool("l32", false, "32bit little-endian") + tags = flag.String("tags", "", "build tags") +) + +// cmdLine returns this programs's commandline arguments +func cmdLine() string { + return "go run mksyscall_solaris.go " + strings.Join(os.Args[1:], " ") +} + +// buildTags returns build tags +func buildTags() string { + return *tags +} + +// Param is function parameter +type Param struct { + Name string + Type string +} + +// usage prints the program usage +func usage() { + fmt.Fprintf(os.Stderr, "usage: go run mksyscall_solaris.go [-b32 | -l32] [-tags x,y] [file ...]\n") + os.Exit(1) +} + +// parseParamList parses parameter list and returns a slice of parameters +func parseParamList(list string) []string { + list = strings.TrimSpace(list) + if list == "" { + return []string{} + } + return regexp.MustCompile(`\s*,\s*`).Split(list, -1) +} + +// parseParam splits a parameter into name and type +func parseParam(p string) Param { + ps := regexp.MustCompile(`^(\S*) (\S*)$`).FindStringSubmatch(p) + if ps == nil { + fmt.Fprintf(os.Stderr, "malformed parameter: %s\n", p) + os.Exit(1) + } + return Param{ps[1], ps[2]} +} + +func main() { + flag.Usage = usage + flag.Parse() + if len(flag.Args()) <= 0 { + fmt.Fprintf(os.Stderr, "no files to parse provided\n") + usage() + } + + endianness := "" + if *b32 { + endianness = "big-endian" + } else if *l32 { + endianness = "little-endian" + } + + pack := "" + text := "" + dynimports := "" + linknames := "" + var vars []string + for _, path := range flag.Args() { + file, err := os.Open(path) + if err != nil { + fmt.Fprintf(os.Stderr, err.Error()) + os.Exit(1) + } + s := bufio.NewScanner(file) + for s.Scan() { + t := s.Text() + t = strings.TrimSpace(t) + t = regexp.MustCompile(`\s+`).ReplaceAllString(t, ` `) + if p := regexp.MustCompile(`^package (\S+)$`).FindStringSubmatch(t); p != nil && pack == "" { + pack = p[1] + } + nonblock := regexp.MustCompile(`^\/\/sysnb `).FindStringSubmatch(t) + if regexp.MustCompile(`^\/\/sys `).FindStringSubmatch(t) == nil && nonblock == nil { + continue + } + + // Line must be of the form + // func Open(path string, mode int, perm int) (fd int, err error) + // Split into name, in params, out params. + f := regexp.MustCompile(`^\/\/sys(nb)? (\w+)\(([^()]*)\)\s*(?:\(([^()]+)\))?\s*(?:=\s*(?:(\w*)\.)?(\w*))?$`).FindStringSubmatch(t) + if f == nil { + fmt.Fprintf(os.Stderr, "%s:%s\nmalformed //sys declaration\n", path, t) + os.Exit(1) + } + funct, inps, outps, modname, sysname := f[2], f[3], f[4], f[5], f[6] + + // Split argument lists on comma. + in := parseParamList(inps) + out := parseParamList(outps) + + inps = strings.Join(in, ", ") + outps = strings.Join(out, ", ") + + // Try in vain to keep people from editing this file. + // The theory is that they jump into the middle of the file + // without reading the header. + text += "// THIS FILE IS GENERATED BY THE COMMAND AT THE TOP; DO NOT EDIT\n\n" + + // So file name. + if modname == "" { + modname = "libc" + } + + // System call name. + if sysname == "" { + sysname = funct + } + + // System call pointer variable name. + sysvarname := fmt.Sprintf("proc%s", sysname) + + strconvfunc := "BytePtrFromString" + strconvtype := "*byte" + + sysname = strings.ToLower(sysname) // All libc functions are lowercase. + + // Runtime import of function to allow cross-platform builds. + dynimports += fmt.Sprintf("//go:cgo_import_dynamic libc_%s %s \"%s.so\"\n", sysname, sysname, modname) + // Link symbol to proc address variable. + linknames += fmt.Sprintf("//go:linkname %s libc_%s\n", sysvarname, sysname) + // Library proc address variable. + vars = append(vars, sysvarname) + + // Go function header. + outlist := strings.Join(out, ", ") + if outlist != "" { + outlist = fmt.Sprintf(" (%s)", outlist) + } + if text != "" { + text += "\n" + } + text += fmt.Sprintf("func %s(%s)%s {\n", funct, strings.Join(in, ", "), outlist) + + // Check if err return available + errvar := "" + for _, param := range out { + p := parseParam(param) + if p.Type == "error" { + errvar = p.Name + continue + } + } + + // Prepare arguments to Syscall. + var args []string + n := 0 + for _, param := range in { + p := parseParam(param) + if regexp.MustCompile(`^\*`).FindStringSubmatch(p.Type) != nil { + args = append(args, "uintptr(unsafe.Pointer("+p.Name+"))") + } else if p.Type == "string" && errvar != "" { + text += fmt.Sprintf("\tvar _p%d %s\n", n, strconvtype) + text += fmt.Sprintf("\t_p%d, %s = %s(%s)\n", n, errvar, strconvfunc, p.Name) + text += fmt.Sprintf("\tif %s != nil {\n\t\treturn\n\t}\n", errvar) + args = append(args, fmt.Sprintf("uintptr(unsafe.Pointer(_p%d))", n)) + n++ + } else if p.Type == "string" { + fmt.Fprintf(os.Stderr, path+":"+funct+" uses string arguments, but has no error return\n") + text += fmt.Sprintf("\tvar _p%d %s\n", n, strconvtype) + text += fmt.Sprintf("\t_p%d, _ = %s(%s)\n", n, strconvfunc, p.Name) + args = append(args, fmt.Sprintf("uintptr(unsafe.Pointer(_p%d))", n)) + n++ + } else if s := regexp.MustCompile(`^\[\](.*)`).FindStringSubmatch(p.Type); s != nil { + // Convert slice into pointer, length. + // Have to be careful not to take address of &a[0] if len == 0: + // pass nil in that case. + text += fmt.Sprintf("\tvar _p%d *%s\n", n, s[1]) + text += fmt.Sprintf("\tif len(%s) > 0 {\n\t\t_p%d = &%s[0]\n\t}\n", p.Name, n, p.Name) + args = append(args, fmt.Sprintf("uintptr(unsafe.Pointer(_p%d))", n), fmt.Sprintf("uintptr(len(%s))", p.Name)) + n++ + } else if p.Type == "int64" && endianness != "" { + if endianness == "big-endian" { + args = append(args, fmt.Sprintf("uintptr(%s>>32)", p.Name), fmt.Sprintf("uintptr(%s)", p.Name)) + } else { + args = append(args, fmt.Sprintf("uintptr(%s)", p.Name), fmt.Sprintf("uintptr(%s>>32)", p.Name)) + } + } else if p.Type == "bool" { + text += fmt.Sprintf("\tvar _p%d uint32\n", n) + text += fmt.Sprintf("\tif %s {\n\t\t_p%d = 1\n\t} else {\n\t\t_p%d = 0\n\t}\n", p.Name, n, n) + args = append(args, fmt.Sprintf("uintptr(_p%d)", n)) + n++ + } else { + args = append(args, fmt.Sprintf("uintptr(%s)", p.Name)) + } + } + nargs := len(args) + + // Determine which form to use; pad args with zeros. + asm := "sysvicall6" + if nonblock != nil { + asm = "rawSysvicall6" + } + if len(args) <= 6 { + for len(args) < 6 { + args = append(args, "0") + } + } else { + fmt.Fprintf(os.Stderr, "%s: too many arguments to system call\n", path) + os.Exit(1) + } + + // Actual call. + arglist := strings.Join(args, ", ") + call := fmt.Sprintf("%s(uintptr(unsafe.Pointer(&%s)), %d, %s)", asm, sysvarname, nargs, arglist) + + // Assign return values. + body := "" + ret := []string{"_", "_", "_"} + doErrno := false + for i := 0; i < len(out); i++ { + p := parseParam(out[i]) + reg := "" + if p.Name == "err" { + reg = "e1" + ret[2] = reg + doErrno = true + } else { + reg = fmt.Sprintf("r%d", i) + ret[i] = reg + } + if p.Type == "bool" { + reg = fmt.Sprintf("%d != 0", reg) + } + if p.Type == "int64" && endianness != "" { + // 64-bit number in r1:r0 or r0:r1. + if i+2 > len(out) { + fmt.Fprintf(os.Stderr, "%s: not enough registers for int64 return\n", path) + os.Exit(1) + } + if endianness == "big-endian" { + reg = fmt.Sprintf("int64(r%d)<<32 | int64(r%d)", i, i+1) + } else { + reg = fmt.Sprintf("int64(r%d)<<32 | int64(r%d)", i+1, i) + } + ret[i] = fmt.Sprintf("r%d", i) + ret[i+1] = fmt.Sprintf("r%d", i+1) + } + if reg != "e1" { + body += fmt.Sprintf("\t%s = %s(%s)\n", p.Name, p.Type, reg) + } + } + if ret[0] == "_" && ret[1] == "_" && ret[2] == "_" { + text += fmt.Sprintf("\t%s\n", call) + } else { + text += fmt.Sprintf("\t%s, %s, %s := %s\n", ret[0], ret[1], ret[2], call) + } + text += body + + if doErrno { + text += "\tif e1 != 0 {\n" + text += "\t\terr = e1\n" + text += "\t}\n" + } + text += "\treturn\n" + text += "}\n" + } + if err := s.Err(); err != nil { + fmt.Fprintf(os.Stderr, err.Error()) + os.Exit(1) + } + file.Close() + } + imp := "" + if pack != "unix" { + imp = "import \"golang.org/x/sys/unix\"\n" + + } + vardecls := "\t" + strings.Join(vars, ",\n\t") + vardecls += " syscallFunc" + fmt.Printf(srcTemplate, cmdLine(), buildTags(), pack, imp, dynimports, linknames, vardecls, text) +} + +const srcTemplate = `// %s +// Code generated by the command above; see README.md. DO NOT EDIT. + +// +build %s + +package %s + +import ( + "syscall" + "unsafe" +) +%s +%s +%s +var ( +%s +) + +%s +` diff --git a/vendor/golang.org/x/sys/unix/mksysnum.go b/vendor/golang.org/x/sys/unix/mksysnum.go new file mode 100644 index 000000000..07f8960ff --- /dev/null +++ b/vendor/golang.org/x/sys/unix/mksysnum.go @@ -0,0 +1,190 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +// Generate system call table for DragonFly, NetBSD, +// FreeBSD, OpenBSD or Darwin from master list +// (for example, /usr/src/sys/kern/syscalls.master or +// sys/syscall.h). +package main + +import ( + "bufio" + "fmt" + "io" + "io/ioutil" + "net/http" + "os" + "regexp" + "strings" +) + +var ( + goos, goarch string +) + +// cmdLine returns this programs's commandline arguments +func cmdLine() string { + return "go run mksysnum.go " + strings.Join(os.Args[1:], " ") +} + +// buildTags returns build tags +func buildTags() string { + return fmt.Sprintf("%s,%s", goarch, goos) +} + +func checkErr(err error) { + if err != nil { + fmt.Fprintf(os.Stderr, "%v\n", err) + os.Exit(1) + } +} + +// source string and substring slice for regexp +type re struct { + str string // source string + sub []string // matched sub-string +} + +// Match performs regular expression match +func (r *re) Match(exp string) bool { + r.sub = regexp.MustCompile(exp).FindStringSubmatch(r.str) + if r.sub != nil { + return true + } + return false +} + +// fetchFile fetches a text file from URL +func fetchFile(URL string) io.Reader { + resp, err := http.Get(URL) + checkErr(err) + defer resp.Body.Close() + body, err := ioutil.ReadAll(resp.Body) + checkErr(err) + return strings.NewReader(string(body)) +} + +// readFile reads a text file from path +func readFile(path string) io.Reader { + file, err := os.Open(os.Args[1]) + checkErr(err) + return file +} + +func format(name, num, proto string) string { + name = strings.ToUpper(name) + // There are multiple entries for enosys and nosys, so comment them out. + nm := re{str: name} + if nm.Match(`^SYS_E?NOSYS$`) { + name = fmt.Sprintf("// %s", name) + } + if name == `SYS_SYS_EXIT` { + name = `SYS_EXIT` + } + return fmt.Sprintf(" %s = %s; // %s\n", name, num, proto) +} + +func main() { + // Get the OS (using GOOS_TARGET if it exist) + goos = os.Getenv("GOOS_TARGET") + if goos == "" { + goos = os.Getenv("GOOS") + } + // Get the architecture (using GOARCH_TARGET if it exists) + goarch = os.Getenv("GOARCH_TARGET") + if goarch == "" { + goarch = os.Getenv("GOARCH") + } + // Check if GOOS and GOARCH environment variables are defined + if goarch == "" || goos == "" { + fmt.Fprintf(os.Stderr, "GOARCH or GOOS not defined in environment\n") + os.Exit(1) + } + + file := strings.TrimSpace(os.Args[1]) + var syscalls io.Reader + if strings.HasPrefix(file, "https://") || strings.HasPrefix(file, "http://") { + // Download syscalls.master file + syscalls = fetchFile(file) + } else { + syscalls = readFile(file) + } + + var text, line string + s := bufio.NewScanner(syscalls) + for s.Scan() { + t := re{str: line} + if t.Match(`^(.*)\\$`) { + // Handle continuation + line = t.sub[1] + line += strings.TrimLeft(s.Text(), " \t") + } else { + // New line + line = s.Text() + } + t = re{str: line} + if t.Match(`\\$`) { + continue + } + t = re{str: line} + + switch goos { + case "dragonfly": + if t.Match(`^([0-9]+)\s+STD\s+({ \S+\s+(\w+).*)$`) { + num, proto := t.sub[1], t.sub[2] + name := fmt.Sprintf("SYS_%s", t.sub[3]) + text += format(name, num, proto) + } + case "freebsd": + if t.Match(`^([0-9]+)\s+\S+\s+(?:NO)?STD\s+({ \S+\s+(\w+).*)$`) { + num, proto := t.sub[1], t.sub[2] + name := fmt.Sprintf("SYS_%s", t.sub[3]) + text += format(name, num, proto) + } + case "openbsd": + if t.Match(`^([0-9]+)\s+STD\s+(NOLOCK\s+)?({ \S+\s+\*?(\w+).*)$`) { + num, proto, name := t.sub[1], t.sub[3], t.sub[4] + text += format(name, num, proto) + } + case "netbsd": + if t.Match(`^([0-9]+)\s+((STD)|(NOERR))\s+(RUMP\s+)?({\s+\S+\s*\*?\s*\|(\S+)\|(\S*)\|(\w+).*\s+})(\s+(\S+))?$`) { + num, proto, compat := t.sub[1], t.sub[6], t.sub[8] + name := t.sub[7] + "_" + t.sub[9] + if t.sub[11] != "" { + name = t.sub[7] + "_" + t.sub[11] + } + name = strings.ToUpper(name) + if compat == "" || compat == "13" || compat == "30" || compat == "50" { + text += fmt.Sprintf(" %s = %s; // %s\n", name, num, proto) + } + } + case "darwin": + if t.Match(`^#define\s+SYS_(\w+)\s+([0-9]+)`) { + name, num := t.sub[1], t.sub[2] + name = strings.ToUpper(name) + text += fmt.Sprintf(" SYS_%s = %s;\n", name, num) + } + default: + fmt.Fprintf(os.Stderr, "unrecognized GOOS=%s\n", goos) + os.Exit(1) + + } + } + err := s.Err() + checkErr(err) + + fmt.Printf(template, cmdLine(), buildTags(), text) +} + +const template = `// %s +// Code generated by the command above; see README.md. DO NOT EDIT. + +// +build %s + +package unix + +const( +%s)` diff --git a/vendor/golang.org/x/sys/unix/types_aix.go b/vendor/golang.org/x/sys/unix/types_aix.go new file mode 100644 index 000000000..25e834940 --- /dev/null +++ b/vendor/golang.org/x/sys/unix/types_aix.go @@ -0,0 +1,236 @@ +// Copyright 2018 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore +// +build aix + +/* +Input to cgo -godefs. See also mkerrors.sh and mkall.sh +*/ + +// +godefs map struct_in_addr [4]byte /* in_addr */ +// +godefs map struct_in6_addr [16]byte /* in6_addr */ + +package unix + +/* +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include +#include +#include +#include + + +#include +#include + +enum { + sizeofPtr = sizeof(void*), +}; + +union sockaddr_all { + struct sockaddr s1; // this one gets used for fields + struct sockaddr_in s2; // these pad it out + struct sockaddr_in6 s3; + struct sockaddr_un s4; + struct sockaddr_dl s5; +}; + +struct sockaddr_any { + struct sockaddr addr; + char pad[sizeof(union sockaddr_all) - sizeof(struct sockaddr)]; +}; + +*/ +import "C" + +// Machine characteristics + +const ( + SizeofPtr = C.sizeofPtr + SizeofShort = C.sizeof_short + SizeofInt = C.sizeof_int + SizeofLong = C.sizeof_long + SizeofLongLong = C.sizeof_longlong + PathMax = C.PATH_MAX +) + +// Basic types + +type ( + _C_short C.short + _C_int C.int + _C_long C.long + _C_long_long C.longlong +) + +type off64 C.off64_t +type off C.off_t +type Mode_t C.mode_t + +// Time + +type Timespec C.struct_timespec + +type StTimespec C.struct_st_timespec + +type Timeval C.struct_timeval + +type Timeval32 C.struct_timeval32 + +type Timex C.struct_timex + +type Time_t C.time_t + +type Tms C.struct_tms + +type Utimbuf C.struct_utimbuf + +type Timezone C.struct_timezone + +// Processes + +type Rusage C.struct_rusage + +type Rlimit C.struct_rlimit64 + +type Pid_t C.pid_t + +type _Gid_t C.gid_t + +type dev_t C.dev_t + +// Files + +type Stat_t C.struct_stat + +type StatxTimestamp C.struct_statx_timestamp + +type Statx_t C.struct_statx + +type Dirent C.struct_dirent + +// Sockets + +type RawSockaddrInet4 C.struct_sockaddr_in + +type RawSockaddrInet6 C.struct_sockaddr_in6 + +type RawSockaddrUnix C.struct_sockaddr_un + +type RawSockaddr C.struct_sockaddr + +type RawSockaddrAny C.struct_sockaddr_any + +type _Socklen C.socklen_t + +type Cmsghdr C.struct_cmsghdr + +type ICMPv6Filter C.struct_icmp6_filter + +type Iovec C.struct_iovec + +type IPMreq C.struct_ip_mreq + +type IPv6Mreq C.struct_ipv6_mreq + +type IPv6MTUInfo C.struct_ip6_mtuinfo + +type Linger C.struct_linger + +type Msghdr C.struct_msghdr + +const ( + SizeofSockaddrInet4 = C.sizeof_struct_sockaddr_in + SizeofSockaddrInet6 = C.sizeof_struct_sockaddr_in6 + SizeofSockaddrAny = C.sizeof_struct_sockaddr_any + SizeofSockaddrUnix = C.sizeof_struct_sockaddr_un + SizeofLinger = C.sizeof_struct_linger + SizeofIPMreq = C.sizeof_struct_ip_mreq + SizeofIPv6Mreq = C.sizeof_struct_ipv6_mreq + SizeofIPv6MTUInfo = C.sizeof_struct_ip6_mtuinfo + SizeofMsghdr = C.sizeof_struct_msghdr + SizeofCmsghdr = C.sizeof_struct_cmsghdr + SizeofICMPv6Filter = C.sizeof_struct_icmp6_filter +) + +// Routing and interface messages + +const ( + SizeofIfMsghdr = C.sizeof_struct_if_msghdr +) + +type IfMsgHdr C.struct_if_msghdr + +// Misc + +type FdSet C.fd_set + +type Utsname C.struct_utsname + +type Ustat_t C.struct_ustat + +type Sigset_t C.sigset_t + +const ( + AT_FDCWD = C.AT_FDCWD + AT_REMOVEDIR = C.AT_REMOVEDIR + AT_SYMLINK_NOFOLLOW = C.AT_SYMLINK_NOFOLLOW +) + +// Terminal handling + +type Termios C.struct_termios + +type Termio C.struct_termio + +type Winsize C.struct_winsize + +//poll + +type PollFd struct { + Fd int32 + Events uint16 + Revents uint16 +} + +const ( + POLLERR = C.POLLERR + POLLHUP = C.POLLHUP + POLLIN = C.POLLIN + POLLNVAL = C.POLLNVAL + POLLOUT = C.POLLOUT + POLLPRI = C.POLLPRI + POLLRDBAND = C.POLLRDBAND + POLLRDNORM = C.POLLRDNORM + POLLWRBAND = C.POLLWRBAND + POLLWRNORM = C.POLLWRNORM +) + +//flock_t + +type Flock_t C.struct_flock64 + +// Statfs + +type Fsid_t C.struct_fsid_t +type Fsid64_t C.struct_fsid64_t + +type Statfs_t C.struct_statfs + +const RNDGETENTCNT = 0x80045200 diff --git a/vendor/golang.org/x/sys/unix/types_darwin.go b/vendor/golang.org/x/sys/unix/types_darwin.go new file mode 100644 index 000000000..9fd2aaa6a --- /dev/null +++ b/vendor/golang.org/x/sys/unix/types_darwin.go @@ -0,0 +1,277 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +/* +Input to cgo -godefs. See README.md +*/ + +// +godefs map struct_in_addr [4]byte /* in_addr */ +// +godefs map struct_in6_addr [16]byte /* in6_addr */ + +package unix + +/* +#define __DARWIN_UNIX03 0 +#define KERNEL +#define _DARWIN_USE_64_BIT_INODE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +enum { + sizeofPtr = sizeof(void*), +}; + +union sockaddr_all { + struct sockaddr s1; // this one gets used for fields + struct sockaddr_in s2; // these pad it out + struct sockaddr_in6 s3; + struct sockaddr_un s4; + struct sockaddr_dl s5; +}; + +struct sockaddr_any { + struct sockaddr addr; + char pad[sizeof(union sockaddr_all) - sizeof(struct sockaddr)]; +}; + +*/ +import "C" + +// Machine characteristics + +const ( + SizeofPtr = C.sizeofPtr + SizeofShort = C.sizeof_short + SizeofInt = C.sizeof_int + SizeofLong = C.sizeof_long + SizeofLongLong = C.sizeof_longlong +) + +// Basic types + +type ( + _C_short C.short + _C_int C.int + _C_long C.long + _C_long_long C.longlong +) + +// Time + +type Timespec C.struct_timespec + +type Timeval C.struct_timeval + +type Timeval32 C.struct_timeval32 + +// Processes + +type Rusage C.struct_rusage + +type Rlimit C.struct_rlimit + +type _Gid_t C.gid_t + +// Files + +type Stat_t C.struct_stat64 + +type Statfs_t C.struct_statfs64 + +type Flock_t C.struct_flock + +type Fstore_t C.struct_fstore + +type Radvisory_t C.struct_radvisory + +type Fbootstraptransfer_t C.struct_fbootstraptransfer + +type Log2phys_t C.struct_log2phys + +type Fsid C.struct_fsid + +type Dirent C.struct_dirent + +// Sockets + +type RawSockaddrInet4 C.struct_sockaddr_in + +type RawSockaddrInet6 C.struct_sockaddr_in6 + +type RawSockaddrUnix C.struct_sockaddr_un + +type RawSockaddrDatalink C.struct_sockaddr_dl + +type RawSockaddr C.struct_sockaddr + +type RawSockaddrAny C.struct_sockaddr_any + +type _Socklen C.socklen_t + +type Linger C.struct_linger + +type Iovec C.struct_iovec + +type IPMreq C.struct_ip_mreq + +type IPv6Mreq C.struct_ipv6_mreq + +type Msghdr C.struct_msghdr + +type Cmsghdr C.struct_cmsghdr + +type Inet4Pktinfo C.struct_in_pktinfo + +type Inet6Pktinfo C.struct_in6_pktinfo + +type IPv6MTUInfo C.struct_ip6_mtuinfo + +type ICMPv6Filter C.struct_icmp6_filter + +const ( + SizeofSockaddrInet4 = C.sizeof_struct_sockaddr_in + SizeofSockaddrInet6 = C.sizeof_struct_sockaddr_in6 + SizeofSockaddrAny = C.sizeof_struct_sockaddr_any + SizeofSockaddrUnix = C.sizeof_struct_sockaddr_un + SizeofSockaddrDatalink = C.sizeof_struct_sockaddr_dl + SizeofLinger = C.sizeof_struct_linger + SizeofIPMreq = C.sizeof_struct_ip_mreq + SizeofIPv6Mreq = C.sizeof_struct_ipv6_mreq + SizeofMsghdr = C.sizeof_struct_msghdr + SizeofCmsghdr = C.sizeof_struct_cmsghdr + SizeofInet4Pktinfo = C.sizeof_struct_in_pktinfo + SizeofInet6Pktinfo = C.sizeof_struct_in6_pktinfo + SizeofIPv6MTUInfo = C.sizeof_struct_ip6_mtuinfo + SizeofICMPv6Filter = C.sizeof_struct_icmp6_filter +) + +// Ptrace requests + +const ( + PTRACE_TRACEME = C.PT_TRACE_ME + PTRACE_CONT = C.PT_CONTINUE + PTRACE_KILL = C.PT_KILL +) + +// Events (kqueue, kevent) + +type Kevent_t C.struct_kevent + +// Select + +type FdSet C.fd_set + +// Routing and interface messages + +const ( + SizeofIfMsghdr = C.sizeof_struct_if_msghdr + SizeofIfData = C.sizeof_struct_if_data + SizeofIfaMsghdr = C.sizeof_struct_ifa_msghdr + SizeofIfmaMsghdr = C.sizeof_struct_ifma_msghdr + SizeofIfmaMsghdr2 = C.sizeof_struct_ifma_msghdr2 + SizeofRtMsghdr = C.sizeof_struct_rt_msghdr + SizeofRtMetrics = C.sizeof_struct_rt_metrics +) + +type IfMsghdr C.struct_if_msghdr + +type IfData C.struct_if_data + +type IfaMsghdr C.struct_ifa_msghdr + +type IfmaMsghdr C.struct_ifma_msghdr + +type IfmaMsghdr2 C.struct_ifma_msghdr2 + +type RtMsghdr C.struct_rt_msghdr + +type RtMetrics C.struct_rt_metrics + +// Berkeley packet filter + +const ( + SizeofBpfVersion = C.sizeof_struct_bpf_version + SizeofBpfStat = C.sizeof_struct_bpf_stat + SizeofBpfProgram = C.sizeof_struct_bpf_program + SizeofBpfInsn = C.sizeof_struct_bpf_insn + SizeofBpfHdr = C.sizeof_struct_bpf_hdr +) + +type BpfVersion C.struct_bpf_version + +type BpfStat C.struct_bpf_stat + +type BpfProgram C.struct_bpf_program + +type BpfInsn C.struct_bpf_insn + +type BpfHdr C.struct_bpf_hdr + +// Terminal handling + +type Termios C.struct_termios + +type Winsize C.struct_winsize + +// fchmodat-like syscalls. + +const ( + AT_FDCWD = C.AT_FDCWD + AT_REMOVEDIR = C.AT_REMOVEDIR + AT_SYMLINK_FOLLOW = C.AT_SYMLINK_FOLLOW + AT_SYMLINK_NOFOLLOW = C.AT_SYMLINK_NOFOLLOW +) + +// poll + +type PollFd C.struct_pollfd + +const ( + POLLERR = C.POLLERR + POLLHUP = C.POLLHUP + POLLIN = C.POLLIN + POLLNVAL = C.POLLNVAL + POLLOUT = C.POLLOUT + POLLPRI = C.POLLPRI + POLLRDBAND = C.POLLRDBAND + POLLRDNORM = C.POLLRDNORM + POLLWRBAND = C.POLLWRBAND + POLLWRNORM = C.POLLWRNORM +) + +// uname + +type Utsname C.struct_utsname diff --git a/vendor/golang.org/x/sys/unix/types_dragonfly.go b/vendor/golang.org/x/sys/unix/types_dragonfly.go new file mode 100644 index 000000000..3365dd79d --- /dev/null +++ b/vendor/golang.org/x/sys/unix/types_dragonfly.go @@ -0,0 +1,263 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +/* +Input to cgo -godefs. See README.md +*/ + +// +godefs map struct_in_addr [4]byte /* in_addr */ +// +godefs map struct_in6_addr [16]byte /* in6_addr */ + +package unix + +/* +#define KERNEL +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +enum { + sizeofPtr = sizeof(void*), +}; + +union sockaddr_all { + struct sockaddr s1; // this one gets used for fields + struct sockaddr_in s2; // these pad it out + struct sockaddr_in6 s3; + struct sockaddr_un s4; + struct sockaddr_dl s5; +}; + +struct sockaddr_any { + struct sockaddr addr; + char pad[sizeof(union sockaddr_all) - sizeof(struct sockaddr)]; +}; + +*/ +import "C" + +// Machine characteristics + +const ( + SizeofPtr = C.sizeofPtr + SizeofShort = C.sizeof_short + SizeofInt = C.sizeof_int + SizeofLong = C.sizeof_long + SizeofLongLong = C.sizeof_longlong +) + +// Basic types + +type ( + _C_short C.short + _C_int C.int + _C_long C.long + _C_long_long C.longlong +) + +// Time + +type Timespec C.struct_timespec + +type Timeval C.struct_timeval + +// Processes + +type Rusage C.struct_rusage + +type Rlimit C.struct_rlimit + +type _Gid_t C.gid_t + +// Files + +type Stat_t C.struct_stat + +type Statfs_t C.struct_statfs + +type Flock_t C.struct_flock + +type Dirent C.struct_dirent + +type Fsid C.struct_fsid + +// File system limits + +const ( + PathMax = C.PATH_MAX +) + +// Sockets + +type RawSockaddrInet4 C.struct_sockaddr_in + +type RawSockaddrInet6 C.struct_sockaddr_in6 + +type RawSockaddrUnix C.struct_sockaddr_un + +type RawSockaddrDatalink C.struct_sockaddr_dl + +type RawSockaddr C.struct_sockaddr + +type RawSockaddrAny C.struct_sockaddr_any + +type _Socklen C.socklen_t + +type Linger C.struct_linger + +type Iovec C.struct_iovec + +type IPMreq C.struct_ip_mreq + +type IPv6Mreq C.struct_ipv6_mreq + +type Msghdr C.struct_msghdr + +type Cmsghdr C.struct_cmsghdr + +type Inet6Pktinfo C.struct_in6_pktinfo + +type IPv6MTUInfo C.struct_ip6_mtuinfo + +type ICMPv6Filter C.struct_icmp6_filter + +const ( + SizeofSockaddrInet4 = C.sizeof_struct_sockaddr_in + SizeofSockaddrInet6 = C.sizeof_struct_sockaddr_in6 + SizeofSockaddrAny = C.sizeof_struct_sockaddr_any + SizeofSockaddrUnix = C.sizeof_struct_sockaddr_un + SizeofSockaddrDatalink = C.sizeof_struct_sockaddr_dl + SizeofLinger = C.sizeof_struct_linger + SizeofIPMreq = C.sizeof_struct_ip_mreq + SizeofIPv6Mreq = C.sizeof_struct_ipv6_mreq + SizeofMsghdr = C.sizeof_struct_msghdr + SizeofCmsghdr = C.sizeof_struct_cmsghdr + SizeofInet6Pktinfo = C.sizeof_struct_in6_pktinfo + SizeofIPv6MTUInfo = C.sizeof_struct_ip6_mtuinfo + SizeofICMPv6Filter = C.sizeof_struct_icmp6_filter +) + +// Ptrace requests + +const ( + PTRACE_TRACEME = C.PT_TRACE_ME + PTRACE_CONT = C.PT_CONTINUE + PTRACE_KILL = C.PT_KILL +) + +// Events (kqueue, kevent) + +type Kevent_t C.struct_kevent + +// Select + +type FdSet C.fd_set + +// Routing and interface messages + +const ( + SizeofIfMsghdr = C.sizeof_struct_if_msghdr + SizeofIfData = C.sizeof_struct_if_data + SizeofIfaMsghdr = C.sizeof_struct_ifa_msghdr + SizeofIfmaMsghdr = C.sizeof_struct_ifma_msghdr + SizeofIfAnnounceMsghdr = C.sizeof_struct_if_announcemsghdr + SizeofRtMsghdr = C.sizeof_struct_rt_msghdr + SizeofRtMetrics = C.sizeof_struct_rt_metrics +) + +type IfMsghdr C.struct_if_msghdr + +type IfData C.struct_if_data + +type IfaMsghdr C.struct_ifa_msghdr + +type IfmaMsghdr C.struct_ifma_msghdr + +type IfAnnounceMsghdr C.struct_if_announcemsghdr + +type RtMsghdr C.struct_rt_msghdr + +type RtMetrics C.struct_rt_metrics + +// Berkeley packet filter + +const ( + SizeofBpfVersion = C.sizeof_struct_bpf_version + SizeofBpfStat = C.sizeof_struct_bpf_stat + SizeofBpfProgram = C.sizeof_struct_bpf_program + SizeofBpfInsn = C.sizeof_struct_bpf_insn + SizeofBpfHdr = C.sizeof_struct_bpf_hdr +) + +type BpfVersion C.struct_bpf_version + +type BpfStat C.struct_bpf_stat + +type BpfProgram C.struct_bpf_program + +type BpfInsn C.struct_bpf_insn + +type BpfHdr C.struct_bpf_hdr + +// Terminal handling + +type Termios C.struct_termios + +type Winsize C.struct_winsize + +// fchmodat-like syscalls. + +const ( + AT_FDCWD = C.AT_FDCWD + AT_SYMLINK_NOFOLLOW = C.AT_SYMLINK_NOFOLLOW +) + +// poll + +type PollFd C.struct_pollfd + +const ( + POLLERR = C.POLLERR + POLLHUP = C.POLLHUP + POLLIN = C.POLLIN + POLLNVAL = C.POLLNVAL + POLLOUT = C.POLLOUT + POLLPRI = C.POLLPRI + POLLRDBAND = C.POLLRDBAND + POLLRDNORM = C.POLLRDNORM + POLLWRBAND = C.POLLWRBAND + POLLWRNORM = C.POLLWRNORM +) + +// Uname + +type Utsname C.struct_utsname diff --git a/vendor/golang.org/x/sys/unix/types_freebsd.go b/vendor/golang.org/x/sys/unix/types_freebsd.go new file mode 100644 index 000000000..747079895 --- /dev/null +++ b/vendor/golang.org/x/sys/unix/types_freebsd.go @@ -0,0 +1,356 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +/* +Input to cgo -godefs. See README.md +*/ + +// +godefs map struct_in_addr [4]byte /* in_addr */ +// +godefs map struct_in6_addr [16]byte /* in6_addr */ + +package unix + +/* +#define _WANT_FREEBSD11_STAT 1 +#define _WANT_FREEBSD11_STATFS 1 +#define _WANT_FREEBSD11_DIRENT 1 +#define _WANT_FREEBSD11_KEVENT 1 + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +enum { + sizeofPtr = sizeof(void*), +}; + +union sockaddr_all { + struct sockaddr s1; // this one gets used for fields + struct sockaddr_in s2; // these pad it out + struct sockaddr_in6 s3; + struct sockaddr_un s4; + struct sockaddr_dl s5; +}; + +struct sockaddr_any { + struct sockaddr addr; + char pad[sizeof(union sockaddr_all) - sizeof(struct sockaddr)]; +}; + +// This structure is a duplicate of if_data on FreeBSD 8-STABLE. +// See /usr/include/net/if.h. +struct if_data8 { + u_char ifi_type; + u_char ifi_physical; + u_char ifi_addrlen; + u_char ifi_hdrlen; + u_char ifi_link_state; + u_char ifi_spare_char1; + u_char ifi_spare_char2; + u_char ifi_datalen; + u_long ifi_mtu; + u_long ifi_metric; + u_long ifi_baudrate; + u_long ifi_ipackets; + u_long ifi_ierrors; + u_long ifi_opackets; + u_long ifi_oerrors; + u_long ifi_collisions; + u_long ifi_ibytes; + u_long ifi_obytes; + u_long ifi_imcasts; + u_long ifi_omcasts; + u_long ifi_iqdrops; + u_long ifi_noproto; + u_long ifi_hwassist; +// FIXME: these are now unions, so maybe need to change definitions? +#undef ifi_epoch + time_t ifi_epoch; +#undef ifi_lastchange + struct timeval ifi_lastchange; +}; + +// This structure is a duplicate of if_msghdr on FreeBSD 8-STABLE. +// See /usr/include/net/if.h. +struct if_msghdr8 { + u_short ifm_msglen; + u_char ifm_version; + u_char ifm_type; + int ifm_addrs; + int ifm_flags; + u_short ifm_index; + struct if_data8 ifm_data; +}; +*/ +import "C" + +// Machine characteristics + +const ( + SizeofPtr = C.sizeofPtr + SizeofShort = C.sizeof_short + SizeofInt = C.sizeof_int + SizeofLong = C.sizeof_long + SizeofLongLong = C.sizeof_longlong +) + +// Basic types + +type ( + _C_short C.short + _C_int C.int + _C_long C.long + _C_long_long C.longlong +) + +// Time + +type Timespec C.struct_timespec + +type Timeval C.struct_timeval + +// Processes + +type Rusage C.struct_rusage + +type Rlimit C.struct_rlimit + +type _Gid_t C.gid_t + +// Files + +const ( + _statfsVersion = C.STATFS_VERSION + _dirblksiz = C.DIRBLKSIZ +) + +type Stat_t C.struct_stat + +type stat_freebsd11_t C.struct_freebsd11_stat + +type Statfs_t C.struct_statfs + +type statfs_freebsd11_t C.struct_freebsd11_statfs + +type Flock_t C.struct_flock + +type Dirent C.struct_dirent + +type dirent_freebsd11 C.struct_freebsd11_dirent + +type Fsid C.struct_fsid + +// File system limits + +const ( + PathMax = C.PATH_MAX +) + +// Advice to Fadvise + +const ( + FADV_NORMAL = C.POSIX_FADV_NORMAL + FADV_RANDOM = C.POSIX_FADV_RANDOM + FADV_SEQUENTIAL = C.POSIX_FADV_SEQUENTIAL + FADV_WILLNEED = C.POSIX_FADV_WILLNEED + FADV_DONTNEED = C.POSIX_FADV_DONTNEED + FADV_NOREUSE = C.POSIX_FADV_NOREUSE +) + +// Sockets + +type RawSockaddrInet4 C.struct_sockaddr_in + +type RawSockaddrInet6 C.struct_sockaddr_in6 + +type RawSockaddrUnix C.struct_sockaddr_un + +type RawSockaddrDatalink C.struct_sockaddr_dl + +type RawSockaddr C.struct_sockaddr + +type RawSockaddrAny C.struct_sockaddr_any + +type _Socklen C.socklen_t + +type Linger C.struct_linger + +type Iovec C.struct_iovec + +type IPMreq C.struct_ip_mreq + +type IPMreqn C.struct_ip_mreqn + +type IPv6Mreq C.struct_ipv6_mreq + +type Msghdr C.struct_msghdr + +type Cmsghdr C.struct_cmsghdr + +type Inet6Pktinfo C.struct_in6_pktinfo + +type IPv6MTUInfo C.struct_ip6_mtuinfo + +type ICMPv6Filter C.struct_icmp6_filter + +const ( + SizeofSockaddrInet4 = C.sizeof_struct_sockaddr_in + SizeofSockaddrInet6 = C.sizeof_struct_sockaddr_in6 + SizeofSockaddrAny = C.sizeof_struct_sockaddr_any + SizeofSockaddrUnix = C.sizeof_struct_sockaddr_un + SizeofSockaddrDatalink = C.sizeof_struct_sockaddr_dl + SizeofLinger = C.sizeof_struct_linger + SizeofIPMreq = C.sizeof_struct_ip_mreq + SizeofIPMreqn = C.sizeof_struct_ip_mreqn + SizeofIPv6Mreq = C.sizeof_struct_ipv6_mreq + SizeofMsghdr = C.sizeof_struct_msghdr + SizeofCmsghdr = C.sizeof_struct_cmsghdr + SizeofInet6Pktinfo = C.sizeof_struct_in6_pktinfo + SizeofIPv6MTUInfo = C.sizeof_struct_ip6_mtuinfo + SizeofICMPv6Filter = C.sizeof_struct_icmp6_filter +) + +// Ptrace requests + +const ( + PTRACE_TRACEME = C.PT_TRACE_ME + PTRACE_CONT = C.PT_CONTINUE + PTRACE_KILL = C.PT_KILL +) + +// Events (kqueue, kevent) + +type Kevent_t C.struct_kevent_freebsd11 + +// Select + +type FdSet C.fd_set + +// Routing and interface messages + +const ( + sizeofIfMsghdr = C.sizeof_struct_if_msghdr + SizeofIfMsghdr = C.sizeof_struct_if_msghdr8 + sizeofIfData = C.sizeof_struct_if_data + SizeofIfData = C.sizeof_struct_if_data8 + SizeofIfaMsghdr = C.sizeof_struct_ifa_msghdr + SizeofIfmaMsghdr = C.sizeof_struct_ifma_msghdr + SizeofIfAnnounceMsghdr = C.sizeof_struct_if_announcemsghdr + SizeofRtMsghdr = C.sizeof_struct_rt_msghdr + SizeofRtMetrics = C.sizeof_struct_rt_metrics +) + +type ifMsghdr C.struct_if_msghdr + +type IfMsghdr C.struct_if_msghdr8 + +type ifData C.struct_if_data + +type IfData C.struct_if_data8 + +type IfaMsghdr C.struct_ifa_msghdr + +type IfmaMsghdr C.struct_ifma_msghdr + +type IfAnnounceMsghdr C.struct_if_announcemsghdr + +type RtMsghdr C.struct_rt_msghdr + +type RtMetrics C.struct_rt_metrics + +// Berkeley packet filter + +const ( + SizeofBpfVersion = C.sizeof_struct_bpf_version + SizeofBpfStat = C.sizeof_struct_bpf_stat + SizeofBpfZbuf = C.sizeof_struct_bpf_zbuf + SizeofBpfProgram = C.sizeof_struct_bpf_program + SizeofBpfInsn = C.sizeof_struct_bpf_insn + SizeofBpfHdr = C.sizeof_struct_bpf_hdr + SizeofBpfZbufHeader = C.sizeof_struct_bpf_zbuf_header +) + +type BpfVersion C.struct_bpf_version + +type BpfStat C.struct_bpf_stat + +type BpfZbuf C.struct_bpf_zbuf + +type BpfProgram C.struct_bpf_program + +type BpfInsn C.struct_bpf_insn + +type BpfHdr C.struct_bpf_hdr + +type BpfZbufHeader C.struct_bpf_zbuf_header + +// Terminal handling + +type Termios C.struct_termios + +type Winsize C.struct_winsize + +// fchmodat-like syscalls. + +const ( + AT_FDCWD = C.AT_FDCWD + AT_REMOVEDIR = C.AT_REMOVEDIR + AT_SYMLINK_FOLLOW = C.AT_SYMLINK_FOLLOW + AT_SYMLINK_NOFOLLOW = C.AT_SYMLINK_NOFOLLOW +) + +// poll + +type PollFd C.struct_pollfd + +const ( + POLLERR = C.POLLERR + POLLHUP = C.POLLHUP + POLLIN = C.POLLIN + POLLINIGNEOF = C.POLLINIGNEOF + POLLNVAL = C.POLLNVAL + POLLOUT = C.POLLOUT + POLLPRI = C.POLLPRI + POLLRDBAND = C.POLLRDBAND + POLLRDNORM = C.POLLRDNORM + POLLWRBAND = C.POLLWRBAND + POLLWRNORM = C.POLLWRNORM +) + +// Capabilities + +type CapRights C.struct_cap_rights + +// Uname + +type Utsname C.struct_utsname diff --git a/vendor/golang.org/x/sys/unix/types_netbsd.go b/vendor/golang.org/x/sys/unix/types_netbsd.go new file mode 100644 index 000000000..2dd4f9542 --- /dev/null +++ b/vendor/golang.org/x/sys/unix/types_netbsd.go @@ -0,0 +1,289 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +/* +Input to cgo -godefs. See README.md +*/ + +// +godefs map struct_in_addr [4]byte /* in_addr */ +// +godefs map struct_in6_addr [16]byte /* in6_addr */ + +package unix + +/* +#define KERNEL +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +enum { + sizeofPtr = sizeof(void*), +}; + +union sockaddr_all { + struct sockaddr s1; // this one gets used for fields + struct sockaddr_in s2; // these pad it out + struct sockaddr_in6 s3; + struct sockaddr_un s4; + struct sockaddr_dl s5; +}; + +struct sockaddr_any { + struct sockaddr addr; + char pad[sizeof(union sockaddr_all) - sizeof(struct sockaddr)]; +}; + +*/ +import "C" + +// Machine characteristics + +const ( + SizeofPtr = C.sizeofPtr + SizeofShort = C.sizeof_short + SizeofInt = C.sizeof_int + SizeofLong = C.sizeof_long + SizeofLongLong = C.sizeof_longlong +) + +// Basic types + +type ( + _C_short C.short + _C_int C.int + _C_long C.long + _C_long_long C.longlong +) + +// Time + +type Timespec C.struct_timespec + +type Timeval C.struct_timeval + +// Processes + +type Rusage C.struct_rusage + +type Rlimit C.struct_rlimit + +type _Gid_t C.gid_t + +// Files + +type Stat_t C.struct_stat + +type Statfs_t C.struct_statfs + +type Flock_t C.struct_flock + +type Dirent C.struct_dirent + +type Fsid C.fsid_t + +// File system limits + +const ( + PathMax = C.PATH_MAX +) + +// Advice to Fadvise + +const ( + FADV_NORMAL = C.POSIX_FADV_NORMAL + FADV_RANDOM = C.POSIX_FADV_RANDOM + FADV_SEQUENTIAL = C.POSIX_FADV_SEQUENTIAL + FADV_WILLNEED = C.POSIX_FADV_WILLNEED + FADV_DONTNEED = C.POSIX_FADV_DONTNEED + FADV_NOREUSE = C.POSIX_FADV_NOREUSE +) + +// Sockets + +type RawSockaddrInet4 C.struct_sockaddr_in + +type RawSockaddrInet6 C.struct_sockaddr_in6 + +type RawSockaddrUnix C.struct_sockaddr_un + +type RawSockaddrDatalink C.struct_sockaddr_dl + +type RawSockaddr C.struct_sockaddr + +type RawSockaddrAny C.struct_sockaddr_any + +type _Socklen C.socklen_t + +type Linger C.struct_linger + +type Iovec C.struct_iovec + +type IPMreq C.struct_ip_mreq + +type IPv6Mreq C.struct_ipv6_mreq + +type Msghdr C.struct_msghdr + +type Cmsghdr C.struct_cmsghdr + +type Inet6Pktinfo C.struct_in6_pktinfo + +type IPv6MTUInfo C.struct_ip6_mtuinfo + +type ICMPv6Filter C.struct_icmp6_filter + +const ( + SizeofSockaddrInet4 = C.sizeof_struct_sockaddr_in + SizeofSockaddrInet6 = C.sizeof_struct_sockaddr_in6 + SizeofSockaddrAny = C.sizeof_struct_sockaddr_any + SizeofSockaddrUnix = C.sizeof_struct_sockaddr_un + SizeofSockaddrDatalink = C.sizeof_struct_sockaddr_dl + SizeofLinger = C.sizeof_struct_linger + SizeofIPMreq = C.sizeof_struct_ip_mreq + SizeofIPv6Mreq = C.sizeof_struct_ipv6_mreq + SizeofMsghdr = C.sizeof_struct_msghdr + SizeofCmsghdr = C.sizeof_struct_cmsghdr + SizeofInet6Pktinfo = C.sizeof_struct_in6_pktinfo + SizeofIPv6MTUInfo = C.sizeof_struct_ip6_mtuinfo + SizeofICMPv6Filter = C.sizeof_struct_icmp6_filter +) + +// Ptrace requests + +const ( + PTRACE_TRACEME = C.PT_TRACE_ME + PTRACE_CONT = C.PT_CONTINUE + PTRACE_KILL = C.PT_KILL +) + +// Events (kqueue, kevent) + +type Kevent_t C.struct_kevent + +// Select + +type FdSet C.fd_set + +// Routing and interface messages + +const ( + SizeofIfMsghdr = C.sizeof_struct_if_msghdr + SizeofIfData = C.sizeof_struct_if_data + SizeofIfaMsghdr = C.sizeof_struct_ifa_msghdr + SizeofIfAnnounceMsghdr = C.sizeof_struct_if_announcemsghdr + SizeofRtMsghdr = C.sizeof_struct_rt_msghdr + SizeofRtMetrics = C.sizeof_struct_rt_metrics +) + +type IfMsghdr C.struct_if_msghdr + +type IfData C.struct_if_data + +type IfaMsghdr C.struct_ifa_msghdr + +type IfAnnounceMsghdr C.struct_if_announcemsghdr + +type RtMsghdr C.struct_rt_msghdr + +type RtMetrics C.struct_rt_metrics + +type Mclpool C.struct_mclpool + +// Berkeley packet filter + +const ( + SizeofBpfVersion = C.sizeof_struct_bpf_version + SizeofBpfStat = C.sizeof_struct_bpf_stat + SizeofBpfProgram = C.sizeof_struct_bpf_program + SizeofBpfInsn = C.sizeof_struct_bpf_insn + SizeofBpfHdr = C.sizeof_struct_bpf_hdr +) + +type BpfVersion C.struct_bpf_version + +type BpfStat C.struct_bpf_stat + +type BpfProgram C.struct_bpf_program + +type BpfInsn C.struct_bpf_insn + +type BpfHdr C.struct_bpf_hdr + +type BpfTimeval C.struct_bpf_timeval + +// Terminal handling + +type Termios C.struct_termios + +type Winsize C.struct_winsize + +type Ptmget C.struct_ptmget + +// fchmodat-like syscalls. + +const ( + AT_FDCWD = C.AT_FDCWD + AT_SYMLINK_NOFOLLOW = C.AT_SYMLINK_NOFOLLOW +) + +// poll + +type PollFd C.struct_pollfd + +const ( + POLLERR = C.POLLERR + POLLHUP = C.POLLHUP + POLLIN = C.POLLIN + POLLNVAL = C.POLLNVAL + POLLOUT = C.POLLOUT + POLLPRI = C.POLLPRI + POLLRDBAND = C.POLLRDBAND + POLLRDNORM = C.POLLRDNORM + POLLWRBAND = C.POLLWRBAND + POLLWRNORM = C.POLLWRNORM +) + +// Sysctl + +type Sysctlnode C.struct_sysctlnode + +// Uname + +type Utsname C.struct_utsname + +// Clockinfo + +const SizeofClockinfo = C.sizeof_struct_clockinfo + +type Clockinfo C.struct_clockinfo diff --git a/vendor/golang.org/x/sys/unix/types_openbsd.go b/vendor/golang.org/x/sys/unix/types_openbsd.go new file mode 100644 index 000000000..4e5e57f9a --- /dev/null +++ b/vendor/golang.org/x/sys/unix/types_openbsd.go @@ -0,0 +1,276 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +/* +Input to cgo -godefs. See README.md +*/ + +// +godefs map struct_in_addr [4]byte /* in_addr */ +// +godefs map struct_in6_addr [16]byte /* in6_addr */ + +package unix + +/* +#define KERNEL +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +enum { + sizeofPtr = sizeof(void*), +}; + +union sockaddr_all { + struct sockaddr s1; // this one gets used for fields + struct sockaddr_in s2; // these pad it out + struct sockaddr_in6 s3; + struct sockaddr_un s4; + struct sockaddr_dl s5; +}; + +struct sockaddr_any { + struct sockaddr addr; + char pad[sizeof(union sockaddr_all) - sizeof(struct sockaddr)]; +}; + +*/ +import "C" + +// Machine characteristics + +const ( + SizeofPtr = C.sizeofPtr + SizeofShort = C.sizeof_short + SizeofInt = C.sizeof_int + SizeofLong = C.sizeof_long + SizeofLongLong = C.sizeof_longlong +) + +// Basic types + +type ( + _C_short C.short + _C_int C.int + _C_long C.long + _C_long_long C.longlong +) + +// Time + +type Timespec C.struct_timespec + +type Timeval C.struct_timeval + +// Processes + +type Rusage C.struct_rusage + +type Rlimit C.struct_rlimit + +type _Gid_t C.gid_t + +// Files + +type Stat_t C.struct_stat + +type Statfs_t C.struct_statfs + +type Flock_t C.struct_flock + +type Dirent C.struct_dirent + +type Fsid C.fsid_t + +// File system limits + +const ( + PathMax = C.PATH_MAX +) + +// Sockets + +type RawSockaddrInet4 C.struct_sockaddr_in + +type RawSockaddrInet6 C.struct_sockaddr_in6 + +type RawSockaddrUnix C.struct_sockaddr_un + +type RawSockaddrDatalink C.struct_sockaddr_dl + +type RawSockaddr C.struct_sockaddr + +type RawSockaddrAny C.struct_sockaddr_any + +type _Socklen C.socklen_t + +type Linger C.struct_linger + +type Iovec C.struct_iovec + +type IPMreq C.struct_ip_mreq + +type IPv6Mreq C.struct_ipv6_mreq + +type Msghdr C.struct_msghdr + +type Cmsghdr C.struct_cmsghdr + +type Inet6Pktinfo C.struct_in6_pktinfo + +type IPv6MTUInfo C.struct_ip6_mtuinfo + +type ICMPv6Filter C.struct_icmp6_filter + +const ( + SizeofSockaddrInet4 = C.sizeof_struct_sockaddr_in + SizeofSockaddrInet6 = C.sizeof_struct_sockaddr_in6 + SizeofSockaddrAny = C.sizeof_struct_sockaddr_any + SizeofSockaddrUnix = C.sizeof_struct_sockaddr_un + SizeofSockaddrDatalink = C.sizeof_struct_sockaddr_dl + SizeofLinger = C.sizeof_struct_linger + SizeofIPMreq = C.sizeof_struct_ip_mreq + SizeofIPv6Mreq = C.sizeof_struct_ipv6_mreq + SizeofMsghdr = C.sizeof_struct_msghdr + SizeofCmsghdr = C.sizeof_struct_cmsghdr + SizeofInet6Pktinfo = C.sizeof_struct_in6_pktinfo + SizeofIPv6MTUInfo = C.sizeof_struct_ip6_mtuinfo + SizeofICMPv6Filter = C.sizeof_struct_icmp6_filter +) + +// Ptrace requests + +const ( + PTRACE_TRACEME = C.PT_TRACE_ME + PTRACE_CONT = C.PT_CONTINUE + PTRACE_KILL = C.PT_KILL +) + +// Events (kqueue, kevent) + +type Kevent_t C.struct_kevent + +// Select + +type FdSet C.fd_set + +// Routing and interface messages + +const ( + SizeofIfMsghdr = C.sizeof_struct_if_msghdr + SizeofIfData = C.sizeof_struct_if_data + SizeofIfaMsghdr = C.sizeof_struct_ifa_msghdr + SizeofIfAnnounceMsghdr = C.sizeof_struct_if_announcemsghdr + SizeofRtMsghdr = C.sizeof_struct_rt_msghdr + SizeofRtMetrics = C.sizeof_struct_rt_metrics +) + +type IfMsghdr C.struct_if_msghdr + +type IfData C.struct_if_data + +type IfaMsghdr C.struct_ifa_msghdr + +type IfAnnounceMsghdr C.struct_if_announcemsghdr + +type RtMsghdr C.struct_rt_msghdr + +type RtMetrics C.struct_rt_metrics + +type Mclpool C.struct_mclpool + +// Berkeley packet filter + +const ( + SizeofBpfVersion = C.sizeof_struct_bpf_version + SizeofBpfStat = C.sizeof_struct_bpf_stat + SizeofBpfProgram = C.sizeof_struct_bpf_program + SizeofBpfInsn = C.sizeof_struct_bpf_insn + SizeofBpfHdr = C.sizeof_struct_bpf_hdr +) + +type BpfVersion C.struct_bpf_version + +type BpfStat C.struct_bpf_stat + +type BpfProgram C.struct_bpf_program + +type BpfInsn C.struct_bpf_insn + +type BpfHdr C.struct_bpf_hdr + +type BpfTimeval C.struct_bpf_timeval + +// Terminal handling + +type Termios C.struct_termios + +type Winsize C.struct_winsize + +// fchmodat-like syscalls. + +const ( + AT_FDCWD = C.AT_FDCWD + AT_SYMLINK_NOFOLLOW = C.AT_SYMLINK_NOFOLLOW +) + +// poll + +type PollFd C.struct_pollfd + +const ( + POLLERR = C.POLLERR + POLLHUP = C.POLLHUP + POLLIN = C.POLLIN + POLLNVAL = C.POLLNVAL + POLLOUT = C.POLLOUT + POLLPRI = C.POLLPRI + POLLRDBAND = C.POLLRDBAND + POLLRDNORM = C.POLLRDNORM + POLLWRBAND = C.POLLWRBAND + POLLWRNORM = C.POLLWRNORM +) + +// Signal Sets + +type Sigset_t C.sigset_t + +// Uname + +type Utsname C.struct_utsname + +// Uvmexp + +const SizeofUvmexp = C.sizeof_struct_uvmexp + +type Uvmexp C.struct_uvmexp diff --git a/vendor/golang.org/x/sys/unix/types_solaris.go b/vendor/golang.org/x/sys/unix/types_solaris.go new file mode 100644 index 000000000..2b716f934 --- /dev/null +++ b/vendor/golang.org/x/sys/unix/types_solaris.go @@ -0,0 +1,266 @@ +// Copyright 2009 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +/* +Input to cgo -godefs. See README.md +*/ + +// +godefs map struct_in_addr [4]byte /* in_addr */ +// +godefs map struct_in6_addr [16]byte /* in6_addr */ + +package unix + +/* +#define KERNEL +// These defines ensure that builds done on newer versions of Solaris are +// backwards-compatible with older versions of Solaris and +// OpenSolaris-based derivatives. +#define __USE_SUNOS_SOCKETS__ // msghdr +#define __USE_LEGACY_PROTOTYPES__ // iovec +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +enum { + sizeofPtr = sizeof(void*), +}; + +union sockaddr_all { + struct sockaddr s1; // this one gets used for fields + struct sockaddr_in s2; // these pad it out + struct sockaddr_in6 s3; + struct sockaddr_un s4; + struct sockaddr_dl s5; +}; + +struct sockaddr_any { + struct sockaddr addr; + char pad[sizeof(union sockaddr_all) - sizeof(struct sockaddr)]; +}; + +*/ +import "C" + +// Machine characteristics + +const ( + SizeofPtr = C.sizeofPtr + SizeofShort = C.sizeof_short + SizeofInt = C.sizeof_int + SizeofLong = C.sizeof_long + SizeofLongLong = C.sizeof_longlong + PathMax = C.PATH_MAX + MaxHostNameLen = C.MAXHOSTNAMELEN +) + +// Basic types + +type ( + _C_short C.short + _C_int C.int + _C_long C.long + _C_long_long C.longlong +) + +// Time + +type Timespec C.struct_timespec + +type Timeval C.struct_timeval + +type Timeval32 C.struct_timeval32 + +type Tms C.struct_tms + +type Utimbuf C.struct_utimbuf + +// Processes + +type Rusage C.struct_rusage + +type Rlimit C.struct_rlimit + +type _Gid_t C.gid_t + +// Files + +type Stat_t C.struct_stat + +type Flock_t C.struct_flock + +type Dirent C.struct_dirent + +// Filesystems + +type _Fsblkcnt_t C.fsblkcnt_t + +type Statvfs_t C.struct_statvfs + +// Sockets + +type RawSockaddrInet4 C.struct_sockaddr_in + +type RawSockaddrInet6 C.struct_sockaddr_in6 + +type RawSockaddrUnix C.struct_sockaddr_un + +type RawSockaddrDatalink C.struct_sockaddr_dl + +type RawSockaddr C.struct_sockaddr + +type RawSockaddrAny C.struct_sockaddr_any + +type _Socklen C.socklen_t + +type Linger C.struct_linger + +type Iovec C.struct_iovec + +type IPMreq C.struct_ip_mreq + +type IPv6Mreq C.struct_ipv6_mreq + +type Msghdr C.struct_msghdr + +type Cmsghdr C.struct_cmsghdr + +type Inet6Pktinfo C.struct_in6_pktinfo + +type IPv6MTUInfo C.struct_ip6_mtuinfo + +type ICMPv6Filter C.struct_icmp6_filter + +const ( + SizeofSockaddrInet4 = C.sizeof_struct_sockaddr_in + SizeofSockaddrInet6 = C.sizeof_struct_sockaddr_in6 + SizeofSockaddrAny = C.sizeof_struct_sockaddr_any + SizeofSockaddrUnix = C.sizeof_struct_sockaddr_un + SizeofSockaddrDatalink = C.sizeof_struct_sockaddr_dl + SizeofLinger = C.sizeof_struct_linger + SizeofIPMreq = C.sizeof_struct_ip_mreq + SizeofIPv6Mreq = C.sizeof_struct_ipv6_mreq + SizeofMsghdr = C.sizeof_struct_msghdr + SizeofCmsghdr = C.sizeof_struct_cmsghdr + SizeofInet6Pktinfo = C.sizeof_struct_in6_pktinfo + SizeofIPv6MTUInfo = C.sizeof_struct_ip6_mtuinfo + SizeofICMPv6Filter = C.sizeof_struct_icmp6_filter +) + +// Select + +type FdSet C.fd_set + +// Misc + +type Utsname C.struct_utsname + +type Ustat_t C.struct_ustat + +const ( + AT_FDCWD = C.AT_FDCWD + AT_SYMLINK_NOFOLLOW = C.AT_SYMLINK_NOFOLLOW + AT_SYMLINK_FOLLOW = C.AT_SYMLINK_FOLLOW + AT_REMOVEDIR = C.AT_REMOVEDIR + AT_EACCESS = C.AT_EACCESS +) + +// Routing and interface messages + +const ( + SizeofIfMsghdr = C.sizeof_struct_if_msghdr + SizeofIfData = C.sizeof_struct_if_data + SizeofIfaMsghdr = C.sizeof_struct_ifa_msghdr + SizeofRtMsghdr = C.sizeof_struct_rt_msghdr + SizeofRtMetrics = C.sizeof_struct_rt_metrics +) + +type IfMsghdr C.struct_if_msghdr + +type IfData C.struct_if_data + +type IfaMsghdr C.struct_ifa_msghdr + +type RtMsghdr C.struct_rt_msghdr + +type RtMetrics C.struct_rt_metrics + +// Berkeley packet filter + +const ( + SizeofBpfVersion = C.sizeof_struct_bpf_version + SizeofBpfStat = C.sizeof_struct_bpf_stat + SizeofBpfProgram = C.sizeof_struct_bpf_program + SizeofBpfInsn = C.sizeof_struct_bpf_insn + SizeofBpfHdr = C.sizeof_struct_bpf_hdr +) + +type BpfVersion C.struct_bpf_version + +type BpfStat C.struct_bpf_stat + +type BpfProgram C.struct_bpf_program + +type BpfInsn C.struct_bpf_insn + +type BpfTimeval C.struct_bpf_timeval + +type BpfHdr C.struct_bpf_hdr + +// Terminal handling + +type Termios C.struct_termios + +type Termio C.struct_termio + +type Winsize C.struct_winsize + +// poll + +type PollFd C.struct_pollfd + +const ( + POLLERR = C.POLLERR + POLLHUP = C.POLLHUP + POLLIN = C.POLLIN + POLLNVAL = C.POLLNVAL + POLLOUT = C.POLLOUT + POLLPRI = C.POLLPRI + POLLRDBAND = C.POLLRDBAND + POLLRDNORM = C.POLLRDNORM + POLLWRBAND = C.POLLWRBAND + POLLWRNORM = C.POLLWRNORM +) diff --git a/vendor/golang.org/x/text/encoding/charmap/maketables.go b/vendor/golang.org/x/text/encoding/charmap/maketables.go new file mode 100644 index 000000000..f7941701e --- /dev/null +++ b/vendor/golang.org/x/text/encoding/charmap/maketables.go @@ -0,0 +1,556 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +package main + +import ( + "bufio" + "fmt" + "log" + "net/http" + "sort" + "strings" + "unicode/utf8" + + "golang.org/x/text/encoding" + "golang.org/x/text/internal/gen" +) + +const ascii = "\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f" + + "\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f" + + ` !"#$%&'()*+,-./0123456789:;<=>?` + + `@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_` + + "`abcdefghijklmnopqrstuvwxyz{|}~\u007f" + +var encodings = []struct { + name string + mib string + comment string + varName string + replacement byte + mapping string +}{ + { + "IBM Code Page 037", + "IBM037", + "", + "CodePage037", + 0x3f, + "http://source.icu-project.org/repos/icu/data/trunk/charset/data/ucm/glibc-IBM037-2.1.2.ucm", + }, + { + "IBM Code Page 437", + "PC8CodePage437", + "", + "CodePage437", + encoding.ASCIISub, + "http://source.icu-project.org/repos/icu/data/trunk/charset/data/ucm/glibc-IBM437-2.1.2.ucm", + }, + { + "IBM Code Page 850", + "PC850Multilingual", + "", + "CodePage850", + encoding.ASCIISub, + "http://source.icu-project.org/repos/icu/data/trunk/charset/data/ucm/glibc-IBM850-2.1.2.ucm", + }, + { + "IBM Code Page 852", + "PCp852", + "", + "CodePage852", + encoding.ASCIISub, + "http://source.icu-project.org/repos/icu/data/trunk/charset/data/ucm/glibc-IBM852-2.1.2.ucm", + }, + { + "IBM Code Page 855", + "IBM855", + "", + "CodePage855", + encoding.ASCIISub, + "http://source.icu-project.org/repos/icu/data/trunk/charset/data/ucm/glibc-IBM855-2.1.2.ucm", + }, + { + "Windows Code Page 858", // PC latin1 with Euro + "IBM00858", + "", + "CodePage858", + encoding.ASCIISub, + "http://source.icu-project.org/repos/icu/data/trunk/charset/data/ucm/windows-858-2000.ucm", + }, + { + "IBM Code Page 860", + "IBM860", + "", + "CodePage860", + encoding.ASCIISub, + "http://source.icu-project.org/repos/icu/data/trunk/charset/data/ucm/glibc-IBM860-2.1.2.ucm", + }, + { + "IBM Code Page 862", + "PC862LatinHebrew", + "", + "CodePage862", + encoding.ASCIISub, + "http://source.icu-project.org/repos/icu/data/trunk/charset/data/ucm/glibc-IBM862-2.1.2.ucm", + }, + { + "IBM Code Page 863", + "IBM863", + "", + "CodePage863", + encoding.ASCIISub, + "http://source.icu-project.org/repos/icu/data/trunk/charset/data/ucm/glibc-IBM863-2.1.2.ucm", + }, + { + "IBM Code Page 865", + "IBM865", + "", + "CodePage865", + encoding.ASCIISub, + "http://source.icu-project.org/repos/icu/data/trunk/charset/data/ucm/glibc-IBM865-2.1.2.ucm", + }, + { + "IBM Code Page 866", + "IBM866", + "", + "CodePage866", + encoding.ASCIISub, + "http://encoding.spec.whatwg.org/index-ibm866.txt", + }, + { + "IBM Code Page 1047", + "IBM1047", + "", + "CodePage1047", + 0x3f, + "http://source.icu-project.org/repos/icu/data/trunk/charset/data/ucm/glibc-IBM1047-2.1.2.ucm", + }, + { + "IBM Code Page 1140", + "IBM01140", + "", + "CodePage1140", + 0x3f, + "http://source.icu-project.org/repos/icu/data/trunk/charset/data/ucm/ibm-1140_P100-1997.ucm", + }, + { + "ISO 8859-1", + "ISOLatin1", + "", + "ISO8859_1", + encoding.ASCIISub, + "http://source.icu-project.org/repos/icu/data/trunk/charset/data/ucm/iso-8859_1-1998.ucm", + }, + { + "ISO 8859-2", + "ISOLatin2", + "", + "ISO8859_2", + encoding.ASCIISub, + "http://encoding.spec.whatwg.org/index-iso-8859-2.txt", + }, + { + "ISO 8859-3", + "ISOLatin3", + "", + "ISO8859_3", + encoding.ASCIISub, + "http://encoding.spec.whatwg.org/index-iso-8859-3.txt", + }, + { + "ISO 8859-4", + "ISOLatin4", + "", + "ISO8859_4", + encoding.ASCIISub, + "http://encoding.spec.whatwg.org/index-iso-8859-4.txt", + }, + { + "ISO 8859-5", + "ISOLatinCyrillic", + "", + "ISO8859_5", + encoding.ASCIISub, + "http://encoding.spec.whatwg.org/index-iso-8859-5.txt", + }, + { + "ISO 8859-6", + "ISOLatinArabic", + "", + "ISO8859_6,ISO8859_6E,ISO8859_6I", + encoding.ASCIISub, + "http://encoding.spec.whatwg.org/index-iso-8859-6.txt", + }, + { + "ISO 8859-7", + "ISOLatinGreek", + "", + "ISO8859_7", + encoding.ASCIISub, + "http://encoding.spec.whatwg.org/index-iso-8859-7.txt", + }, + { + "ISO 8859-8", + "ISOLatinHebrew", + "", + "ISO8859_8,ISO8859_8E,ISO8859_8I", + encoding.ASCIISub, + "http://encoding.spec.whatwg.org/index-iso-8859-8.txt", + }, + { + "ISO 8859-9", + "ISOLatin5", + "", + "ISO8859_9", + encoding.ASCIISub, + "http://source.icu-project.org/repos/icu/data/trunk/charset/data/ucm/iso-8859_9-1999.ucm", + }, + { + "ISO 8859-10", + "ISOLatin6", + "", + "ISO8859_10", + encoding.ASCIISub, + "http://encoding.spec.whatwg.org/index-iso-8859-10.txt", + }, + { + "ISO 8859-13", + "ISO885913", + "", + "ISO8859_13", + encoding.ASCIISub, + "http://encoding.spec.whatwg.org/index-iso-8859-13.txt", + }, + { + "ISO 8859-14", + "ISO885914", + "", + "ISO8859_14", + encoding.ASCIISub, + "http://encoding.spec.whatwg.org/index-iso-8859-14.txt", + }, + { + "ISO 8859-15", + "ISO885915", + "", + "ISO8859_15", + encoding.ASCIISub, + "http://encoding.spec.whatwg.org/index-iso-8859-15.txt", + }, + { + "ISO 8859-16", + "ISO885916", + "", + "ISO8859_16", + encoding.ASCIISub, + "http://encoding.spec.whatwg.org/index-iso-8859-16.txt", + }, + { + "KOI8-R", + "KOI8R", + "", + "KOI8R", + encoding.ASCIISub, + "http://encoding.spec.whatwg.org/index-koi8-r.txt", + }, + { + "KOI8-U", + "KOI8U", + "", + "KOI8U", + encoding.ASCIISub, + "http://encoding.spec.whatwg.org/index-koi8-u.txt", + }, + { + "Macintosh", + "Macintosh", + "", + "Macintosh", + encoding.ASCIISub, + "http://encoding.spec.whatwg.org/index-macintosh.txt", + }, + { + "Macintosh Cyrillic", + "MacintoshCyrillic", + "", + "MacintoshCyrillic", + encoding.ASCIISub, + "http://encoding.spec.whatwg.org/index-x-mac-cyrillic.txt", + }, + { + "Windows 874", + "Windows874", + "", + "Windows874", + encoding.ASCIISub, + "http://encoding.spec.whatwg.org/index-windows-874.txt", + }, + { + "Windows 1250", + "Windows1250", + "", + "Windows1250", + encoding.ASCIISub, + "http://encoding.spec.whatwg.org/index-windows-1250.txt", + }, + { + "Windows 1251", + "Windows1251", + "", + "Windows1251", + encoding.ASCIISub, + "http://encoding.spec.whatwg.org/index-windows-1251.txt", + }, + { + "Windows 1252", + "Windows1252", + "", + "Windows1252", + encoding.ASCIISub, + "http://encoding.spec.whatwg.org/index-windows-1252.txt", + }, + { + "Windows 1253", + "Windows1253", + "", + "Windows1253", + encoding.ASCIISub, + "http://encoding.spec.whatwg.org/index-windows-1253.txt", + }, + { + "Windows 1254", + "Windows1254", + "", + "Windows1254", + encoding.ASCIISub, + "http://encoding.spec.whatwg.org/index-windows-1254.txt", + }, + { + "Windows 1255", + "Windows1255", + "", + "Windows1255", + encoding.ASCIISub, + "http://encoding.spec.whatwg.org/index-windows-1255.txt", + }, + { + "Windows 1256", + "Windows1256", + "", + "Windows1256", + encoding.ASCIISub, + "http://encoding.spec.whatwg.org/index-windows-1256.txt", + }, + { + "Windows 1257", + "Windows1257", + "", + "Windows1257", + encoding.ASCIISub, + "http://encoding.spec.whatwg.org/index-windows-1257.txt", + }, + { + "Windows 1258", + "Windows1258", + "", + "Windows1258", + encoding.ASCIISub, + "http://encoding.spec.whatwg.org/index-windows-1258.txt", + }, + { + "X-User-Defined", + "XUserDefined", + "It is defined at http://encoding.spec.whatwg.org/#x-user-defined", + "XUserDefined", + encoding.ASCIISub, + ascii + + "\uf780\uf781\uf782\uf783\uf784\uf785\uf786\uf787" + + "\uf788\uf789\uf78a\uf78b\uf78c\uf78d\uf78e\uf78f" + + "\uf790\uf791\uf792\uf793\uf794\uf795\uf796\uf797" + + "\uf798\uf799\uf79a\uf79b\uf79c\uf79d\uf79e\uf79f" + + "\uf7a0\uf7a1\uf7a2\uf7a3\uf7a4\uf7a5\uf7a6\uf7a7" + + "\uf7a8\uf7a9\uf7aa\uf7ab\uf7ac\uf7ad\uf7ae\uf7af" + + "\uf7b0\uf7b1\uf7b2\uf7b3\uf7b4\uf7b5\uf7b6\uf7b7" + + "\uf7b8\uf7b9\uf7ba\uf7bb\uf7bc\uf7bd\uf7be\uf7bf" + + "\uf7c0\uf7c1\uf7c2\uf7c3\uf7c4\uf7c5\uf7c6\uf7c7" + + "\uf7c8\uf7c9\uf7ca\uf7cb\uf7cc\uf7cd\uf7ce\uf7cf" + + "\uf7d0\uf7d1\uf7d2\uf7d3\uf7d4\uf7d5\uf7d6\uf7d7" + + "\uf7d8\uf7d9\uf7da\uf7db\uf7dc\uf7dd\uf7de\uf7df" + + "\uf7e0\uf7e1\uf7e2\uf7e3\uf7e4\uf7e5\uf7e6\uf7e7" + + "\uf7e8\uf7e9\uf7ea\uf7eb\uf7ec\uf7ed\uf7ee\uf7ef" + + "\uf7f0\uf7f1\uf7f2\uf7f3\uf7f4\uf7f5\uf7f6\uf7f7" + + "\uf7f8\uf7f9\uf7fa\uf7fb\uf7fc\uf7fd\uf7fe\uf7ff", + }, +} + +func getWHATWG(url string) string { + res, err := http.Get(url) + if err != nil { + log.Fatalf("%q: Get: %v", url, err) + } + defer res.Body.Close() + + mapping := make([]rune, 128) + for i := range mapping { + mapping[i] = '\ufffd' + } + + scanner := bufio.NewScanner(res.Body) + for scanner.Scan() { + s := strings.TrimSpace(scanner.Text()) + if s == "" || s[0] == '#' { + continue + } + x, y := 0, 0 + if _, err := fmt.Sscanf(s, "%d\t0x%x", &x, &y); err != nil { + log.Fatalf("could not parse %q", s) + } + if x < 0 || 128 <= x { + log.Fatalf("code %d is out of range", x) + } + if 0x80 <= y && y < 0xa0 { + // We diverge from the WHATWG spec by mapping control characters + // in the range [0x80, 0xa0) to U+FFFD. + continue + } + mapping[x] = rune(y) + } + return ascii + string(mapping) +} + +func getUCM(url string) string { + res, err := http.Get(url) + if err != nil { + log.Fatalf("%q: Get: %v", url, err) + } + defer res.Body.Close() + + mapping := make([]rune, 256) + for i := range mapping { + mapping[i] = '\ufffd' + } + + charsFound := 0 + scanner := bufio.NewScanner(res.Body) + for scanner.Scan() { + s := strings.TrimSpace(scanner.Text()) + if s == "" || s[0] == '#' { + continue + } + var c byte + var r rune + if _, err := fmt.Sscanf(s, ` \x%x |0`, &r, &c); err != nil { + continue + } + mapping[c] = r + charsFound++ + } + + if charsFound < 200 { + log.Fatalf("%q: only %d characters found (wrong page format?)", url, charsFound) + } + + return string(mapping) +} + +func main() { + mibs := map[string]bool{} + all := []string{} + + w := gen.NewCodeWriter() + defer w.WriteGoFile("tables.go", "charmap") + + printf := func(s string, a ...interface{}) { fmt.Fprintf(w, s, a...) } + + printf("import (\n") + printf("\t\"golang.org/x/text/encoding\"\n") + printf("\t\"golang.org/x/text/encoding/internal/identifier\"\n") + printf(")\n\n") + for _, e := range encodings { + varNames := strings.Split(e.varName, ",") + all = append(all, varNames...) + varName := varNames[0] + switch { + case strings.HasPrefix(e.mapping, "http://encoding.spec.whatwg.org/"): + e.mapping = getWHATWG(e.mapping) + case strings.HasPrefix(e.mapping, "http://source.icu-project.org/repos/icu/data/trunk/charset/data/ucm/"): + e.mapping = getUCM(e.mapping) + } + + asciiSuperset, low := strings.HasPrefix(e.mapping, ascii), 0x00 + if asciiSuperset { + low = 0x80 + } + lvn := 1 + if strings.HasPrefix(varName, "ISO") || strings.HasPrefix(varName, "KOI") { + lvn = 3 + } + lowerVarName := strings.ToLower(varName[:lvn]) + varName[lvn:] + printf("// %s is the %s encoding.\n", varName, e.name) + if e.comment != "" { + printf("//\n// %s\n", e.comment) + } + printf("var %s *Charmap = &%s\n\nvar %s = Charmap{\nname: %q,\n", + varName, lowerVarName, lowerVarName, e.name) + if mibs[e.mib] { + log.Fatalf("MIB type %q declared multiple times.", e.mib) + } + printf("mib: identifier.%s,\n", e.mib) + printf("asciiSuperset: %t,\n", asciiSuperset) + printf("low: 0x%02x,\n", low) + printf("replacement: 0x%02x,\n", e.replacement) + + printf("decode: [256]utf8Enc{\n") + i, backMapping := 0, map[rune]byte{} + for _, c := range e.mapping { + if _, ok := backMapping[c]; !ok && c != utf8.RuneError { + backMapping[c] = byte(i) + } + var buf [8]byte + n := utf8.EncodeRune(buf[:], c) + if n > 3 { + panic(fmt.Sprintf("rune %q (%U) is too long", c, c)) + } + printf("{%d,[3]byte{0x%02x,0x%02x,0x%02x}},", n, buf[0], buf[1], buf[2]) + if i%2 == 1 { + printf("\n") + } + i++ + } + printf("},\n") + + printf("encode: [256]uint32{\n") + encode := make([]uint32, 0, 256) + for c, i := range backMapping { + encode = append(encode, uint32(i)<<24|uint32(c)) + } + sort.Sort(byRune(encode)) + for len(encode) < cap(encode) { + encode = append(encode, encode[len(encode)-1]) + } + for i, enc := range encode { + printf("0x%08x,", enc) + if i%8 == 7 { + printf("\n") + } + } + printf("},\n}\n") + + // Add an estimate of the size of a single Charmap{} struct value, which + // includes two 256 elem arrays of 4 bytes and some extra fields, which + // align to 3 uint64s on 64-bit architectures. + w.Size += 2*4*256 + 3*8 + } + // TODO: add proper line breaking. + printf("var listAll = []encoding.Encoding{\n%s,\n}\n\n", strings.Join(all, ",\n")) +} + +type byRune []uint32 + +func (b byRune) Len() int { return len(b) } +func (b byRune) Less(i, j int) bool { return b[i]&0xffffff < b[j]&0xffffff } +func (b byRune) Swap(i, j int) { b[i], b[j] = b[j], b[i] } diff --git a/vendor/golang.org/x/text/encoding/htmlindex/gen.go b/vendor/golang.org/x/text/encoding/htmlindex/gen.go new file mode 100644 index 000000000..ac6b4a77f --- /dev/null +++ b/vendor/golang.org/x/text/encoding/htmlindex/gen.go @@ -0,0 +1,173 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +package main + +import ( + "bytes" + "encoding/json" + "fmt" + "log" + "strings" + + "golang.org/x/text/internal/gen" +) + +type group struct { + Encodings []struct { + Labels []string + Name string + } +} + +func main() { + gen.Init() + + r := gen.Open("https://encoding.spec.whatwg.org", "whatwg", "encodings.json") + var groups []group + if err := json.NewDecoder(r).Decode(&groups); err != nil { + log.Fatalf("Error reading encodings.json: %v", err) + } + + w := &bytes.Buffer{} + fmt.Fprintln(w, "type htmlEncoding byte") + fmt.Fprintln(w, "const (") + for i, g := range groups { + for _, e := range g.Encodings { + key := strings.ToLower(e.Name) + name := consts[key] + if name == "" { + log.Fatalf("No const defined for %s.", key) + } + if i == 0 { + fmt.Fprintf(w, "%s htmlEncoding = iota\n", name) + } else { + fmt.Fprintf(w, "%s\n", name) + } + } + } + fmt.Fprintln(w, "numEncodings") + fmt.Fprint(w, ")\n\n") + + fmt.Fprintln(w, "var canonical = [numEncodings]string{") + for _, g := range groups { + for _, e := range g.Encodings { + fmt.Fprintf(w, "%q,\n", strings.ToLower(e.Name)) + } + } + fmt.Fprint(w, "}\n\n") + + fmt.Fprintln(w, "var nameMap = map[string]htmlEncoding{") + for _, g := range groups { + for _, e := range g.Encodings { + for _, l := range e.Labels { + key := strings.ToLower(e.Name) + name := consts[key] + fmt.Fprintf(w, "%q: %s,\n", l, name) + } + } + } + fmt.Fprint(w, "}\n\n") + + var tags []string + fmt.Fprintln(w, "var localeMap = []htmlEncoding{") + for _, loc := range locales { + tags = append(tags, loc.tag) + fmt.Fprintf(w, "%s, // %s \n", consts[loc.name], loc.tag) + } + fmt.Fprint(w, "}\n\n") + + fmt.Fprintf(w, "const locales = %q\n", strings.Join(tags, " ")) + + gen.WriteGoFile("tables.go", "htmlindex", w.Bytes()) +} + +// consts maps canonical encoding name to internal constant. +var consts = map[string]string{ + "utf-8": "utf8", + "ibm866": "ibm866", + "iso-8859-2": "iso8859_2", + "iso-8859-3": "iso8859_3", + "iso-8859-4": "iso8859_4", + "iso-8859-5": "iso8859_5", + "iso-8859-6": "iso8859_6", + "iso-8859-7": "iso8859_7", + "iso-8859-8": "iso8859_8", + "iso-8859-8-i": "iso8859_8I", + "iso-8859-10": "iso8859_10", + "iso-8859-13": "iso8859_13", + "iso-8859-14": "iso8859_14", + "iso-8859-15": "iso8859_15", + "iso-8859-16": "iso8859_16", + "koi8-r": "koi8r", + "koi8-u": "koi8u", + "macintosh": "macintosh", + "windows-874": "windows874", + "windows-1250": "windows1250", + "windows-1251": "windows1251", + "windows-1252": "windows1252", + "windows-1253": "windows1253", + "windows-1254": "windows1254", + "windows-1255": "windows1255", + "windows-1256": "windows1256", + "windows-1257": "windows1257", + "windows-1258": "windows1258", + "x-mac-cyrillic": "macintoshCyrillic", + "gbk": "gbk", + "gb18030": "gb18030", + // "hz-gb-2312": "hzgb2312", // Was removed from WhatWG + "big5": "big5", + "euc-jp": "eucjp", + "iso-2022-jp": "iso2022jp", + "shift_jis": "shiftJIS", + "euc-kr": "euckr", + "replacement": "replacement", + "utf-16be": "utf16be", + "utf-16le": "utf16le", + "x-user-defined": "xUserDefined", +} + +// locales is taken from +// https://html.spec.whatwg.org/multipage/syntax.html#encoding-sniffing-algorithm. +var locales = []struct{ tag, name string }{ + // The default value. Explicitly state latin to benefit from the exact + // script option, while still making 1252 the default encoding for languages + // written in Latin script. + {"und_Latn", "windows-1252"}, + {"ar", "windows-1256"}, + {"ba", "windows-1251"}, + {"be", "windows-1251"}, + {"bg", "windows-1251"}, + {"cs", "windows-1250"}, + {"el", "iso-8859-7"}, + {"et", "windows-1257"}, + {"fa", "windows-1256"}, + {"he", "windows-1255"}, + {"hr", "windows-1250"}, + {"hu", "iso-8859-2"}, + {"ja", "shift_jis"}, + {"kk", "windows-1251"}, + {"ko", "euc-kr"}, + {"ku", "windows-1254"}, + {"ky", "windows-1251"}, + {"lt", "windows-1257"}, + {"lv", "windows-1257"}, + {"mk", "windows-1251"}, + {"pl", "iso-8859-2"}, + {"ru", "windows-1251"}, + {"sah", "windows-1251"}, + {"sk", "windows-1250"}, + {"sl", "iso-8859-2"}, + {"sr", "windows-1251"}, + {"tg", "windows-1251"}, + {"th", "windows-874"}, + {"tr", "windows-1254"}, + {"tt", "windows-1251"}, + {"uk", "windows-1251"}, + {"vi", "windows-1258"}, + {"zh-hans", "gb18030"}, + {"zh-hant", "big5"}, +} diff --git a/vendor/golang.org/x/text/encoding/internal/identifier/gen.go b/vendor/golang.org/x/text/encoding/internal/identifier/gen.go new file mode 100644 index 000000000..0c8eba7e5 --- /dev/null +++ b/vendor/golang.org/x/text/encoding/internal/identifier/gen.go @@ -0,0 +1,137 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +package main + +import ( + "bytes" + "encoding/xml" + "fmt" + "io" + "log" + "strings" + + "golang.org/x/text/internal/gen" +) + +type registry struct { + XMLName xml.Name `xml:"registry"` + Updated string `xml:"updated"` + Registry []struct { + ID string `xml:"id,attr"` + Record []struct { + Name string `xml:"name"` + Xref []struct { + Type string `xml:"type,attr"` + Data string `xml:"data,attr"` + } `xml:"xref"` + Desc struct { + Data string `xml:",innerxml"` + // Any []struct { + // Data string `xml:",chardata"` + // } `xml:",any"` + // Data string `xml:",chardata"` + } `xml:"description,"` + MIB string `xml:"value"` + Alias []string `xml:"alias"` + MIME string `xml:"preferred_alias"` + } `xml:"record"` + } `xml:"registry"` +} + +func main() { + r := gen.OpenIANAFile("assignments/character-sets/character-sets.xml") + reg := ®istry{} + if err := xml.NewDecoder(r).Decode(®); err != nil && err != io.EOF { + log.Fatalf("Error decoding charset registry: %v", err) + } + if len(reg.Registry) == 0 || reg.Registry[0].ID != "character-sets-1" { + log.Fatalf("Unexpected ID %s", reg.Registry[0].ID) + } + + w := &bytes.Buffer{} + fmt.Fprintf(w, "const (\n") + for _, rec := range reg.Registry[0].Record { + constName := "" + for _, a := range rec.Alias { + if strings.HasPrefix(a, "cs") && strings.IndexByte(a, '-') == -1 { + // Some of the constant definitions have comments in them. Strip those. + constName = strings.Title(strings.SplitN(a[2:], "\n", 2)[0]) + } + } + if constName == "" { + switch rec.MIB { + case "2085": + constName = "HZGB2312" // Not listed as alias for some reason. + default: + log.Fatalf("No cs alias defined for %s.", rec.MIB) + } + } + if rec.MIME != "" { + rec.MIME = fmt.Sprintf(" (MIME: %s)", rec.MIME) + } + fmt.Fprintf(w, "// %s is the MIB identifier with IANA name %s%s.\n//\n", constName, rec.Name, rec.MIME) + if len(rec.Desc.Data) > 0 { + fmt.Fprint(w, "// ") + d := xml.NewDecoder(strings.NewReader(rec.Desc.Data)) + inElem := true + attr := "" + for { + t, err := d.Token() + if err != nil { + if err != io.EOF { + log.Fatal(err) + } + break + } + switch x := t.(type) { + case xml.CharData: + attr = "" // Don't need attribute info. + a := bytes.Split([]byte(x), []byte("\n")) + for i, b := range a { + if b = bytes.TrimSpace(b); len(b) != 0 { + if !inElem && i > 0 { + fmt.Fprint(w, "\n// ") + } + inElem = false + fmt.Fprintf(w, "%s ", string(b)) + } + } + case xml.StartElement: + if x.Name.Local == "xref" { + inElem = true + use := false + for _, a := range x.Attr { + if a.Name.Local == "type" { + use = use || a.Value != "person" + } + if a.Name.Local == "data" && use { + attr = a.Value + " " + } + } + } + case xml.EndElement: + inElem = false + fmt.Fprint(w, attr) + } + } + fmt.Fprint(w, "\n") + } + for _, x := range rec.Xref { + switch x.Type { + case "rfc": + fmt.Fprintf(w, "// Reference: %s\n", strings.ToUpper(x.Data)) + case "uri": + fmt.Fprintf(w, "// Reference: %s\n", x.Data) + } + } + fmt.Fprintf(w, "%s MIB = %s\n", constName, rec.MIB) + fmt.Fprintln(w) + } + fmt.Fprintln(w, ")") + + gen.WriteGoFile("mib.go", "identifier", w.Bytes()) +} diff --git a/vendor/golang.org/x/text/encoding/japanese/maketables.go b/vendor/golang.org/x/text/encoding/japanese/maketables.go new file mode 100644 index 000000000..d6c10deb0 --- /dev/null +++ b/vendor/golang.org/x/text/encoding/japanese/maketables.go @@ -0,0 +1,161 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +package main + +// This program generates tables.go: +// go run maketables.go | gofmt > tables.go + +// TODO: Emoji extensions? +// http://www.unicode.org/faq/emoji_dingbats.html +// http://www.unicode.org/Public/UNIDATA/EmojiSources.txt + +import ( + "bufio" + "fmt" + "log" + "net/http" + "sort" + "strings" +) + +type entry struct { + jisCode, table int +} + +func main() { + fmt.Printf("// generated by go run maketables.go; DO NOT EDIT\n\n") + fmt.Printf("// Package japanese provides Japanese encodings such as EUC-JP and Shift JIS.\n") + fmt.Printf(`package japanese // import "golang.org/x/text/encoding/japanese"` + "\n\n") + + reverse := [65536]entry{} + for i := range reverse { + reverse[i].table = -1 + } + + tables := []struct { + url string + name string + }{ + {"http://encoding.spec.whatwg.org/index-jis0208.txt", "0208"}, + {"http://encoding.spec.whatwg.org/index-jis0212.txt", "0212"}, + } + for i, table := range tables { + res, err := http.Get(table.url) + if err != nil { + log.Fatalf("%q: Get: %v", table.url, err) + } + defer res.Body.Close() + + mapping := [65536]uint16{} + + scanner := bufio.NewScanner(res.Body) + for scanner.Scan() { + s := strings.TrimSpace(scanner.Text()) + if s == "" || s[0] == '#' { + continue + } + x, y := 0, uint16(0) + if _, err := fmt.Sscanf(s, "%d 0x%x", &x, &y); err != nil { + log.Fatalf("%q: could not parse %q", table.url, s) + } + if x < 0 || 120*94 <= x { + log.Fatalf("%q: JIS code %d is out of range", table.url, x) + } + mapping[x] = y + if reverse[y].table == -1 { + reverse[y] = entry{jisCode: x, table: i} + } + } + if err := scanner.Err(); err != nil { + log.Fatalf("%q: scanner error: %v", table.url, err) + } + + fmt.Printf("// jis%sDecode is the decoding table from JIS %s code to Unicode.\n// It is defined at %s\n", + table.name, table.name, table.url) + fmt.Printf("var jis%sDecode = [...]uint16{\n", table.name) + for i, m := range mapping { + if m != 0 { + fmt.Printf("\t%d: 0x%04X,\n", i, m) + } + } + fmt.Printf("}\n\n") + } + + // Any run of at least separation continuous zero entries in the reverse map will + // be a separate encode table. + const separation = 1024 + + intervals := []interval(nil) + low, high := -1, -1 + for i, v := range reverse { + if v.table == -1 { + continue + } + if low < 0 { + low = i + } else if i-high >= separation { + if high >= 0 { + intervals = append(intervals, interval{low, high}) + } + low = i + } + high = i + 1 + } + if high >= 0 { + intervals = append(intervals, interval{low, high}) + } + sort.Sort(byDecreasingLength(intervals)) + + fmt.Printf("const (\n") + fmt.Printf("\tjis0208 = 1\n") + fmt.Printf("\tjis0212 = 2\n") + fmt.Printf("\tcodeMask = 0x7f\n") + fmt.Printf("\tcodeShift = 7\n") + fmt.Printf("\ttableShift = 14\n") + fmt.Printf(")\n\n") + + fmt.Printf("const numEncodeTables = %d\n\n", len(intervals)) + fmt.Printf("// encodeX are the encoding tables from Unicode to JIS code,\n") + fmt.Printf("// sorted by decreasing length.\n") + for i, v := range intervals { + fmt.Printf("// encode%d: %5d entries for runes in [%5d, %5d).\n", i, v.len(), v.low, v.high) + } + fmt.Printf("//\n") + fmt.Printf("// The high two bits of the value record whether the JIS code comes from the\n") + fmt.Printf("// JIS0208 table (high bits == 1) or the JIS0212 table (high bits == 2).\n") + fmt.Printf("// The low 14 bits are two 7-bit unsigned integers j1 and j2 that form the\n") + fmt.Printf("// JIS code (94*j1 + j2) within that table.\n") + fmt.Printf("\n") + + for i, v := range intervals { + fmt.Printf("const encode%dLow, encode%dHigh = %d, %d\n\n", i, i, v.low, v.high) + fmt.Printf("var encode%d = [...]uint16{\n", i) + for j := v.low; j < v.high; j++ { + x := reverse[j] + if x.table == -1 { + continue + } + fmt.Printf("\t%d - %d: jis%s<<14 | 0x%02X<<7 | 0x%02X,\n", + j, v.low, tables[x.table].name, x.jisCode/94, x.jisCode%94) + } + fmt.Printf("}\n\n") + } +} + +// interval is a half-open interval [low, high). +type interval struct { + low, high int +} + +func (i interval) len() int { return i.high - i.low } + +// byDecreasingLength sorts intervals by decreasing length. +type byDecreasingLength []interval + +func (b byDecreasingLength) Len() int { return len(b) } +func (b byDecreasingLength) Less(i, j int) bool { return b[i].len() > b[j].len() } +func (b byDecreasingLength) Swap(i, j int) { b[i], b[j] = b[j], b[i] } diff --git a/vendor/golang.org/x/text/encoding/korean/maketables.go b/vendor/golang.org/x/text/encoding/korean/maketables.go new file mode 100644 index 000000000..c84034fb6 --- /dev/null +++ b/vendor/golang.org/x/text/encoding/korean/maketables.go @@ -0,0 +1,143 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +package main + +// This program generates tables.go: +// go run maketables.go | gofmt > tables.go + +import ( + "bufio" + "fmt" + "log" + "net/http" + "sort" + "strings" +) + +func main() { + fmt.Printf("// generated by go run maketables.go; DO NOT EDIT\n\n") + fmt.Printf("// Package korean provides Korean encodings such as EUC-KR.\n") + fmt.Printf(`package korean // import "golang.org/x/text/encoding/korean"` + "\n\n") + + res, err := http.Get("http://encoding.spec.whatwg.org/index-euc-kr.txt") + if err != nil { + log.Fatalf("Get: %v", err) + } + defer res.Body.Close() + + mapping := [65536]uint16{} + reverse := [65536]uint16{} + + scanner := bufio.NewScanner(res.Body) + for scanner.Scan() { + s := strings.TrimSpace(scanner.Text()) + if s == "" || s[0] == '#' { + continue + } + x, y := uint16(0), uint16(0) + if _, err := fmt.Sscanf(s, "%d 0x%x", &x, &y); err != nil { + log.Fatalf("could not parse %q", s) + } + if x < 0 || 178*(0xc7-0x81)+(0xfe-0xc7)*94+(0xff-0xa1) <= x { + log.Fatalf("EUC-KR code %d is out of range", x) + } + mapping[x] = y + if reverse[y] == 0 { + c0, c1 := uint16(0), uint16(0) + if x < 178*(0xc7-0x81) { + c0 = uint16(x/178) + 0x81 + c1 = uint16(x % 178) + switch { + case c1 < 1*26: + c1 += 0x41 + case c1 < 2*26: + c1 += 0x47 + default: + c1 += 0x4d + } + } else { + x -= 178 * (0xc7 - 0x81) + c0 = uint16(x/94) + 0xc7 + c1 = uint16(x%94) + 0xa1 + } + reverse[y] = c0<<8 | c1 + } + } + if err := scanner.Err(); err != nil { + log.Fatalf("scanner error: %v", err) + } + + fmt.Printf("// decode is the decoding table from EUC-KR code to Unicode.\n") + fmt.Printf("// It is defined at http://encoding.spec.whatwg.org/index-euc-kr.txt\n") + fmt.Printf("var decode = [...]uint16{\n") + for i, v := range mapping { + if v != 0 { + fmt.Printf("\t%d: 0x%04X,\n", i, v) + } + } + fmt.Printf("}\n\n") + + // Any run of at least separation continuous zero entries in the reverse map will + // be a separate encode table. + const separation = 1024 + + intervals := []interval(nil) + low, high := -1, -1 + for i, v := range reverse { + if v == 0 { + continue + } + if low < 0 { + low = i + } else if i-high >= separation { + if high >= 0 { + intervals = append(intervals, interval{low, high}) + } + low = i + } + high = i + 1 + } + if high >= 0 { + intervals = append(intervals, interval{low, high}) + } + sort.Sort(byDecreasingLength(intervals)) + + fmt.Printf("const numEncodeTables = %d\n\n", len(intervals)) + fmt.Printf("// encodeX are the encoding tables from Unicode to EUC-KR code,\n") + fmt.Printf("// sorted by decreasing length.\n") + for i, v := range intervals { + fmt.Printf("// encode%d: %5d entries for runes in [%5d, %5d).\n", i, v.len(), v.low, v.high) + } + fmt.Printf("\n") + + for i, v := range intervals { + fmt.Printf("const encode%dLow, encode%dHigh = %d, %d\n\n", i, i, v.low, v.high) + fmt.Printf("var encode%d = [...]uint16{\n", i) + for j := v.low; j < v.high; j++ { + x := reverse[j] + if x == 0 { + continue + } + fmt.Printf("\t%d-%d: 0x%04X,\n", j, v.low, x) + } + fmt.Printf("}\n\n") + } +} + +// interval is a half-open interval [low, high). +type interval struct { + low, high int +} + +func (i interval) len() int { return i.high - i.low } + +// byDecreasingLength sorts intervals by decreasing length. +type byDecreasingLength []interval + +func (b byDecreasingLength) Len() int { return len(b) } +func (b byDecreasingLength) Less(i, j int) bool { return b[i].len() > b[j].len() } +func (b byDecreasingLength) Swap(i, j int) { b[i], b[j] = b[j], b[i] } diff --git a/vendor/golang.org/x/text/encoding/simplifiedchinese/maketables.go b/vendor/golang.org/x/text/encoding/simplifiedchinese/maketables.go new file mode 100644 index 000000000..55016c786 --- /dev/null +++ b/vendor/golang.org/x/text/encoding/simplifiedchinese/maketables.go @@ -0,0 +1,161 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +package main + +// This program generates tables.go: +// go run maketables.go | gofmt > tables.go + +import ( + "bufio" + "fmt" + "log" + "net/http" + "sort" + "strings" +) + +func main() { + fmt.Printf("// generated by go run maketables.go; DO NOT EDIT\n\n") + fmt.Printf("// Package simplifiedchinese provides Simplified Chinese encodings such as GBK.\n") + fmt.Printf(`package simplifiedchinese // import "golang.org/x/text/encoding/simplifiedchinese"` + "\n\n") + + printGB18030() + printGBK() +} + +func printGB18030() { + res, err := http.Get("http://encoding.spec.whatwg.org/index-gb18030.txt") + if err != nil { + log.Fatalf("Get: %v", err) + } + defer res.Body.Close() + + fmt.Printf("// gb18030 is the table from http://encoding.spec.whatwg.org/index-gb18030.txt\n") + fmt.Printf("var gb18030 = [...][2]uint16{\n") + scanner := bufio.NewScanner(res.Body) + for scanner.Scan() { + s := strings.TrimSpace(scanner.Text()) + if s == "" || s[0] == '#' { + continue + } + x, y := uint32(0), uint32(0) + if _, err := fmt.Sscanf(s, "%d 0x%x", &x, &y); err != nil { + log.Fatalf("could not parse %q", s) + } + if x < 0x10000 && y < 0x10000 { + fmt.Printf("\t{0x%04x, 0x%04x},\n", x, y) + } + } + fmt.Printf("}\n\n") +} + +func printGBK() { + res, err := http.Get("http://encoding.spec.whatwg.org/index-gbk.txt") + if err != nil { + log.Fatalf("Get: %v", err) + } + defer res.Body.Close() + + mapping := [65536]uint16{} + reverse := [65536]uint16{} + + scanner := bufio.NewScanner(res.Body) + for scanner.Scan() { + s := strings.TrimSpace(scanner.Text()) + if s == "" || s[0] == '#' { + continue + } + x, y := uint16(0), uint16(0) + if _, err := fmt.Sscanf(s, "%d 0x%x", &x, &y); err != nil { + log.Fatalf("could not parse %q", s) + } + if x < 0 || 126*190 <= x { + log.Fatalf("GBK code %d is out of range", x) + } + mapping[x] = y + if reverse[y] == 0 { + c0, c1 := x/190, x%190 + if c1 >= 0x3f { + c1++ + } + reverse[y] = (0x81+c0)<<8 | (0x40 + c1) + } + } + if err := scanner.Err(); err != nil { + log.Fatalf("scanner error: %v", err) + } + + fmt.Printf("// decode is the decoding table from GBK code to Unicode.\n") + fmt.Printf("// It is defined at http://encoding.spec.whatwg.org/index-gbk.txt\n") + fmt.Printf("var decode = [...]uint16{\n") + for i, v := range mapping { + if v != 0 { + fmt.Printf("\t%d: 0x%04X,\n", i, v) + } + } + fmt.Printf("}\n\n") + + // Any run of at least separation continuous zero entries in the reverse map will + // be a separate encode table. + const separation = 1024 + + intervals := []interval(nil) + low, high := -1, -1 + for i, v := range reverse { + if v == 0 { + continue + } + if low < 0 { + low = i + } else if i-high >= separation { + if high >= 0 { + intervals = append(intervals, interval{low, high}) + } + low = i + } + high = i + 1 + } + if high >= 0 { + intervals = append(intervals, interval{low, high}) + } + sort.Sort(byDecreasingLength(intervals)) + + fmt.Printf("const numEncodeTables = %d\n\n", len(intervals)) + fmt.Printf("// encodeX are the encoding tables from Unicode to GBK code,\n") + fmt.Printf("// sorted by decreasing length.\n") + for i, v := range intervals { + fmt.Printf("// encode%d: %5d entries for runes in [%5d, %5d).\n", i, v.len(), v.low, v.high) + } + fmt.Printf("\n") + + for i, v := range intervals { + fmt.Printf("const encode%dLow, encode%dHigh = %d, %d\n\n", i, i, v.low, v.high) + fmt.Printf("var encode%d = [...]uint16{\n", i) + for j := v.low; j < v.high; j++ { + x := reverse[j] + if x == 0 { + continue + } + fmt.Printf("\t%d-%d: 0x%04X,\n", j, v.low, x) + } + fmt.Printf("}\n\n") + } +} + +// interval is a half-open interval [low, high). +type interval struct { + low, high int +} + +func (i interval) len() int { return i.high - i.low } + +// byDecreasingLength sorts intervals by decreasing length. +type byDecreasingLength []interval + +func (b byDecreasingLength) Len() int { return len(b) } +func (b byDecreasingLength) Less(i, j int) bool { return b[i].len() > b[j].len() } +func (b byDecreasingLength) Swap(i, j int) { b[i], b[j] = b[j], b[i] } diff --git a/vendor/golang.org/x/text/encoding/traditionalchinese/maketables.go b/vendor/golang.org/x/text/encoding/traditionalchinese/maketables.go new file mode 100644 index 000000000..cf7fdb31a --- /dev/null +++ b/vendor/golang.org/x/text/encoding/traditionalchinese/maketables.go @@ -0,0 +1,140 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +package main + +// This program generates tables.go: +// go run maketables.go | gofmt > tables.go + +import ( + "bufio" + "fmt" + "log" + "net/http" + "sort" + "strings" +) + +func main() { + fmt.Printf("// generated by go run maketables.go; DO NOT EDIT\n\n") + fmt.Printf("// Package traditionalchinese provides Traditional Chinese encodings such as Big5.\n") + fmt.Printf(`package traditionalchinese // import "golang.org/x/text/encoding/traditionalchinese"` + "\n\n") + + res, err := http.Get("http://encoding.spec.whatwg.org/index-big5.txt") + if err != nil { + log.Fatalf("Get: %v", err) + } + defer res.Body.Close() + + mapping := [65536]uint32{} + reverse := [65536 * 4]uint16{} + + scanner := bufio.NewScanner(res.Body) + for scanner.Scan() { + s := strings.TrimSpace(scanner.Text()) + if s == "" || s[0] == '#' { + continue + } + x, y := uint16(0), uint32(0) + if _, err := fmt.Sscanf(s, "%d 0x%x", &x, &y); err != nil { + log.Fatalf("could not parse %q", s) + } + if x < 0 || 126*157 <= x { + log.Fatalf("Big5 code %d is out of range", x) + } + mapping[x] = y + + // The WHATWG spec http://encoding.spec.whatwg.org/#indexes says that + // "The index pointer for code point in index is the first pointer + // corresponding to code point in index", which would normally mean + // that the code below should be guarded by "if reverse[y] == 0", but + // last instead of first seems to match the behavior of + // "iconv -f UTF-8 -t BIG5". For example, U+8005 者 occurs twice in + // http://encoding.spec.whatwg.org/index-big5.txt, as index 2148 + // (encoded as "\x8e\xcd") and index 6543 (encoded as "\xaa\xcc") + // and "echo 者 | iconv -f UTF-8 -t BIG5 | xxd" gives "\xaa\xcc". + c0, c1 := x/157, x%157 + if c1 < 0x3f { + c1 += 0x40 + } else { + c1 += 0x62 + } + reverse[y] = (0x81+c0)<<8 | c1 + } + if err := scanner.Err(); err != nil { + log.Fatalf("scanner error: %v", err) + } + + fmt.Printf("// decode is the decoding table from Big5 code to Unicode.\n") + fmt.Printf("// It is defined at http://encoding.spec.whatwg.org/index-big5.txt\n") + fmt.Printf("var decode = [...]uint32{\n") + for i, v := range mapping { + if v != 0 { + fmt.Printf("\t%d: 0x%08X,\n", i, v) + } + } + fmt.Printf("}\n\n") + + // Any run of at least separation continuous zero entries in the reverse map will + // be a separate encode table. + const separation = 1024 + + intervals := []interval(nil) + low, high := -1, -1 + for i, v := range reverse { + if v == 0 { + continue + } + if low < 0 { + low = i + } else if i-high >= separation { + if high >= 0 { + intervals = append(intervals, interval{low, high}) + } + low = i + } + high = i + 1 + } + if high >= 0 { + intervals = append(intervals, interval{low, high}) + } + sort.Sort(byDecreasingLength(intervals)) + + fmt.Printf("const numEncodeTables = %d\n\n", len(intervals)) + fmt.Printf("// encodeX are the encoding tables from Unicode to Big5 code,\n") + fmt.Printf("// sorted by decreasing length.\n") + for i, v := range intervals { + fmt.Printf("// encode%d: %5d entries for runes in [%6d, %6d).\n", i, v.len(), v.low, v.high) + } + fmt.Printf("\n") + + for i, v := range intervals { + fmt.Printf("const encode%dLow, encode%dHigh = %d, %d\n\n", i, i, v.low, v.high) + fmt.Printf("var encode%d = [...]uint16{\n", i) + for j := v.low; j < v.high; j++ { + x := reverse[j] + if x == 0 { + continue + } + fmt.Printf("\t%d-%d: 0x%04X,\n", j, v.low, x) + } + fmt.Printf("}\n\n") + } +} + +// interval is a half-open interval [low, high). +type interval struct { + low, high int +} + +func (i interval) len() int { return i.high - i.low } + +// byDecreasingLength sorts intervals by decreasing length. +type byDecreasingLength []interval + +func (b byDecreasingLength) Len() int { return len(b) } +func (b byDecreasingLength) Less(i, j int) bool { return b[i].len() > b[j].len() } +func (b byDecreasingLength) Swap(i, j int) { b[i], b[j] = b[j], b[i] } diff --git a/vendor/golang.org/x/text/language/gen.go b/vendor/golang.org/x/text/language/gen.go new file mode 100644 index 000000000..302f1940a --- /dev/null +++ b/vendor/golang.org/x/text/language/gen.go @@ -0,0 +1,1712 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +// Language tag table generator. +// Data read from the web. + +package main + +import ( + "bufio" + "flag" + "fmt" + "io" + "io/ioutil" + "log" + "math" + "reflect" + "regexp" + "sort" + "strconv" + "strings" + + "golang.org/x/text/internal/gen" + "golang.org/x/text/internal/tag" + "golang.org/x/text/unicode/cldr" +) + +var ( + test = flag.Bool("test", + false, + "test existing tables; can be used to compare web data with package data.") + outputFile = flag.String("output", + "tables.go", + "output file for generated tables") +) + +var comment = []string{ + ` +lang holds an alphabetically sorted list of ISO-639 language identifiers. +All entries are 4 bytes. The index of the identifier (divided by 4) is the language tag. +For 2-byte language identifiers, the two successive bytes have the following meaning: + - if the first letter of the 2- and 3-letter ISO codes are the same: + the second and third letter of the 3-letter ISO code. + - otherwise: a 0 and a by 2 bits right-shifted index into altLangISO3. +For 3-byte language identifiers the 4th byte is 0.`, + ` +langNoIndex is a bit vector of all 3-letter language codes that are not used as an index +in lookup tables. The language ids for these language codes are derived directly +from the letters and are not consecutive.`, + ` +altLangISO3 holds an alphabetically sorted list of 3-letter language code alternatives +to 2-letter language codes that cannot be derived using the method described above. +Each 3-letter code is followed by its 1-byte langID.`, + ` +altLangIndex is used to convert indexes in altLangISO3 to langIDs.`, + ` +langAliasMap maps langIDs to their suggested replacements.`, + ` +script is an alphabetically sorted list of ISO 15924 codes. The index +of the script in the string, divided by 4, is the internal scriptID.`, + ` +isoRegionOffset needs to be added to the index of regionISO to obtain the regionID +for 2-letter ISO codes. (The first isoRegionOffset regionIDs are reserved for +the UN.M49 codes used for groups.)`, + ` +regionISO holds a list of alphabetically sorted 2-letter ISO region codes. +Each 2-letter codes is followed by two bytes with the following meaning: + - [A-Z}{2}: the first letter of the 2-letter code plus these two + letters form the 3-letter ISO code. + - 0, n: index into altRegionISO3.`, + ` +regionTypes defines the status of a region for various standards.`, + ` +m49 maps regionIDs to UN.M49 codes. The first isoRegionOffset entries are +codes indicating collections of regions.`, + ` +m49Index gives indexes into fromM49 based on the three most significant bits +of a 10-bit UN.M49 code. To search an UN.M49 code in fromM49, search in + fromM49[m49Index[msb39(code)]:m49Index[msb3(code)+1]] +for an entry where the first 7 bits match the 7 lsb of the UN.M49 code. +The region code is stored in the 9 lsb of the indexed value.`, + ` +fromM49 contains entries to map UN.M49 codes to regions. See m49Index for details.`, + ` +altRegionISO3 holds a list of 3-letter region codes that cannot be +mapped to 2-letter codes using the default algorithm. This is a short list.`, + ` +altRegionIDs holds a list of regionIDs the positions of which match those +of the 3-letter ISO codes in altRegionISO3.`, + ` +variantNumSpecialized is the number of specialized variants in variants.`, + ` +suppressScript is an index from langID to the dominant script for that language, +if it exists. If a script is given, it should be suppressed from the language tag.`, + ` +likelyLang is a lookup table, indexed by langID, for the most likely +scripts and regions given incomplete information. If more entries exist for a +given language, region and script are the index and size respectively +of the list in likelyLangList.`, + ` +likelyLangList holds lists info associated with likelyLang.`, + ` +likelyRegion is a lookup table, indexed by regionID, for the most likely +languages and scripts given incomplete information. If more entries exist +for a given regionID, lang and script are the index and size respectively +of the list in likelyRegionList. +TODO: exclude containers and user-definable regions from the list.`, + ` +likelyRegionList holds lists info associated with likelyRegion.`, + ` +likelyScript is a lookup table, indexed by scriptID, for the most likely +languages and regions given a script.`, + ` +matchLang holds pairs of langIDs of base languages that are typically +mutually intelligible. Each pair is associated with a confidence and +whether the intelligibility goes one or both ways.`, + ` +matchScript holds pairs of scriptIDs where readers of one script +can typically also read the other. Each is associated with a confidence.`, + ` +nRegionGroups is the number of region groups.`, + ` +regionInclusion maps region identifiers to sets of regions in regionInclusionBits, +where each set holds all groupings that are directly connected in a region +containment graph.`, + ` +regionInclusionBits is an array of bit vectors where every vector represents +a set of region groupings. These sets are used to compute the distance +between two regions for the purpose of language matching.`, + ` +regionInclusionNext marks, for each entry in regionInclusionBits, the set of +all groups that are reachable from the groups set in the respective entry.`, +} + +// TODO: consider changing some of these structures to tries. This can reduce +// memory, but may increase the need for memory allocations. This could be +// mitigated if we can piggyback on language tags for common cases. + +func failOnError(e error) { + if e != nil { + log.Panic(e) + } +} + +type setType int + +const ( + Indexed setType = 1 + iota // all elements must be of same size + Linear +) + +type stringSet struct { + s []string + sorted, frozen bool + + // We often need to update values after the creation of an index is completed. + // We include a convenience map for keeping track of this. + update map[string]string + typ setType // used for checking. +} + +func (ss *stringSet) clone() stringSet { + c := *ss + c.s = append([]string(nil), c.s...) + return c +} + +func (ss *stringSet) setType(t setType) { + if ss.typ != t && ss.typ != 0 { + log.Panicf("type %d cannot be assigned as it was already %d", t, ss.typ) + } +} + +// parse parses a whitespace-separated string and initializes ss with its +// components. +func (ss *stringSet) parse(s string) { + scan := bufio.NewScanner(strings.NewReader(s)) + scan.Split(bufio.ScanWords) + for scan.Scan() { + ss.add(scan.Text()) + } +} + +func (ss *stringSet) assertChangeable() { + if ss.frozen { + log.Panic("attempt to modify a frozen stringSet") + } +} + +func (ss *stringSet) add(s string) { + ss.assertChangeable() + ss.s = append(ss.s, s) + ss.sorted = ss.frozen +} + +func (ss *stringSet) freeze() { + ss.compact() + ss.frozen = true +} + +func (ss *stringSet) compact() { + if ss.sorted { + return + } + a := ss.s + sort.Strings(a) + k := 0 + for i := 1; i < len(a); i++ { + if a[k] != a[i] { + a[k+1] = a[i] + k++ + } + } + ss.s = a[:k+1] + ss.sorted = ss.frozen +} + +type funcSorter struct { + fn func(a, b string) bool + sort.StringSlice +} + +func (s funcSorter) Less(i, j int) bool { + return s.fn(s.StringSlice[i], s.StringSlice[j]) +} + +func (ss *stringSet) sortFunc(f func(a, b string) bool) { + ss.compact() + sort.Sort(funcSorter{f, sort.StringSlice(ss.s)}) +} + +func (ss *stringSet) remove(s string) { + ss.assertChangeable() + if i, ok := ss.find(s); ok { + copy(ss.s[i:], ss.s[i+1:]) + ss.s = ss.s[:len(ss.s)-1] + } +} + +func (ss *stringSet) replace(ol, nu string) { + ss.s[ss.index(ol)] = nu + ss.sorted = ss.frozen +} + +func (ss *stringSet) index(s string) int { + ss.setType(Indexed) + i, ok := ss.find(s) + if !ok { + if i < len(ss.s) { + log.Panicf("find: item %q is not in list. Closest match is %q.", s, ss.s[i]) + } + log.Panicf("find: item %q is not in list", s) + + } + return i +} + +func (ss *stringSet) find(s string) (int, bool) { + ss.compact() + i := sort.SearchStrings(ss.s, s) + return i, i != len(ss.s) && ss.s[i] == s +} + +func (ss *stringSet) slice() []string { + ss.compact() + return ss.s +} + +func (ss *stringSet) updateLater(v, key string) { + if ss.update == nil { + ss.update = map[string]string{} + } + ss.update[v] = key +} + +// join joins the string and ensures that all entries are of the same length. +func (ss *stringSet) join() string { + ss.setType(Indexed) + n := len(ss.s[0]) + for _, s := range ss.s { + if len(s) != n { + log.Panicf("join: not all entries are of the same length: %q", s) + } + } + ss.s = append(ss.s, strings.Repeat("\xff", n)) + return strings.Join(ss.s, "") +} + +// ianaEntry holds information for an entry in the IANA Language Subtag Repository. +// All types use the same entry. +// See http://tools.ietf.org/html/bcp47#section-5.1 for a description of the various +// fields. +type ianaEntry struct { + typ string + description []string + scope string + added string + preferred string + deprecated string + suppressScript string + macro string + prefix []string +} + +type builder struct { + w *gen.CodeWriter + hw io.Writer // MultiWriter for w and w.Hash + data *cldr.CLDR + supp *cldr.SupplementalData + + // indices + locale stringSet // common locales + lang stringSet // canonical language ids (2 or 3 letter ISO codes) with data + langNoIndex stringSet // 3-letter ISO codes with no associated data + script stringSet // 4-letter ISO codes + region stringSet // 2-letter ISO or 3-digit UN M49 codes + variant stringSet // 4-8-alphanumeric variant code. + + // Region codes that are groups with their corresponding group IDs. + groups map[int]index + + // langInfo + registry map[string]*ianaEntry +} + +type index uint + +func newBuilder(w *gen.CodeWriter) *builder { + r := gen.OpenCLDRCoreZip() + defer r.Close() + d := &cldr.Decoder{} + data, err := d.DecodeZip(r) + failOnError(err) + b := builder{ + w: w, + hw: io.MultiWriter(w, w.Hash), + data: data, + supp: data.Supplemental(), + } + b.parseRegistry() + return &b +} + +func (b *builder) parseRegistry() { + r := gen.OpenIANAFile("assignments/language-subtag-registry") + defer r.Close() + b.registry = make(map[string]*ianaEntry) + + scan := bufio.NewScanner(r) + scan.Split(bufio.ScanWords) + var record *ianaEntry + for more := scan.Scan(); more; { + key := scan.Text() + more = scan.Scan() + value := scan.Text() + switch key { + case "Type:": + record = &ianaEntry{typ: value} + case "Subtag:", "Tag:": + if s := strings.SplitN(value, "..", 2); len(s) > 1 { + for a := s[0]; a <= s[1]; a = inc(a) { + b.addToRegistry(a, record) + } + } else { + b.addToRegistry(value, record) + } + case "Suppress-Script:": + record.suppressScript = value + case "Added:": + record.added = value + case "Deprecated:": + record.deprecated = value + case "Macrolanguage:": + record.macro = value + case "Preferred-Value:": + record.preferred = value + case "Prefix:": + record.prefix = append(record.prefix, value) + case "Scope:": + record.scope = value + case "Description:": + buf := []byte(value) + for more = scan.Scan(); more; more = scan.Scan() { + b := scan.Bytes() + if b[0] == '%' || b[len(b)-1] == ':' { + break + } + buf = append(buf, ' ') + buf = append(buf, b...) + } + record.description = append(record.description, string(buf)) + continue + default: + continue + } + more = scan.Scan() + } + if scan.Err() != nil { + log.Panic(scan.Err()) + } +} + +func (b *builder) addToRegistry(key string, entry *ianaEntry) { + if info, ok := b.registry[key]; ok { + if info.typ != "language" || entry.typ != "extlang" { + log.Fatalf("parseRegistry: tag %q already exists", key) + } + } else { + b.registry[key] = entry + } +} + +var commentIndex = make(map[string]string) + +func init() { + for _, s := range comment { + key := strings.TrimSpace(strings.SplitN(s, " ", 2)[0]) + commentIndex[key] = s + } +} + +func (b *builder) comment(name string) { + if s := commentIndex[name]; len(s) > 0 { + b.w.WriteComment(s) + } else { + fmt.Fprintln(b.w) + } +} + +func (b *builder) pf(f string, x ...interface{}) { + fmt.Fprintf(b.hw, f, x...) + fmt.Fprint(b.hw, "\n") +} + +func (b *builder) p(x ...interface{}) { + fmt.Fprintln(b.hw, x...) +} + +func (b *builder) addSize(s int) { + b.w.Size += s + b.pf("// Size: %d bytes", s) +} + +func (b *builder) writeConst(name string, x interface{}) { + b.comment(name) + b.w.WriteConst(name, x) +} + +// writeConsts computes f(v) for all v in values and writes the results +// as constants named _v to a single constant block. +func (b *builder) writeConsts(f func(string) int, values ...string) { + b.pf("const (") + for _, v := range values { + b.pf("\t_%s = %v", v, f(v)) + } + b.pf(")") +} + +// writeType writes the type of the given value, which must be a struct. +func (b *builder) writeType(value interface{}) { + b.comment(reflect.TypeOf(value).Name()) + b.w.WriteType(value) +} + +func (b *builder) writeSlice(name string, ss interface{}) { + b.writeSliceAddSize(name, 0, ss) +} + +func (b *builder) writeSliceAddSize(name string, extraSize int, ss interface{}) { + b.comment(name) + b.w.Size += extraSize + v := reflect.ValueOf(ss) + t := v.Type().Elem() + b.pf("// Size: %d bytes, %d elements", v.Len()*int(t.Size())+extraSize, v.Len()) + + fmt.Fprintf(b.w, "var %s = ", name) + b.w.WriteArray(ss) + b.p() +} + +type fromTo struct { + from, to uint16 +} + +func (b *builder) writeSortedMap(name string, ss *stringSet, index func(s string) uint16) { + ss.sortFunc(func(a, b string) bool { + return index(a) < index(b) + }) + m := []fromTo{} + for _, s := range ss.s { + m = append(m, fromTo{index(s), index(ss.update[s])}) + } + b.writeSlice(name, m) +} + +const base = 'z' - 'a' + 1 + +func strToInt(s string) uint { + v := uint(0) + for i := 0; i < len(s); i++ { + v *= base + v += uint(s[i] - 'a') + } + return v +} + +// converts the given integer to the original ASCII string passed to strToInt. +// len(s) must match the number of characters obtained. +func intToStr(v uint, s []byte) { + for i := len(s) - 1; i >= 0; i-- { + s[i] = byte(v%base) + 'a' + v /= base + } +} + +func (b *builder) writeBitVector(name string, ss []string) { + vec := make([]uint8, int(math.Ceil(math.Pow(base, float64(len(ss[0])))/8))) + for _, s := range ss { + v := strToInt(s) + vec[v/8] |= 1 << (v % 8) + } + b.writeSlice(name, vec) +} + +// TODO: convert this type into a list or two-stage trie. +func (b *builder) writeMapFunc(name string, m map[string]string, f func(string) uint16) { + b.comment(name) + v := reflect.ValueOf(m) + sz := v.Len() * (2 + int(v.Type().Key().Size())) + for _, k := range m { + sz += len(k) + } + b.addSize(sz) + keys := []string{} + b.pf(`var %s = map[string]uint16{`, name) + for k := range m { + keys = append(keys, k) + } + sort.Strings(keys) + for _, k := range keys { + b.pf("\t%q: %v,", k, f(m[k])) + } + b.p("}") +} + +func (b *builder) writeMap(name string, m interface{}) { + b.comment(name) + v := reflect.ValueOf(m) + sz := v.Len() * (2 + int(v.Type().Key().Size()) + int(v.Type().Elem().Size())) + b.addSize(sz) + f := strings.FieldsFunc(fmt.Sprintf("%#v", m), func(r rune) bool { + return strings.IndexRune("{}, ", r) != -1 + }) + sort.Strings(f[1:]) + b.pf(`var %s = %s{`, name, f[0]) + for _, kv := range f[1:] { + b.pf("\t%s,", kv) + } + b.p("}") +} + +func (b *builder) langIndex(s string) uint16 { + if s == "und" { + return 0 + } + if i, ok := b.lang.find(s); ok { + return uint16(i) + } + return uint16(strToInt(s)) + uint16(len(b.lang.s)) +} + +// inc advances the string to its lexicographical successor. +func inc(s string) string { + const maxTagLength = 4 + var buf [maxTagLength]byte + intToStr(strToInt(strings.ToLower(s))+1, buf[:len(s)]) + for i := 0; i < len(s); i++ { + if s[i] <= 'Z' { + buf[i] -= 'a' - 'A' + } + } + return string(buf[:len(s)]) +} + +func (b *builder) parseIndices() { + meta := b.supp.Metadata + + for k, v := range b.registry { + var ss *stringSet + switch v.typ { + case "language": + if len(k) == 2 || v.suppressScript != "" || v.scope == "special" { + b.lang.add(k) + continue + } else { + ss = &b.langNoIndex + } + case "region": + ss = &b.region + case "script": + ss = &b.script + case "variant": + ss = &b.variant + default: + continue + } + ss.add(k) + } + // Include any language for which there is data. + for _, lang := range b.data.Locales() { + if x := b.data.RawLDML(lang); false || + x.LocaleDisplayNames != nil || + x.Characters != nil || + x.Delimiters != nil || + x.Measurement != nil || + x.Dates != nil || + x.Numbers != nil || + x.Units != nil || + x.ListPatterns != nil || + x.Collations != nil || + x.Segmentations != nil || + x.Rbnf != nil || + x.Annotations != nil || + x.Metadata != nil { + + from := strings.Split(lang, "_") + if lang := from[0]; lang != "root" { + b.lang.add(lang) + } + } + } + // Include locales for plural rules, which uses a different structure. + for _, plurals := range b.data.Supplemental().Plurals { + for _, rules := range plurals.PluralRules { + for _, lang := range strings.Split(rules.Locales, " ") { + if lang = strings.Split(lang, "_")[0]; lang != "root" { + b.lang.add(lang) + } + } + } + } + // Include languages in likely subtags. + for _, m := range b.supp.LikelySubtags.LikelySubtag { + from := strings.Split(m.From, "_") + b.lang.add(from[0]) + } + // Include ISO-639 alpha-3 bibliographic entries. + for _, a := range meta.Alias.LanguageAlias { + if a.Reason == "bibliographic" { + b.langNoIndex.add(a.Type) + } + } + // Include regions in territoryAlias (not all are in the IANA registry!) + for _, reg := range b.supp.Metadata.Alias.TerritoryAlias { + if len(reg.Type) == 2 { + b.region.add(reg.Type) + } + } + + for _, s := range b.lang.s { + if len(s) == 3 { + b.langNoIndex.remove(s) + } + } + b.writeConst("numLanguages", len(b.lang.slice())+len(b.langNoIndex.slice())) + b.writeConst("numScripts", len(b.script.slice())) + b.writeConst("numRegions", len(b.region.slice())) + + // Add dummy codes at the start of each list to represent "unspecified". + b.lang.add("---") + b.script.add("----") + b.region.add("---") + + // common locales + b.locale.parse(meta.DefaultContent.Locales) +} + +// TODO: region inclusion data will probably not be use used in future matchers. + +func (b *builder) computeRegionGroups() { + b.groups = make(map[int]index) + + // Create group indices. + for i := 1; b.region.s[i][0] < 'A'; i++ { // Base M49 indices on regionID. + b.groups[i] = index(len(b.groups)) + } + for _, g := range b.supp.TerritoryContainment.Group { + // Skip UN and EURO zone as they are flattening the containment + // relationship. + if g.Type == "EZ" || g.Type == "UN" { + continue + } + group := b.region.index(g.Type) + if _, ok := b.groups[group]; !ok { + b.groups[group] = index(len(b.groups)) + } + } + if len(b.groups) > 64 { + log.Fatalf("only 64 groups supported, found %d", len(b.groups)) + } + b.writeConst("nRegionGroups", len(b.groups)) +} + +var langConsts = []string{ + "af", "am", "ar", "az", "bg", "bn", "ca", "cs", "da", "de", "el", "en", "es", + "et", "fa", "fi", "fil", "fr", "gu", "he", "hi", "hr", "hu", "hy", "id", "is", + "it", "ja", "ka", "kk", "km", "kn", "ko", "ky", "lo", "lt", "lv", "mk", "ml", + "mn", "mo", "mr", "ms", "mul", "my", "nb", "ne", "nl", "no", "pa", "pl", "pt", + "ro", "ru", "sh", "si", "sk", "sl", "sq", "sr", "sv", "sw", "ta", "te", "th", + "tl", "tn", "tr", "uk", "ur", "uz", "vi", "zh", "zu", + + // constants for grandfathered tags (if not already defined) + "jbo", "ami", "bnn", "hak", "tlh", "lb", "nv", "pwn", "tao", "tay", "tsu", + "nn", "sfb", "vgt", "sgg", "cmn", "nan", "hsn", +} + +// writeLanguage generates all tables needed for language canonicalization. +func (b *builder) writeLanguage() { + meta := b.supp.Metadata + + b.writeConst("nonCanonicalUnd", b.lang.index("und")) + b.writeConsts(func(s string) int { return int(b.langIndex(s)) }, langConsts...) + b.writeConst("langPrivateStart", b.langIndex("qaa")) + b.writeConst("langPrivateEnd", b.langIndex("qtz")) + + // Get language codes that need to be mapped (overlong 3-letter codes, + // deprecated 2-letter codes, legacy and grandfathered tags.) + langAliasMap := stringSet{} + aliasTypeMap := map[string]langAliasType{} + + // altLangISO3 get the alternative ISO3 names that need to be mapped. + altLangISO3 := stringSet{} + // Add dummy start to avoid the use of index 0. + altLangISO3.add("---") + altLangISO3.updateLater("---", "aa") + + lang := b.lang.clone() + for _, a := range meta.Alias.LanguageAlias { + if a.Replacement == "" { + a.Replacement = "und" + } + // TODO: support mapping to tags + repl := strings.SplitN(a.Replacement, "_", 2)[0] + if a.Reason == "overlong" { + if len(a.Replacement) == 2 && len(a.Type) == 3 { + lang.updateLater(a.Replacement, a.Type) + } + } else if len(a.Type) <= 3 { + switch a.Reason { + case "macrolanguage": + aliasTypeMap[a.Type] = langMacro + case "deprecated": + // handled elsewhere + continue + case "bibliographic", "legacy": + if a.Type == "no" { + continue + } + aliasTypeMap[a.Type] = langLegacy + default: + log.Fatalf("new %s alias: %s", a.Reason, a.Type) + } + langAliasMap.add(a.Type) + langAliasMap.updateLater(a.Type, repl) + } + } + // Manually add the mapping of "nb" (Norwegian) to its macro language. + // This can be removed if CLDR adopts this change. + langAliasMap.add("nb") + langAliasMap.updateLater("nb", "no") + aliasTypeMap["nb"] = langMacro + + for k, v := range b.registry { + // Also add deprecated values for 3-letter ISO codes, which CLDR omits. + if v.typ == "language" && v.deprecated != "" && v.preferred != "" { + langAliasMap.add(k) + langAliasMap.updateLater(k, v.preferred) + aliasTypeMap[k] = langDeprecated + } + } + // Fix CLDR mappings. + lang.updateLater("tl", "tgl") + lang.updateLater("sh", "hbs") + lang.updateLater("mo", "mol") + lang.updateLater("no", "nor") + lang.updateLater("tw", "twi") + lang.updateLater("nb", "nob") + lang.updateLater("ak", "aka") + lang.updateLater("bh", "bih") + + // Ensure that each 2-letter code is matched with a 3-letter code. + for _, v := range lang.s[1:] { + s, ok := lang.update[v] + if !ok { + if s, ok = lang.update[langAliasMap.update[v]]; !ok { + continue + } + lang.update[v] = s + } + if v[0] != s[0] { + altLangISO3.add(s) + altLangISO3.updateLater(s, v) + } + } + + // Complete canonicalized language tags. + lang.freeze() + for i, v := range lang.s { + // We can avoid these manual entries by using the IANA registry directly. + // Seems easier to update the list manually, as changes are rare. + // The panic in this loop will trigger if we miss an entry. + add := "" + if s, ok := lang.update[v]; ok { + if s[0] == v[0] { + add = s[1:] + } else { + add = string([]byte{0, byte(altLangISO3.index(s))}) + } + } else if len(v) == 3 { + add = "\x00" + } else { + log.Panicf("no data for long form of %q", v) + } + lang.s[i] += add + } + b.writeConst("lang", tag.Index(lang.join())) + + b.writeConst("langNoIndexOffset", len(b.lang.s)) + + // space of all valid 3-letter language identifiers. + b.writeBitVector("langNoIndex", b.langNoIndex.slice()) + + altLangIndex := []uint16{} + for i, s := range altLangISO3.slice() { + altLangISO3.s[i] += string([]byte{byte(len(altLangIndex))}) + if i > 0 { + idx := b.lang.index(altLangISO3.update[s]) + altLangIndex = append(altLangIndex, uint16(idx)) + } + } + b.writeConst("altLangISO3", tag.Index(altLangISO3.join())) + b.writeSlice("altLangIndex", altLangIndex) + + b.writeSortedMap("langAliasMap", &langAliasMap, b.langIndex) + types := make([]langAliasType, len(langAliasMap.s)) + for i, s := range langAliasMap.s { + types[i] = aliasTypeMap[s] + } + b.writeSlice("langAliasTypes", types) +} + +var scriptConsts = []string{ + "Latn", "Hani", "Hans", "Hant", "Qaaa", "Qaai", "Qabx", "Zinh", "Zyyy", + "Zzzz", +} + +func (b *builder) writeScript() { + b.writeConsts(b.script.index, scriptConsts...) + b.writeConst("script", tag.Index(b.script.join())) + + supp := make([]uint8, len(b.lang.slice())) + for i, v := range b.lang.slice()[1:] { + if sc := b.registry[v].suppressScript; sc != "" { + supp[i+1] = uint8(b.script.index(sc)) + } + } + b.writeSlice("suppressScript", supp) + + // There is only one deprecated script in CLDR. This value is hard-coded. + // We check here if the code must be updated. + for _, a := range b.supp.Metadata.Alias.ScriptAlias { + if a.Type != "Qaai" { + log.Panicf("unexpected deprecated stript %q", a.Type) + } + } +} + +func parseM49(s string) int16 { + if len(s) == 0 { + return 0 + } + v, err := strconv.ParseUint(s, 10, 10) + failOnError(err) + return int16(v) +} + +var regionConsts = []string{ + "001", "419", "BR", "CA", "ES", "GB", "MD", "PT", "UK", "US", + "ZZ", "XA", "XC", "XK", // Unofficial tag for Kosovo. +} + +func (b *builder) writeRegion() { + b.writeConsts(b.region.index, regionConsts...) + + isoOffset := b.region.index("AA") + m49map := make([]int16, len(b.region.slice())) + fromM49map := make(map[int16]int) + altRegionISO3 := "" + altRegionIDs := []uint16{} + + b.writeConst("isoRegionOffset", isoOffset) + + // 2-letter region lookup and mapping to numeric codes. + regionISO := b.region.clone() + regionISO.s = regionISO.s[isoOffset:] + regionISO.sorted = false + + regionTypes := make([]byte, len(b.region.s)) + + // Is the region valid BCP 47? + for s, e := range b.registry { + if len(s) == 2 && s == strings.ToUpper(s) { + i := b.region.index(s) + for _, d := range e.description { + if strings.Contains(d, "Private use") { + regionTypes[i] = iso3166UserAssigned + } + } + regionTypes[i] |= bcp47Region + } + } + + // Is the region a valid ccTLD? + r := gen.OpenIANAFile("domains/root/db") + defer r.Close() + + buf, err := ioutil.ReadAll(r) + failOnError(err) + re := regexp.MustCompile(`"/domains/root/db/([a-z]{2}).html"`) + for _, m := range re.FindAllSubmatch(buf, -1) { + i := b.region.index(strings.ToUpper(string(m[1]))) + regionTypes[i] |= ccTLD + } + + b.writeSlice("regionTypes", regionTypes) + + iso3Set := make(map[string]int) + update := func(iso2, iso3 string) { + i := regionISO.index(iso2) + if j, ok := iso3Set[iso3]; !ok && iso3[0] == iso2[0] { + regionISO.s[i] += iso3[1:] + iso3Set[iso3] = -1 + } else { + if ok && j >= 0 { + regionISO.s[i] += string([]byte{0, byte(j)}) + } else { + iso3Set[iso3] = len(altRegionISO3) + regionISO.s[i] += string([]byte{0, byte(len(altRegionISO3))}) + altRegionISO3 += iso3 + altRegionIDs = append(altRegionIDs, uint16(isoOffset+i)) + } + } + } + for _, tc := range b.supp.CodeMappings.TerritoryCodes { + i := regionISO.index(tc.Type) + isoOffset + if d := m49map[i]; d != 0 { + log.Panicf("%s found as a duplicate UN.M49 code of %03d", tc.Numeric, d) + } + m49 := parseM49(tc.Numeric) + m49map[i] = m49 + if r := fromM49map[m49]; r == 0 { + fromM49map[m49] = i + } else if r != i { + dep := b.registry[regionISO.s[r-isoOffset]].deprecated + if t := b.registry[tc.Type]; t != nil && dep != "" && (t.deprecated == "" || t.deprecated > dep) { + fromM49map[m49] = i + } + } + } + for _, ta := range b.supp.Metadata.Alias.TerritoryAlias { + if len(ta.Type) == 3 && ta.Type[0] <= '9' && len(ta.Replacement) == 2 { + from := parseM49(ta.Type) + if r := fromM49map[from]; r == 0 { + fromM49map[from] = regionISO.index(ta.Replacement) + isoOffset + } + } + } + for _, tc := range b.supp.CodeMappings.TerritoryCodes { + if len(tc.Alpha3) == 3 { + update(tc.Type, tc.Alpha3) + } + } + // This entries are not included in territoryCodes. Mostly 3-letter variants + // of deleted codes and an entry for QU. + for _, m := range []struct{ iso2, iso3 string }{ + {"CT", "CTE"}, + {"DY", "DHY"}, + {"HV", "HVO"}, + {"JT", "JTN"}, + {"MI", "MID"}, + {"NH", "NHB"}, + {"NQ", "ATN"}, + {"PC", "PCI"}, + {"PU", "PUS"}, + {"PZ", "PCZ"}, + {"RH", "RHO"}, + {"VD", "VDR"}, + {"WK", "WAK"}, + // These three-letter codes are used for others as well. + {"FQ", "ATF"}, + } { + update(m.iso2, m.iso3) + } + for i, s := range regionISO.s { + if len(s) != 4 { + regionISO.s[i] = s + " " + } + } + b.writeConst("regionISO", tag.Index(regionISO.join())) + b.writeConst("altRegionISO3", altRegionISO3) + b.writeSlice("altRegionIDs", altRegionIDs) + + // Create list of deprecated regions. + // TODO: consider inserting SF -> FI. Not included by CLDR, but is the only + // Transitionally-reserved mapping not included. + regionOldMap := stringSet{} + // Include regions in territoryAlias (not all are in the IANA registry!) + for _, reg := range b.supp.Metadata.Alias.TerritoryAlias { + if len(reg.Type) == 2 && reg.Reason == "deprecated" && len(reg.Replacement) == 2 { + regionOldMap.add(reg.Type) + regionOldMap.updateLater(reg.Type, reg.Replacement) + i, _ := regionISO.find(reg.Type) + j, _ := regionISO.find(reg.Replacement) + if k := m49map[i+isoOffset]; k == 0 { + m49map[i+isoOffset] = m49map[j+isoOffset] + } + } + } + b.writeSortedMap("regionOldMap", ®ionOldMap, func(s string) uint16 { + return uint16(b.region.index(s)) + }) + // 3-digit region lookup, groupings. + for i := 1; i < isoOffset; i++ { + m := parseM49(b.region.s[i]) + m49map[i] = m + fromM49map[m] = i + } + b.writeSlice("m49", m49map) + + const ( + searchBits = 7 + regionBits = 9 + ) + if len(m49map) >= 1< %d", len(m49map), 1<>searchBits] = int16(len(fromM49)) + } + b.writeSlice("m49Index", m49Index) + b.writeSlice("fromM49", fromM49) +} + +const ( + // TODO: put these lists in regionTypes as user data? Could be used for + // various optimizations and refinements and could be exposed in the API. + iso3166Except = "AC CP DG EA EU FX IC SU TA UK" + iso3166Trans = "AN BU CS NT TP YU ZR" // SF is not in our set of Regions. + // DY and RH are actually not deleted, but indeterminately reserved. + iso3166DelCLDR = "CT DD DY FQ HV JT MI NH NQ PC PU PZ RH VD WK YD" +) + +const ( + iso3166UserAssigned = 1 << iota + ccTLD + bcp47Region +) + +func find(list []string, s string) int { + for i, t := range list { + if t == s { + return i + } + } + return -1 +} + +// writeVariants generates per-variant information and creates a map from variant +// name to index value. We assign index values such that sorting multiple +// variants by index value will result in the correct order. +// There are two types of variants: specialized and general. Specialized variants +// are only applicable to certain language or language-script pairs. Generalized +// variants apply to any language. Generalized variants always sort after +// specialized variants. We will therefore always assign a higher index value +// to a generalized variant than any other variant. Generalized variants are +// sorted alphabetically among themselves. +// Specialized variants may also sort after other specialized variants. Such +// variants will be ordered after any of the variants they may follow. +// We assume that if a variant x is followed by a variant y, then for any prefix +// p of x, p-x is a prefix of y. This allows us to order tags based on the +// maximum of the length of any of its prefixes. +// TODO: it is possible to define a set of Prefix values on variants such that +// a total order cannot be defined to the point that this algorithm breaks. +// In other words, we cannot guarantee the same order of variants for the +// future using the same algorithm or for non-compliant combinations of +// variants. For this reason, consider using simple alphabetic sorting +// of variants and ignore Prefix restrictions altogether. +func (b *builder) writeVariant() { + generalized := stringSet{} + specialized := stringSet{} + specializedExtend := stringSet{} + // Collate the variants by type and check assumptions. + for _, v := range b.variant.slice() { + e := b.registry[v] + if len(e.prefix) == 0 { + generalized.add(v) + continue + } + c := strings.Split(e.prefix[0], "-") + hasScriptOrRegion := false + if len(c) > 1 { + _, hasScriptOrRegion = b.script.find(c[1]) + if !hasScriptOrRegion { + _, hasScriptOrRegion = b.region.find(c[1]) + + } + } + if len(c) == 1 || len(c) == 2 && hasScriptOrRegion { + // Variant is preceded by a language. + specialized.add(v) + continue + } + // Variant is preceded by another variant. + specializedExtend.add(v) + prefix := c[0] + "-" + if hasScriptOrRegion { + prefix += c[1] + } + for _, p := range e.prefix { + // Verify that the prefix minus the last element is a prefix of the + // predecessor element. + i := strings.LastIndex(p, "-") + pred := b.registry[p[i+1:]] + if find(pred.prefix, p[:i]) < 0 { + log.Fatalf("prefix %q for variant %q not consistent with predecessor spec", p, v) + } + // The sorting used below does not work in the general case. It works + // if we assume that variants that may be followed by others only have + // prefixes of the same length. Verify this. + count := strings.Count(p[:i], "-") + for _, q := range pred.prefix { + if c := strings.Count(q, "-"); c != count { + log.Fatalf("variant %q preceding %q has a prefix %q of size %d; want %d", p[i+1:], v, q, c, count) + } + } + if !strings.HasPrefix(p, prefix) { + log.Fatalf("prefix %q of variant %q should start with %q", p, v, prefix) + } + } + } + + // Sort extended variants. + a := specializedExtend.s + less := func(v, w string) bool { + // Sort by the maximum number of elements. + maxCount := func(s string) (max int) { + for _, p := range b.registry[s].prefix { + if c := strings.Count(p, "-"); c > max { + max = c + } + } + return + } + if cv, cw := maxCount(v), maxCount(w); cv != cw { + return cv < cw + } + // Sort by name as tie breaker. + return v < w + } + sort.Sort(funcSorter{less, sort.StringSlice(a)}) + specializedExtend.frozen = true + + // Create index from variant name to index. + variantIndex := make(map[string]uint8) + add := func(s []string) { + for _, v := range s { + variantIndex[v] = uint8(len(variantIndex)) + } + } + add(specialized.slice()) + add(specializedExtend.s) + numSpecialized := len(variantIndex) + add(generalized.slice()) + if n := len(variantIndex); n > 255 { + log.Fatalf("maximum number of variants exceeded: was %d; want <= 255", n) + } + b.writeMap("variantIndex", variantIndex) + b.writeConst("variantNumSpecialized", numSpecialized) +} + +func (b *builder) writeLanguageInfo() { +} + +// writeLikelyData writes tables that are used both for finding parent relations and for +// language matching. Each entry contains additional bits to indicate the status of the +// data to know when it cannot be used for parent relations. +func (b *builder) writeLikelyData() { + const ( + isList = 1 << iota + scriptInFrom + regionInFrom + ) + type ( // generated types + likelyScriptRegion struct { + region uint16 + script uint8 + flags uint8 + } + likelyLangScript struct { + lang uint16 + script uint8 + flags uint8 + } + likelyLangRegion struct { + lang uint16 + region uint16 + } + // likelyTag is used for getting likely tags for group regions, where + // the likely region might be a region contained in the group. + likelyTag struct { + lang uint16 + region uint16 + script uint8 + } + ) + var ( // generated variables + likelyRegionGroup = make([]likelyTag, len(b.groups)) + likelyLang = make([]likelyScriptRegion, len(b.lang.s)) + likelyRegion = make([]likelyLangScript, len(b.region.s)) + likelyScript = make([]likelyLangRegion, len(b.script.s)) + likelyLangList = []likelyScriptRegion{} + likelyRegionList = []likelyLangScript{} + ) + type fromTo struct { + from, to []string + } + langToOther := map[int][]fromTo{} + regionToOther := map[int][]fromTo{} + for _, m := range b.supp.LikelySubtags.LikelySubtag { + from := strings.Split(m.From, "_") + to := strings.Split(m.To, "_") + if len(to) != 3 { + log.Fatalf("invalid number of subtags in %q: found %d, want 3", m.To, len(to)) + } + if len(from) > 3 { + log.Fatalf("invalid number of subtags: found %d, want 1-3", len(from)) + } + if from[0] != to[0] && from[0] != "und" { + log.Fatalf("unexpected language change in expansion: %s -> %s", from, to) + } + if len(from) == 3 { + if from[2] != to[2] { + log.Fatalf("unexpected region change in expansion: %s -> %s", from, to) + } + if from[0] != "und" { + log.Fatalf("unexpected fully specified from tag: %s -> %s", from, to) + } + } + if len(from) == 1 || from[0] != "und" { + id := 0 + if from[0] != "und" { + id = b.lang.index(from[0]) + } + langToOther[id] = append(langToOther[id], fromTo{from, to}) + } else if len(from) == 2 && len(from[1]) == 4 { + sid := b.script.index(from[1]) + likelyScript[sid].lang = uint16(b.langIndex(to[0])) + likelyScript[sid].region = uint16(b.region.index(to[2])) + } else { + r := b.region.index(from[len(from)-1]) + if id, ok := b.groups[r]; ok { + if from[0] != "und" { + log.Fatalf("region changed unexpectedly: %s -> %s", from, to) + } + likelyRegionGroup[id].lang = uint16(b.langIndex(to[0])) + likelyRegionGroup[id].script = uint8(b.script.index(to[1])) + likelyRegionGroup[id].region = uint16(b.region.index(to[2])) + } else { + regionToOther[r] = append(regionToOther[r], fromTo{from, to}) + } + } + } + b.writeType(likelyLangRegion{}) + b.writeSlice("likelyScript", likelyScript) + + for id := range b.lang.s { + list := langToOther[id] + if len(list) == 1 { + likelyLang[id].region = uint16(b.region.index(list[0].to[2])) + likelyLang[id].script = uint8(b.script.index(list[0].to[1])) + } else if len(list) > 1 { + likelyLang[id].flags = isList + likelyLang[id].region = uint16(len(likelyLangList)) + likelyLang[id].script = uint8(len(list)) + for _, x := range list { + flags := uint8(0) + if len(x.from) > 1 { + if x.from[1] == x.to[2] { + flags = regionInFrom + } else { + flags = scriptInFrom + } + } + likelyLangList = append(likelyLangList, likelyScriptRegion{ + region: uint16(b.region.index(x.to[2])), + script: uint8(b.script.index(x.to[1])), + flags: flags, + }) + } + } + } + // TODO: merge suppressScript data with this table. + b.writeType(likelyScriptRegion{}) + b.writeSlice("likelyLang", likelyLang) + b.writeSlice("likelyLangList", likelyLangList) + + for id := range b.region.s { + list := regionToOther[id] + if len(list) == 1 { + likelyRegion[id].lang = uint16(b.langIndex(list[0].to[0])) + likelyRegion[id].script = uint8(b.script.index(list[0].to[1])) + if len(list[0].from) > 2 { + likelyRegion[id].flags = scriptInFrom + } + } else if len(list) > 1 { + likelyRegion[id].flags = isList + likelyRegion[id].lang = uint16(len(likelyRegionList)) + likelyRegion[id].script = uint8(len(list)) + for i, x := range list { + if len(x.from) == 2 && i != 0 || i > 0 && len(x.from) != 3 { + log.Fatalf("unspecified script must be first in list: %v at %d", x.from, i) + } + x := likelyLangScript{ + lang: uint16(b.langIndex(x.to[0])), + script: uint8(b.script.index(x.to[1])), + } + if len(list[0].from) > 2 { + x.flags = scriptInFrom + } + likelyRegionList = append(likelyRegionList, x) + } + } + } + b.writeType(likelyLangScript{}) + b.writeSlice("likelyRegion", likelyRegion) + b.writeSlice("likelyRegionList", likelyRegionList) + + b.writeType(likelyTag{}) + b.writeSlice("likelyRegionGroup", likelyRegionGroup) +} + +type mutualIntelligibility struct { + want, have uint16 + distance uint8 + oneway bool +} + +type scriptIntelligibility struct { + wantLang, haveLang uint16 + wantScript, haveScript uint8 + distance uint8 + // Always oneway +} + +type regionIntelligibility struct { + lang uint16 // compact language id + script uint8 // 0 means any + group uint8 // 0 means any; if bit 7 is set it means inverse + distance uint8 + // Always twoway. +} + +// writeMatchData writes tables with languages and scripts for which there is +// mutual intelligibility. The data is based on CLDR's languageMatching data. +// Note that we use a different algorithm than the one defined by CLDR and that +// we slightly modify the data. For example, we convert scores to confidence levels. +// We also drop all region-related data as we use a different algorithm to +// determine region equivalence. +func (b *builder) writeMatchData() { + lm := b.supp.LanguageMatching.LanguageMatches + cldr.MakeSlice(&lm).SelectAnyOf("type", "written_new") + + regionHierarchy := map[string][]string{} + for _, g := range b.supp.TerritoryContainment.Group { + regions := strings.Split(g.Contains, " ") + regionHierarchy[g.Type] = append(regionHierarchy[g.Type], regions...) + } + regionToGroups := make([]uint8, len(b.region.s)) + + idToIndex := map[string]uint8{} + for i, mv := range lm[0].MatchVariable { + if i > 6 { + log.Fatalf("Too many groups: %d", i) + } + idToIndex[mv.Id] = uint8(i + 1) + // TODO: also handle '-' + for _, r := range strings.Split(mv.Value, "+") { + todo := []string{r} + for k := 0; k < len(todo); k++ { + r := todo[k] + regionToGroups[b.region.index(r)] |= 1 << uint8(i) + todo = append(todo, regionHierarchy[r]...) + } + } + } + b.writeSlice("regionToGroups", regionToGroups) + + // maps language id to in- and out-of-group region. + paradigmLocales := [][3]uint16{} + locales := strings.Split(lm[0].ParadigmLocales[0].Locales, " ") + for i := 0; i < len(locales); i += 2 { + x := [3]uint16{} + for j := 0; j < 2; j++ { + pc := strings.SplitN(locales[i+j], "-", 2) + x[0] = b.langIndex(pc[0]) + if len(pc) == 2 { + x[1+j] = uint16(b.region.index(pc[1])) + } + } + paradigmLocales = append(paradigmLocales, x) + } + b.writeSlice("paradigmLocales", paradigmLocales) + + b.writeType(mutualIntelligibility{}) + b.writeType(scriptIntelligibility{}) + b.writeType(regionIntelligibility{}) + + matchLang := []mutualIntelligibility{} + matchScript := []scriptIntelligibility{} + matchRegion := []regionIntelligibility{} + // Convert the languageMatch entries in lists keyed by desired language. + for _, m := range lm[0].LanguageMatch { + // Different versions of CLDR use different separators. + desired := strings.Replace(m.Desired, "-", "_", -1) + supported := strings.Replace(m.Supported, "-", "_", -1) + d := strings.Split(desired, "_") + s := strings.Split(supported, "_") + if len(d) != len(s) { + log.Fatalf("not supported: desired=%q; supported=%q", desired, supported) + continue + } + distance, _ := strconv.ParseInt(m.Distance, 10, 8) + switch len(d) { + case 2: + if desired == supported && desired == "*_*" { + continue + } + // language-script pair. + matchScript = append(matchScript, scriptIntelligibility{ + wantLang: uint16(b.langIndex(d[0])), + haveLang: uint16(b.langIndex(s[0])), + wantScript: uint8(b.script.index(d[1])), + haveScript: uint8(b.script.index(s[1])), + distance: uint8(distance), + }) + if m.Oneway != "true" { + matchScript = append(matchScript, scriptIntelligibility{ + wantLang: uint16(b.langIndex(s[0])), + haveLang: uint16(b.langIndex(d[0])), + wantScript: uint8(b.script.index(s[1])), + haveScript: uint8(b.script.index(d[1])), + distance: uint8(distance), + }) + } + case 1: + if desired == supported && desired == "*" { + continue + } + if distance == 1 { + // nb == no is already handled by macro mapping. Check there + // really is only this case. + if d[0] != "no" || s[0] != "nb" { + log.Fatalf("unhandled equivalence %s == %s", s[0], d[0]) + } + continue + } + // TODO: consider dropping oneway field and just doubling the entry. + matchLang = append(matchLang, mutualIntelligibility{ + want: uint16(b.langIndex(d[0])), + have: uint16(b.langIndex(s[0])), + distance: uint8(distance), + oneway: m.Oneway == "true", + }) + case 3: + if desired == supported && desired == "*_*_*" { + continue + } + if desired != supported { + // This is now supported by CLDR, but only one case, which + // should already be covered by paradigm locales. For instance, + // test case "und, en, en-GU, en-IN, en-GB ; en-ZA ; en-GB" in + // testdata/CLDRLocaleMatcherTest.txt tests this. + if supported != "en_*_GB" { + log.Fatalf("not supported: desired=%q; supported=%q", desired, supported) + } + continue + } + ri := regionIntelligibility{ + lang: b.langIndex(d[0]), + distance: uint8(distance), + } + if d[1] != "*" { + ri.script = uint8(b.script.index(d[1])) + } + switch { + case d[2] == "*": + ri.group = 0x80 // not contained in anything + case strings.HasPrefix(d[2], "$!"): + ri.group = 0x80 + d[2] = "$" + d[2][len("$!"):] + fallthrough + case strings.HasPrefix(d[2], "$"): + ri.group |= idToIndex[d[2]] + } + matchRegion = append(matchRegion, ri) + default: + log.Fatalf("not supported: desired=%q; supported=%q", desired, supported) + } + } + sort.SliceStable(matchLang, func(i, j int) bool { + return matchLang[i].distance < matchLang[j].distance + }) + b.writeSlice("matchLang", matchLang) + + sort.SliceStable(matchScript, func(i, j int) bool { + return matchScript[i].distance < matchScript[j].distance + }) + b.writeSlice("matchScript", matchScript) + + sort.SliceStable(matchRegion, func(i, j int) bool { + return matchRegion[i].distance < matchRegion[j].distance + }) + b.writeSlice("matchRegion", matchRegion) +} + +func (b *builder) writeRegionInclusionData() { + var ( + // mm holds for each group the set of groups with a distance of 1. + mm = make(map[int][]index) + + // containment holds for each group the transitive closure of + // containment of other groups. + containment = make(map[index][]index) + ) + for _, g := range b.supp.TerritoryContainment.Group { + // Skip UN and EURO zone as they are flattening the containment + // relationship. + if g.Type == "EZ" || g.Type == "UN" { + continue + } + group := b.region.index(g.Type) + groupIdx := b.groups[group] + for _, mem := range strings.Split(g.Contains, " ") { + r := b.region.index(mem) + mm[r] = append(mm[r], groupIdx) + if g, ok := b.groups[r]; ok { + mm[group] = append(mm[group], g) + containment[groupIdx] = append(containment[groupIdx], g) + } + } + } + + regionContainment := make([]uint64, len(b.groups)) + for _, g := range b.groups { + l := containment[g] + + // Compute the transitive closure of containment. + for i := 0; i < len(l); i++ { + l = append(l, containment[l[i]]...) + } + + // Compute the bitmask. + regionContainment[g] = 1 << g + for _, v := range l { + regionContainment[g] |= 1 << v + } + } + b.writeSlice("regionContainment", regionContainment) + + regionInclusion := make([]uint8, len(b.region.s)) + bvs := make(map[uint64]index) + // Make the first bitvector positions correspond with the groups. + for r, i := range b.groups { + bv := uint64(1 << i) + for _, g := range mm[r] { + bv |= 1 << g + } + bvs[bv] = i + regionInclusion[r] = uint8(bvs[bv]) + } + for r := 1; r < len(b.region.s); r++ { + if _, ok := b.groups[r]; !ok { + bv := uint64(0) + for _, g := range mm[r] { + bv |= 1 << g + } + if bv == 0 { + // Pick the world for unspecified regions. + bv = 1 << b.groups[b.region.index("001")] + } + if _, ok := bvs[bv]; !ok { + bvs[bv] = index(len(bvs)) + } + regionInclusion[r] = uint8(bvs[bv]) + } + } + b.writeSlice("regionInclusion", regionInclusion) + regionInclusionBits := make([]uint64, len(bvs)) + for k, v := range bvs { + regionInclusionBits[v] = uint64(k) + } + // Add bit vectors for increasingly large distances until a fixed point is reached. + regionInclusionNext := []uint8{} + for i := 0; i < len(regionInclusionBits); i++ { + bits := regionInclusionBits[i] + next := bits + for i := uint(0); i < uint(len(b.groups)); i++ { + if bits&(1< b'. Using + // bytes.Replace will do. + out := bytes.Replace(buf.Bytes(), []byte("language."), nil, -1) + if err := ioutil.WriteFile("index.go", out, 0600); err != nil { + log.Fatalf("Could not create file index.go: %v", err) + } + }() + + m := map[language.Tag]bool{} + for _, lang := range data.Locales() { + // We include all locales unconditionally to be consistent with en_US. + // We want en_US, even though it has no data associated with it. + + // TODO: put any of the languages for which no data exists at the end + // of the index. This allows all components based on ICU to use that + // as the cutoff point. + // if x := data.RawLDML(lang); false || + // x.LocaleDisplayNames != nil || + // x.Characters != nil || + // x.Delimiters != nil || + // x.Measurement != nil || + // x.Dates != nil || + // x.Numbers != nil || + // x.Units != nil || + // x.ListPatterns != nil || + // x.Collations != nil || + // x.Segmentations != nil || + // x.Rbnf != nil || + // x.Annotations != nil || + // x.Metadata != nil { + + // TODO: support POSIX natively, albeit non-standard. + tag := language.Make(strings.Replace(lang, "_POSIX", "-u-va-posix", 1)) + m[tag] = true + // } + } + // Include locales for plural rules, which uses a different structure. + for _, plurals := range data.Supplemental().Plurals { + for _, rules := range plurals.PluralRules { + for _, lang := range strings.Split(rules.Locales, " ") { + m[language.Make(lang)] = true + } + } + } + + var core, special []language.Tag + + for t := range m { + if x := t.Extensions(); len(x) != 0 && fmt.Sprint(x) != "[u-va-posix]" { + log.Fatalf("Unexpected extension %v in %v", x, t) + } + if len(t.Variants()) == 0 && len(t.Extensions()) == 0 { + core = append(core, t) + } else { + special = append(special, t) + } + } + + w.WriteComment(` + NumCompactTags is the number of common tags. The maximum tag is + NumCompactTags-1.`) + w.WriteConst("NumCompactTags", len(core)+len(special)) + + sort.Sort(byAlpha(special)) + w.WriteVar("specialTags", special) + + // TODO: order by frequency? + sort.Sort(byAlpha(core)) + + // Size computations are just an estimate. + w.Size += int(reflect.TypeOf(map[uint32]uint16{}).Size()) + w.Size += len(core) * 6 // size of uint32 and uint16 + + fmt.Fprintln(w) + fmt.Fprintln(w, "var coreTags = map[uint32]uint16{") + fmt.Fprintln(w, "0x0: 0, // und") + i := len(special) + 1 // Und and special tags already written. + for _, t := range core { + if t == language.Und { + continue + } + fmt.Fprint(w.Hash, t, i) + b, s, r := t.Raw() + fmt.Fprintf(w, "0x%s%s%s: %d, // %s\n", + getIndex(b, 3), // 3 is enough as it is guaranteed to be a compact number + getIndex(s, 2), + getIndex(r, 3), + i, t) + i++ + } + fmt.Fprintln(w, "}") +} + +// getIndex prints the subtag type and extracts its index of size nibble. +// If the index is less than n nibbles, the result is prefixed with 0s. +func getIndex(x interface{}, n int) string { + s := fmt.Sprintf("%#v", x) // s is of form Type{typeID: 0x00} + s = s[strings.Index(s, "0x")+2 : len(s)-1] + return strings.Repeat("0", n-len(s)) + s +} + +type byAlpha []language.Tag + +func (a byAlpha) Len() int { return len(a) } +func (a byAlpha) Swap(i, j int) { a[i], a[j] = a[j], a[i] } +func (a byAlpha) Less(i, j int) bool { return a[i].String() < a[j].String() } diff --git a/vendor/golang.org/x/text/unicode/bidi/gen.go b/vendor/golang.org/x/text/unicode/bidi/gen.go new file mode 100644 index 000000000..4e1c7ba0b --- /dev/null +++ b/vendor/golang.org/x/text/unicode/bidi/gen.go @@ -0,0 +1,133 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +package main + +import ( + "flag" + "log" + + "golang.org/x/text/internal/gen" + "golang.org/x/text/internal/triegen" + "golang.org/x/text/internal/ucd" +) + +var outputFile = flag.String("out", "tables.go", "output file") + +func main() { + gen.Init() + gen.Repackage("gen_trieval.go", "trieval.go", "bidi") + gen.Repackage("gen_ranges.go", "ranges_test.go", "bidi") + + genTables() +} + +// bidiClass names and codes taken from class "bc" in +// http://www.unicode.org/Public/8.0.0/ucd/PropertyValueAliases.txt +var bidiClass = map[string]Class{ + "AL": AL, // ArabicLetter + "AN": AN, // ArabicNumber + "B": B, // ParagraphSeparator + "BN": BN, // BoundaryNeutral + "CS": CS, // CommonSeparator + "EN": EN, // EuropeanNumber + "ES": ES, // EuropeanSeparator + "ET": ET, // EuropeanTerminator + "L": L, // LeftToRight + "NSM": NSM, // NonspacingMark + "ON": ON, // OtherNeutral + "R": R, // RightToLeft + "S": S, // SegmentSeparator + "WS": WS, // WhiteSpace + + "FSI": Control, + "PDF": Control, + "PDI": Control, + "LRE": Control, + "LRI": Control, + "LRO": Control, + "RLE": Control, + "RLI": Control, + "RLO": Control, +} + +func genTables() { + if numClass > 0x0F { + log.Fatalf("Too many Class constants (%#x > 0x0F).", numClass) + } + w := gen.NewCodeWriter() + defer w.WriteVersionedGoFile(*outputFile, "bidi") + + gen.WriteUnicodeVersion(w) + + t := triegen.NewTrie("bidi") + + // Build data about bracket mapping. These bits need to be or-ed with + // any other bits. + orMask := map[rune]uint64{} + + xorMap := map[rune]int{} + xorMasks := []rune{0} // First value is no-op. + + ucd.Parse(gen.OpenUCDFile("BidiBrackets.txt"), func(p *ucd.Parser) { + r1 := p.Rune(0) + r2 := p.Rune(1) + xor := r1 ^ r2 + if _, ok := xorMap[xor]; !ok { + xorMap[xor] = len(xorMasks) + xorMasks = append(xorMasks, xor) + } + entry := uint64(xorMap[xor]) << xorMaskShift + switch p.String(2) { + case "o": + entry |= openMask + case "c", "n": + default: + log.Fatalf("Unknown bracket class %q.", p.String(2)) + } + orMask[r1] = entry + }) + + w.WriteComment(` + xorMasks contains masks to be xor-ed with brackets to get the reverse + version.`) + w.WriteVar("xorMasks", xorMasks) + + done := map[rune]bool{} + + insert := func(r rune, c Class) { + if !done[r] { + t.Insert(r, orMask[r]|uint64(c)) + done[r] = true + } + } + + // Insert the derived BiDi properties. + ucd.Parse(gen.OpenUCDFile("extracted/DerivedBidiClass.txt"), func(p *ucd.Parser) { + r := p.Rune(0) + class, ok := bidiClass[p.String(1)] + if !ok { + log.Fatalf("%U: Unknown BiDi class %q", r, p.String(1)) + } + insert(r, class) + }) + visitDefaults(insert) + + // TODO: use sparse blocks. This would reduce table size considerably + // from the looks of it. + + sz, err := t.Gen(w) + if err != nil { + log.Fatal(err) + } + w.Size += sz +} + +// dummy values to make methods in gen_common compile. The real versions +// will be generated by this file to tables.go. +var ( + xorMasks []rune +) diff --git a/vendor/golang.org/x/text/unicode/bidi/gen_ranges.go b/vendor/golang.org/x/text/unicode/bidi/gen_ranges.go new file mode 100644 index 000000000..51bd68fa7 --- /dev/null +++ b/vendor/golang.org/x/text/unicode/bidi/gen_ranges.go @@ -0,0 +1,57 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +package main + +import ( + "unicode" + + "golang.org/x/text/internal/gen" + "golang.org/x/text/internal/ucd" + "golang.org/x/text/unicode/rangetable" +) + +// These tables are hand-extracted from: +// http://www.unicode.org/Public/8.0.0/ucd/extracted/DerivedBidiClass.txt +func visitDefaults(fn func(r rune, c Class)) { + // first write default values for ranges listed above. + visitRunes(fn, AL, []rune{ + 0x0600, 0x07BF, // Arabic + 0x08A0, 0x08FF, // Arabic Extended-A + 0xFB50, 0xFDCF, // Arabic Presentation Forms + 0xFDF0, 0xFDFF, + 0xFE70, 0xFEFF, + 0x0001EE00, 0x0001EEFF, // Arabic Mathematical Alpha Symbols + }) + visitRunes(fn, R, []rune{ + 0x0590, 0x05FF, // Hebrew + 0x07C0, 0x089F, // Nko et al. + 0xFB1D, 0xFB4F, + 0x00010800, 0x00010FFF, // Cypriot Syllabary et. al. + 0x0001E800, 0x0001EDFF, + 0x0001EF00, 0x0001EFFF, + }) + visitRunes(fn, ET, []rune{ // European Terminator + 0x20A0, 0x20Cf, // Currency symbols + }) + rangetable.Visit(unicode.Noncharacter_Code_Point, func(r rune) { + fn(r, BN) // Boundary Neutral + }) + ucd.Parse(gen.OpenUCDFile("DerivedCoreProperties.txt"), func(p *ucd.Parser) { + if p.String(1) == "Default_Ignorable_Code_Point" { + fn(p.Rune(0), BN) // Boundary Neutral + } + }) +} + +func visitRunes(fn func(r rune, c Class), c Class, runes []rune) { + for i := 0; i < len(runes); i += 2 { + lo, hi := runes[i], runes[i+1] + for j := lo; j <= hi; j++ { + fn(j, c) + } + } +} diff --git a/vendor/golang.org/x/text/unicode/bidi/gen_trieval.go b/vendor/golang.org/x/text/unicode/bidi/gen_trieval.go new file mode 100644 index 000000000..9cb994289 --- /dev/null +++ b/vendor/golang.org/x/text/unicode/bidi/gen_trieval.go @@ -0,0 +1,64 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +package main + +// Class is the Unicode BiDi class. Each rune has a single class. +type Class uint + +const ( + L Class = iota // LeftToRight + R // RightToLeft + EN // EuropeanNumber + ES // EuropeanSeparator + ET // EuropeanTerminator + AN // ArabicNumber + CS // CommonSeparator + B // ParagraphSeparator + S // SegmentSeparator + WS // WhiteSpace + ON // OtherNeutral + BN // BoundaryNeutral + NSM // NonspacingMark + AL // ArabicLetter + Control // Control LRO - PDI + + numClass + + LRO // LeftToRightOverride + RLO // RightToLeftOverride + LRE // LeftToRightEmbedding + RLE // RightToLeftEmbedding + PDF // PopDirectionalFormat + LRI // LeftToRightIsolate + RLI // RightToLeftIsolate + FSI // FirstStrongIsolate + PDI // PopDirectionalIsolate + + unknownClass = ^Class(0) +) + +var controlToClass = map[rune]Class{ + 0x202D: LRO, // LeftToRightOverride, + 0x202E: RLO, // RightToLeftOverride, + 0x202A: LRE, // LeftToRightEmbedding, + 0x202B: RLE, // RightToLeftEmbedding, + 0x202C: PDF, // PopDirectionalFormat, + 0x2066: LRI, // LeftToRightIsolate, + 0x2067: RLI, // RightToLeftIsolate, + 0x2068: FSI, // FirstStrongIsolate, + 0x2069: PDI, // PopDirectionalIsolate, +} + +// A trie entry has the following bits: +// 7..5 XOR mask for brackets +// 4 1: Bracket open, 0: Bracket close +// 3..0 Class type + +const ( + openMask = 0x10 + xorMaskShift = 5 +) diff --git a/vendor/golang.org/x/text/unicode/norm/maketables.go b/vendor/golang.org/x/text/unicode/norm/maketables.go new file mode 100644 index 000000000..338c395ee --- /dev/null +++ b/vendor/golang.org/x/text/unicode/norm/maketables.go @@ -0,0 +1,976 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +// Normalization table generator. +// Data read from the web. +// See forminfo.go for a description of the trie values associated with each rune. + +package main + +import ( + "bytes" + "flag" + "fmt" + "io" + "log" + "sort" + "strconv" + "strings" + + "golang.org/x/text/internal/gen" + "golang.org/x/text/internal/triegen" + "golang.org/x/text/internal/ucd" +) + +func main() { + gen.Init() + loadUnicodeData() + compactCCC() + loadCompositionExclusions() + completeCharFields(FCanonical) + completeCharFields(FCompatibility) + computeNonStarterCounts() + verifyComputed() + printChars() + testDerived() + printTestdata() + makeTables() +} + +var ( + tablelist = flag.String("tables", + "all", + "comma-separated list of which tables to generate; "+ + "can be 'decomp', 'recomp', 'info' and 'all'") + test = flag.Bool("test", + false, + "test existing tables against DerivedNormalizationProps and generate test data for regression testing") + verbose = flag.Bool("verbose", + false, + "write data to stdout as it is parsed") +) + +const MaxChar = 0x10FFFF // anything above this shouldn't exist + +// Quick Check properties of runes allow us to quickly +// determine whether a rune may occur in a normal form. +// For a given normal form, a rune may be guaranteed to occur +// verbatim (QC=Yes), may or may not combine with another +// rune (QC=Maybe), or may not occur (QC=No). +type QCResult int + +const ( + QCUnknown QCResult = iota + QCYes + QCNo + QCMaybe +) + +func (r QCResult) String() string { + switch r { + case QCYes: + return "Yes" + case QCNo: + return "No" + case QCMaybe: + return "Maybe" + } + return "***UNKNOWN***" +} + +const ( + FCanonical = iota // NFC or NFD + FCompatibility // NFKC or NFKD + FNumberOfFormTypes +) + +const ( + MComposed = iota // NFC or NFKC + MDecomposed // NFD or NFKD + MNumberOfModes +) + +// This contains only the properties we're interested in. +type Char struct { + name string + codePoint rune // if zero, this index is not a valid code point. + ccc uint8 // canonical combining class + origCCC uint8 + excludeInComp bool // from CompositionExclusions.txt + compatDecomp bool // it has a compatibility expansion + + nTrailingNonStarters uint8 + nLeadingNonStarters uint8 // must be equal to trailing if non-zero + + forms [FNumberOfFormTypes]FormInfo // For FCanonical and FCompatibility + + state State +} + +var chars = make([]Char, MaxChar+1) +var cccMap = make(map[uint8]uint8) + +func (c Char) String() string { + buf := new(bytes.Buffer) + + fmt.Fprintf(buf, "%U [%s]:\n", c.codePoint, c.name) + fmt.Fprintf(buf, " ccc: %v\n", c.ccc) + fmt.Fprintf(buf, " excludeInComp: %v\n", c.excludeInComp) + fmt.Fprintf(buf, " compatDecomp: %v\n", c.compatDecomp) + fmt.Fprintf(buf, " state: %v\n", c.state) + fmt.Fprintf(buf, " NFC:\n") + fmt.Fprint(buf, c.forms[FCanonical]) + fmt.Fprintf(buf, " NFKC:\n") + fmt.Fprint(buf, c.forms[FCompatibility]) + + return buf.String() +} + +// In UnicodeData.txt, some ranges are marked like this: +// 3400;;Lo;0;L;;;;;N;;;;; +// 4DB5;;Lo;0;L;;;;;N;;;;; +// parseCharacter keeps a state variable indicating the weirdness. +type State int + +const ( + SNormal State = iota // known to be zero for the type + SFirst + SLast + SMissing +) + +var lastChar = rune('\u0000') + +func (c Char) isValid() bool { + return c.codePoint != 0 && c.state != SMissing +} + +type FormInfo struct { + quickCheck [MNumberOfModes]QCResult // index: MComposed or MDecomposed + verified [MNumberOfModes]bool // index: MComposed or MDecomposed + + combinesForward bool // May combine with rune on the right + combinesBackward bool // May combine with rune on the left + isOneWay bool // Never appears in result + inDecomp bool // Some decompositions result in this char. + decomp Decomposition + expandedDecomp Decomposition +} + +func (f FormInfo) String() string { + buf := bytes.NewBuffer(make([]byte, 0)) + + fmt.Fprintf(buf, " quickCheck[C]: %v\n", f.quickCheck[MComposed]) + fmt.Fprintf(buf, " quickCheck[D]: %v\n", f.quickCheck[MDecomposed]) + fmt.Fprintf(buf, " cmbForward: %v\n", f.combinesForward) + fmt.Fprintf(buf, " cmbBackward: %v\n", f.combinesBackward) + fmt.Fprintf(buf, " isOneWay: %v\n", f.isOneWay) + fmt.Fprintf(buf, " inDecomp: %v\n", f.inDecomp) + fmt.Fprintf(buf, " decomposition: %X\n", f.decomp) + fmt.Fprintf(buf, " expandedDecomp: %X\n", f.expandedDecomp) + + return buf.String() +} + +type Decomposition []rune + +func parseDecomposition(s string, skipfirst bool) (a []rune, err error) { + decomp := strings.Split(s, " ") + if len(decomp) > 0 && skipfirst { + decomp = decomp[1:] + } + for _, d := range decomp { + point, err := strconv.ParseUint(d, 16, 64) + if err != nil { + return a, err + } + a = append(a, rune(point)) + } + return a, nil +} + +func loadUnicodeData() { + f := gen.OpenUCDFile("UnicodeData.txt") + defer f.Close() + p := ucd.New(f) + for p.Next() { + r := p.Rune(ucd.CodePoint) + char := &chars[r] + + char.ccc = uint8(p.Uint(ucd.CanonicalCombiningClass)) + decmap := p.String(ucd.DecompMapping) + + exp, err := parseDecomposition(decmap, false) + isCompat := false + if err != nil { + if len(decmap) > 0 { + exp, err = parseDecomposition(decmap, true) + if err != nil { + log.Fatalf(`%U: bad decomp |%v|: "%s"`, r, decmap, err) + } + isCompat = true + } + } + + char.name = p.String(ucd.Name) + char.codePoint = r + char.forms[FCompatibility].decomp = exp + if !isCompat { + char.forms[FCanonical].decomp = exp + } else { + char.compatDecomp = true + } + if len(decmap) > 0 { + char.forms[FCompatibility].decomp = exp + } + } + if err := p.Err(); err != nil { + log.Fatal(err) + } +} + +// compactCCC converts the sparse set of CCC values to a continguous one, +// reducing the number of bits needed from 8 to 6. +func compactCCC() { + m := make(map[uint8]uint8) + for i := range chars { + c := &chars[i] + m[c.ccc] = 0 + } + cccs := []int{} + for v, _ := range m { + cccs = append(cccs, int(v)) + } + sort.Ints(cccs) + for i, c := range cccs { + cccMap[uint8(i)] = uint8(c) + m[uint8(c)] = uint8(i) + } + for i := range chars { + c := &chars[i] + c.origCCC = c.ccc + c.ccc = m[c.ccc] + } + if len(m) >= 1<<6 { + log.Fatalf("too many difference CCC values: %d >= 64", len(m)) + } +} + +// CompositionExclusions.txt has form: +// 0958 # ... +// See http://unicode.org/reports/tr44/ for full explanation +func loadCompositionExclusions() { + f := gen.OpenUCDFile("CompositionExclusions.txt") + defer f.Close() + p := ucd.New(f) + for p.Next() { + c := &chars[p.Rune(0)] + if c.excludeInComp { + log.Fatalf("%U: Duplicate entry in exclusions.", c.codePoint) + } + c.excludeInComp = true + } + if e := p.Err(); e != nil { + log.Fatal(e) + } +} + +// hasCompatDecomp returns true if any of the recursive +// decompositions contains a compatibility expansion. +// In this case, the character may not occur in NFK*. +func hasCompatDecomp(r rune) bool { + c := &chars[r] + if c.compatDecomp { + return true + } + for _, d := range c.forms[FCompatibility].decomp { + if hasCompatDecomp(d) { + return true + } + } + return false +} + +// Hangul related constants. +const ( + HangulBase = 0xAC00 + HangulEnd = 0xD7A4 // hangulBase + Jamo combinations (19 * 21 * 28) + + JamoLBase = 0x1100 + JamoLEnd = 0x1113 + JamoVBase = 0x1161 + JamoVEnd = 0x1176 + JamoTBase = 0x11A8 + JamoTEnd = 0x11C3 + + JamoLVTCount = 19 * 21 * 28 + JamoTCount = 28 +) + +func isHangul(r rune) bool { + return HangulBase <= r && r < HangulEnd +} + +func isHangulWithoutJamoT(r rune) bool { + if !isHangul(r) { + return false + } + r -= HangulBase + return r < JamoLVTCount && r%JamoTCount == 0 +} + +func ccc(r rune) uint8 { + return chars[r].ccc +} + +// Insert a rune in a buffer, ordered by Canonical Combining Class. +func insertOrdered(b Decomposition, r rune) Decomposition { + n := len(b) + b = append(b, 0) + cc := ccc(r) + if cc > 0 { + // Use bubble sort. + for ; n > 0; n-- { + if ccc(b[n-1]) <= cc { + break + } + b[n] = b[n-1] + } + } + b[n] = r + return b +} + +// Recursively decompose. +func decomposeRecursive(form int, r rune, d Decomposition) Decomposition { + dcomp := chars[r].forms[form].decomp + if len(dcomp) == 0 { + return insertOrdered(d, r) + } + for _, c := range dcomp { + d = decomposeRecursive(form, c, d) + } + return d +} + +func completeCharFields(form int) { + // Phase 0: pre-expand decomposition. + for i := range chars { + f := &chars[i].forms[form] + if len(f.decomp) == 0 { + continue + } + exp := make(Decomposition, 0) + for _, c := range f.decomp { + exp = decomposeRecursive(form, c, exp) + } + f.expandedDecomp = exp + } + + // Phase 1: composition exclusion, mark decomposition. + for i := range chars { + c := &chars[i] + f := &c.forms[form] + + // Marks script-specific exclusions and version restricted. + f.isOneWay = c.excludeInComp + + // Singletons + f.isOneWay = f.isOneWay || len(f.decomp) == 1 + + // Non-starter decompositions + if len(f.decomp) > 1 { + chk := c.ccc != 0 || chars[f.decomp[0]].ccc != 0 + f.isOneWay = f.isOneWay || chk + } + + // Runes that decompose into more than two runes. + f.isOneWay = f.isOneWay || len(f.decomp) > 2 + + if form == FCompatibility { + f.isOneWay = f.isOneWay || hasCompatDecomp(c.codePoint) + } + + for _, r := range f.decomp { + chars[r].forms[form].inDecomp = true + } + } + + // Phase 2: forward and backward combining. + for i := range chars { + c := &chars[i] + f := &c.forms[form] + + if !f.isOneWay && len(f.decomp) == 2 { + f0 := &chars[f.decomp[0]].forms[form] + f1 := &chars[f.decomp[1]].forms[form] + if !f0.isOneWay { + f0.combinesForward = true + } + if !f1.isOneWay { + f1.combinesBackward = true + } + } + if isHangulWithoutJamoT(rune(i)) { + f.combinesForward = true + } + } + + // Phase 3: quick check values. + for i := range chars { + c := &chars[i] + f := &c.forms[form] + + switch { + case len(f.decomp) > 0: + f.quickCheck[MDecomposed] = QCNo + case isHangul(rune(i)): + f.quickCheck[MDecomposed] = QCNo + default: + f.quickCheck[MDecomposed] = QCYes + } + switch { + case f.isOneWay: + f.quickCheck[MComposed] = QCNo + case (i & 0xffff00) == JamoLBase: + f.quickCheck[MComposed] = QCYes + if JamoLBase <= i && i < JamoLEnd { + f.combinesForward = true + } + if JamoVBase <= i && i < JamoVEnd { + f.quickCheck[MComposed] = QCMaybe + f.combinesBackward = true + f.combinesForward = true + } + if JamoTBase <= i && i < JamoTEnd { + f.quickCheck[MComposed] = QCMaybe + f.combinesBackward = true + } + case !f.combinesBackward: + f.quickCheck[MComposed] = QCYes + default: + f.quickCheck[MComposed] = QCMaybe + } + } +} + +func computeNonStarterCounts() { + // Phase 4: leading and trailing non-starter count + for i := range chars { + c := &chars[i] + + runes := []rune{rune(i)} + // We always use FCompatibility so that the CGJ insertion points do not + // change for repeated normalizations with different forms. + if exp := c.forms[FCompatibility].expandedDecomp; len(exp) > 0 { + runes = exp + } + // We consider runes that combine backwards to be non-starters for the + // purpose of Stream-Safe Text Processing. + for _, r := range runes { + if cr := &chars[r]; cr.ccc == 0 && !cr.forms[FCompatibility].combinesBackward { + break + } + c.nLeadingNonStarters++ + } + for i := len(runes) - 1; i >= 0; i-- { + if cr := &chars[runes[i]]; cr.ccc == 0 && !cr.forms[FCompatibility].combinesBackward { + break + } + c.nTrailingNonStarters++ + } + if c.nTrailingNonStarters > 3 { + log.Fatalf("%U: Decomposition with more than 3 (%d) trailing modifiers (%U)", i, c.nTrailingNonStarters, runes) + } + + if isHangul(rune(i)) { + c.nTrailingNonStarters = 2 + if isHangulWithoutJamoT(rune(i)) { + c.nTrailingNonStarters = 1 + } + } + + if l, t := c.nLeadingNonStarters, c.nTrailingNonStarters; l > 0 && l != t { + log.Fatalf("%U: number of leading and trailing non-starters should be equal (%d vs %d)", i, l, t) + } + if t := c.nTrailingNonStarters; t > 3 { + log.Fatalf("%U: number of trailing non-starters is %d > 3", t) + } + } +} + +func printBytes(w io.Writer, b []byte, name string) { + fmt.Fprintf(w, "// %s: %d bytes\n", name, len(b)) + fmt.Fprintf(w, "var %s = [...]byte {", name) + for i, c := range b { + switch { + case i%64 == 0: + fmt.Fprintf(w, "\n// Bytes %x - %x\n", i, i+63) + case i%8 == 0: + fmt.Fprintf(w, "\n") + } + fmt.Fprintf(w, "0x%.2X, ", c) + } + fmt.Fprint(w, "\n}\n\n") +} + +// See forminfo.go for format. +func makeEntry(f *FormInfo, c *Char) uint16 { + e := uint16(0) + if r := c.codePoint; HangulBase <= r && r < HangulEnd { + e |= 0x40 + } + if f.combinesForward { + e |= 0x20 + } + if f.quickCheck[MDecomposed] == QCNo { + e |= 0x4 + } + switch f.quickCheck[MComposed] { + case QCYes: + case QCNo: + e |= 0x10 + case QCMaybe: + e |= 0x18 + default: + log.Fatalf("Illegal quickcheck value %v.", f.quickCheck[MComposed]) + } + e |= uint16(c.nTrailingNonStarters) + return e +} + +// decompSet keeps track of unique decompositions, grouped by whether +// the decomposition is followed by a trailing and/or leading CCC. +type decompSet [7]map[string]bool + +const ( + normalDecomp = iota + firstMulti + firstCCC + endMulti + firstLeadingCCC + firstCCCZeroExcept + firstStarterWithNLead + lastDecomp +) + +var cname = []string{"firstMulti", "firstCCC", "endMulti", "firstLeadingCCC", "firstCCCZeroExcept", "firstStarterWithNLead", "lastDecomp"} + +func makeDecompSet() decompSet { + m := decompSet{} + for i := range m { + m[i] = make(map[string]bool) + } + return m +} +func (m *decompSet) insert(key int, s string) { + m[key][s] = true +} + +func printCharInfoTables(w io.Writer) int { + mkstr := func(r rune, f *FormInfo) (int, string) { + d := f.expandedDecomp + s := string([]rune(d)) + if max := 1 << 6; len(s) >= max { + const msg = "%U: too many bytes in decomposition: %d >= %d" + log.Fatalf(msg, r, len(s), max) + } + head := uint8(len(s)) + if f.quickCheck[MComposed] != QCYes { + head |= 0x40 + } + if f.combinesForward { + head |= 0x80 + } + s = string([]byte{head}) + s + + lccc := ccc(d[0]) + tccc := ccc(d[len(d)-1]) + cc := ccc(r) + if cc != 0 && lccc == 0 && tccc == 0 { + log.Fatalf("%U: trailing and leading ccc are 0 for non-zero ccc %d", r, cc) + } + if tccc < lccc && lccc != 0 { + const msg = "%U: lccc (%d) must be <= tcc (%d)" + log.Fatalf(msg, r, lccc, tccc) + } + index := normalDecomp + nTrail := chars[r].nTrailingNonStarters + nLead := chars[r].nLeadingNonStarters + if tccc > 0 || lccc > 0 || nTrail > 0 { + tccc <<= 2 + tccc |= nTrail + s += string([]byte{tccc}) + index = endMulti + for _, r := range d[1:] { + if ccc(r) == 0 { + index = firstCCC + } + } + if lccc > 0 || nLead > 0 { + s += string([]byte{lccc}) + if index == firstCCC { + log.Fatalf("%U: multi-segment decomposition not supported for decompositions with leading CCC != 0", r) + } + index = firstLeadingCCC + } + if cc != lccc { + if cc != 0 { + log.Fatalf("%U: for lccc != ccc, expected ccc to be 0; was %d", r, cc) + } + index = firstCCCZeroExcept + } + } else if len(d) > 1 { + index = firstMulti + } + return index, s + } + + decompSet := makeDecompSet() + const nLeadStr = "\x00\x01" // 0-byte length and tccc with nTrail. + decompSet.insert(firstStarterWithNLead, nLeadStr) + + // Store the uniqued decompositions in a byte buffer, + // preceded by their byte length. + for _, c := range chars { + for _, f := range c.forms { + if len(f.expandedDecomp) == 0 { + continue + } + if f.combinesBackward { + log.Fatalf("%U: combinesBackward and decompose", c.codePoint) + } + index, s := mkstr(c.codePoint, &f) + decompSet.insert(index, s) + } + } + + decompositions := bytes.NewBuffer(make([]byte, 0, 10000)) + size := 0 + positionMap := make(map[string]uint16) + decompositions.WriteString("\000") + fmt.Fprintln(w, "const (") + for i, m := range decompSet { + sa := []string{} + for s := range m { + sa = append(sa, s) + } + sort.Strings(sa) + for _, s := range sa { + p := decompositions.Len() + decompositions.WriteString(s) + positionMap[s] = uint16(p) + } + if cname[i] != "" { + fmt.Fprintf(w, "%s = 0x%X\n", cname[i], decompositions.Len()) + } + } + fmt.Fprintln(w, "maxDecomp = 0x8000") + fmt.Fprintln(w, ")") + b := decompositions.Bytes() + printBytes(w, b, "decomps") + size += len(b) + + varnames := []string{"nfc", "nfkc"} + for i := 0; i < FNumberOfFormTypes; i++ { + trie := triegen.NewTrie(varnames[i]) + + for r, c := range chars { + f := c.forms[i] + d := f.expandedDecomp + if len(d) != 0 { + _, key := mkstr(c.codePoint, &f) + trie.Insert(rune(r), uint64(positionMap[key])) + if c.ccc != ccc(d[0]) { + // We assume the lead ccc of a decomposition !=0 in this case. + if ccc(d[0]) == 0 { + log.Fatalf("Expected leading CCC to be non-zero; ccc is %d", c.ccc) + } + } + } else if c.nLeadingNonStarters > 0 && len(f.expandedDecomp) == 0 && c.ccc == 0 && !f.combinesBackward { + // Handle cases where it can't be detected that the nLead should be equal + // to nTrail. + trie.Insert(c.codePoint, uint64(positionMap[nLeadStr])) + } else if v := makeEntry(&f, &c)<<8 | uint16(c.ccc); v != 0 { + trie.Insert(c.codePoint, uint64(0x8000|v)) + } + } + sz, err := trie.Gen(w, triegen.Compact(&normCompacter{name: varnames[i]})) + if err != nil { + log.Fatal(err) + } + size += sz + } + return size +} + +func contains(sa []string, s string) bool { + for _, a := range sa { + if a == s { + return true + } + } + return false +} + +func makeTables() { + w := &bytes.Buffer{} + + size := 0 + if *tablelist == "" { + return + } + list := strings.Split(*tablelist, ",") + if *tablelist == "all" { + list = []string{"recomp", "info"} + } + + // Compute maximum decomposition size. + max := 0 + for _, c := range chars { + if n := len(string(c.forms[FCompatibility].expandedDecomp)); n > max { + max = n + } + } + + fmt.Fprintln(w, "const (") + fmt.Fprintln(w, "\t// Version is the Unicode edition from which the tables are derived.") + fmt.Fprintf(w, "\tVersion = %q\n", gen.UnicodeVersion()) + fmt.Fprintln(w) + fmt.Fprintln(w, "\t// MaxTransformChunkSize indicates the maximum number of bytes that Transform") + fmt.Fprintln(w, "\t// may need to write atomically for any Form. Making a destination buffer at") + fmt.Fprintln(w, "\t// least this size ensures that Transform can always make progress and that") + fmt.Fprintln(w, "\t// the user does not need to grow the buffer on an ErrShortDst.") + fmt.Fprintf(w, "\tMaxTransformChunkSize = %d+maxNonStarters*4\n", len(string(0x034F))+max) + fmt.Fprintln(w, ")\n") + + // Print the CCC remap table. + size += len(cccMap) + fmt.Fprintf(w, "var ccc = [%d]uint8{", len(cccMap)) + for i := 0; i < len(cccMap); i++ { + if i%8 == 0 { + fmt.Fprintln(w) + } + fmt.Fprintf(w, "%3d, ", cccMap[uint8(i)]) + } + fmt.Fprintln(w, "\n}\n") + + if contains(list, "info") { + size += printCharInfoTables(w) + } + + if contains(list, "recomp") { + // Note that we use 32 bit keys, instead of 64 bit. + // This clips the bits of three entries, but we know + // this won't cause a collision. The compiler will catch + // any changes made to UnicodeData.txt that introduces + // a collision. + // Note that the recomposition map for NFC and NFKC + // are identical. + + // Recomposition map + nrentries := 0 + for _, c := range chars { + f := c.forms[FCanonical] + if !f.isOneWay && len(f.decomp) > 0 { + nrentries++ + } + } + sz := nrentries * 8 + size += sz + fmt.Fprintf(w, "// recompMap: %d bytes (entries only)\n", sz) + fmt.Fprintln(w, "var recompMap = map[uint32]rune{") + for i, c := range chars { + f := c.forms[FCanonical] + d := f.decomp + if !f.isOneWay && len(d) > 0 { + key := uint32(uint16(d[0]))<<16 + uint32(uint16(d[1])) + fmt.Fprintf(w, "0x%.8X: 0x%.4X,\n", key, i) + } + } + fmt.Fprintf(w, "}\n\n") + } + + fmt.Fprintf(w, "// Total size of tables: %dKB (%d bytes)\n", (size+512)/1024, size) + gen.WriteVersionedGoFile("tables.go", "norm", w.Bytes()) +} + +func printChars() { + if *verbose { + for _, c := range chars { + if !c.isValid() || c.state == SMissing { + continue + } + fmt.Println(c) + } + } +} + +// verifyComputed does various consistency tests. +func verifyComputed() { + for i, c := range chars { + for _, f := range c.forms { + isNo := (f.quickCheck[MDecomposed] == QCNo) + if (len(f.decomp) > 0) != isNo && !isHangul(rune(i)) { + log.Fatalf("%U: NF*D QC must be No if rune decomposes", i) + } + + isMaybe := f.quickCheck[MComposed] == QCMaybe + if f.combinesBackward != isMaybe { + log.Fatalf("%U: NF*C QC must be Maybe if combinesBackward", i) + } + if len(f.decomp) > 0 && f.combinesForward && isMaybe { + log.Fatalf("%U: NF*C QC must be Yes or No if combinesForward and decomposes", i) + } + + if len(f.expandedDecomp) != 0 { + continue + } + if a, b := c.nLeadingNonStarters > 0, (c.ccc > 0 || f.combinesBackward); a != b { + // We accept these runes to be treated differently (it only affects + // segment breaking in iteration, most likely on improper use), but + // reconsider if more characters are added. + // U+FF9E HALFWIDTH KATAKANA VOICED SOUND MARK;Lm;0;L; 3099;;;;N;;;;; + // U+FF9F HALFWIDTH KATAKANA SEMI-VOICED SOUND MARK;Lm;0;L; 309A;;;;N;;;;; + // U+3133 HANGUL LETTER KIYEOK-SIOS;Lo;0;L; 11AA;;;;N;HANGUL LETTER GIYEOG SIOS;;;; + // U+318E HANGUL LETTER ARAEAE;Lo;0;L; 11A1;;;;N;HANGUL LETTER ALAE AE;;;; + // U+FFA3 HALFWIDTH HANGUL LETTER KIYEOK-SIOS;Lo;0;L; 3133;;;;N;HALFWIDTH HANGUL LETTER GIYEOG SIOS;;;; + // U+FFDC HALFWIDTH HANGUL LETTER I;Lo;0;L; 3163;;;;N;;;;; + if i != 0xFF9E && i != 0xFF9F && !(0x3133 <= i && i <= 0x318E) && !(0xFFA3 <= i && i <= 0xFFDC) { + log.Fatalf("%U: nLead was %v; want %v", i, a, b) + } + } + } + nfc := c.forms[FCanonical] + nfkc := c.forms[FCompatibility] + if nfc.combinesBackward != nfkc.combinesBackward { + log.Fatalf("%U: Cannot combine combinesBackward\n", c.codePoint) + } + } +} + +// Use values in DerivedNormalizationProps.txt to compare against the +// values we computed. +// DerivedNormalizationProps.txt has form: +// 00C0..00C5 ; NFD_QC; N # ... +// 0374 ; NFD_QC; N # ... +// See http://unicode.org/reports/tr44/ for full explanation +func testDerived() { + f := gen.OpenUCDFile("DerivedNormalizationProps.txt") + defer f.Close() + p := ucd.New(f) + for p.Next() { + r := p.Rune(0) + c := &chars[r] + + var ftype, mode int + qt := p.String(1) + switch qt { + case "NFC_QC": + ftype, mode = FCanonical, MComposed + case "NFD_QC": + ftype, mode = FCanonical, MDecomposed + case "NFKC_QC": + ftype, mode = FCompatibility, MComposed + case "NFKD_QC": + ftype, mode = FCompatibility, MDecomposed + default: + continue + } + var qr QCResult + switch p.String(2) { + case "Y": + qr = QCYes + case "N": + qr = QCNo + case "M": + qr = QCMaybe + default: + log.Fatalf(`Unexpected quick check value "%s"`, p.String(2)) + } + if got := c.forms[ftype].quickCheck[mode]; got != qr { + log.Printf("%U: FAILED %s (was %v need %v)\n", r, qt, got, qr) + } + c.forms[ftype].verified[mode] = true + } + if err := p.Err(); err != nil { + log.Fatal(err) + } + // Any unspecified value must be QCYes. Verify this. + for i, c := range chars { + for j, fd := range c.forms { + for k, qr := range fd.quickCheck { + if !fd.verified[k] && qr != QCYes { + m := "%U: FAIL F:%d M:%d (was %v need Yes) %s\n" + log.Printf(m, i, j, k, qr, c.name) + } + } + } + } +} + +var testHeader = `const ( + Yes = iota + No + Maybe +) + +type formData struct { + qc uint8 + combinesForward bool + decomposition string +} + +type runeData struct { + r rune + ccc uint8 + nLead uint8 + nTrail uint8 + f [2]formData // 0: canonical; 1: compatibility +} + +func f(qc uint8, cf bool, dec string) [2]formData { + return [2]formData{{qc, cf, dec}, {qc, cf, dec}} +} + +func g(qc, qck uint8, cf, cfk bool, d, dk string) [2]formData { + return [2]formData{{qc, cf, d}, {qck, cfk, dk}} +} + +var testData = []runeData{ +` + +func printTestdata() { + type lastInfo struct { + ccc uint8 + nLead uint8 + nTrail uint8 + f string + } + + last := lastInfo{} + w := &bytes.Buffer{} + fmt.Fprintf(w, testHeader) + for r, c := range chars { + f := c.forms[FCanonical] + qc, cf, d := f.quickCheck[MComposed], f.combinesForward, string(f.expandedDecomp) + f = c.forms[FCompatibility] + qck, cfk, dk := f.quickCheck[MComposed], f.combinesForward, string(f.expandedDecomp) + s := "" + if d == dk && qc == qck && cf == cfk { + s = fmt.Sprintf("f(%s, %v, %q)", qc, cf, d) + } else { + s = fmt.Sprintf("g(%s, %s, %v, %v, %q, %q)", qc, qck, cf, cfk, d, dk) + } + current := lastInfo{c.ccc, c.nLeadingNonStarters, c.nTrailingNonStarters, s} + if last != current { + fmt.Fprintf(w, "\t{0x%x, %d, %d, %d, %s},\n", r, c.origCCC, c.nLeadingNonStarters, c.nTrailingNonStarters, s) + last = current + } + } + fmt.Fprintln(w, "}") + gen.WriteVersionedGoFile("data_test.go", "norm", w.Bytes()) +} diff --git a/vendor/golang.org/x/text/unicode/norm/triegen.go b/vendor/golang.org/x/text/unicode/norm/triegen.go new file mode 100644 index 000000000..45d711900 --- /dev/null +++ b/vendor/golang.org/x/text/unicode/norm/triegen.go @@ -0,0 +1,117 @@ +// Copyright 2011 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +// Trie table generator. +// Used by make*tables tools to generate a go file with trie data structures +// for mapping UTF-8 to a 16-bit value. All but the last byte in a UTF-8 byte +// sequence are used to lookup offsets in the index table to be used for the +// next byte. The last byte is used to index into a table with 16-bit values. + +package main + +import ( + "fmt" + "io" +) + +const maxSparseEntries = 16 + +type normCompacter struct { + sparseBlocks [][]uint64 + sparseOffset []uint16 + sparseCount int + name string +} + +func mostFrequentStride(a []uint64) int { + counts := make(map[int]int) + var v int + for _, x := range a { + if stride := int(x) - v; v != 0 && stride >= 0 { + counts[stride]++ + } + v = int(x) + } + var maxs, maxc int + for stride, cnt := range counts { + if cnt > maxc || (cnt == maxc && stride < maxs) { + maxs, maxc = stride, cnt + } + } + return maxs +} + +func countSparseEntries(a []uint64) int { + stride := mostFrequentStride(a) + var v, count int + for _, tv := range a { + if int(tv)-v != stride { + if tv != 0 { + count++ + } + } + v = int(tv) + } + return count +} + +func (c *normCompacter) Size(v []uint64) (sz int, ok bool) { + if n := countSparseEntries(v); n <= maxSparseEntries { + return (n+1)*4 + 2, true + } + return 0, false +} + +func (c *normCompacter) Store(v []uint64) uint32 { + h := uint32(len(c.sparseOffset)) + c.sparseBlocks = append(c.sparseBlocks, v) + c.sparseOffset = append(c.sparseOffset, uint16(c.sparseCount)) + c.sparseCount += countSparseEntries(v) + 1 + return h +} + +func (c *normCompacter) Handler() string { + return c.name + "Sparse.lookup" +} + +func (c *normCompacter) Print(w io.Writer) (retErr error) { + p := func(f string, x ...interface{}) { + if _, err := fmt.Fprintf(w, f, x...); retErr == nil && err != nil { + retErr = err + } + } + + ls := len(c.sparseBlocks) + p("// %sSparseOffset: %d entries, %d bytes\n", c.name, ls, ls*2) + p("var %sSparseOffset = %#v\n\n", c.name, c.sparseOffset) + + ns := c.sparseCount + p("// %sSparseValues: %d entries, %d bytes\n", c.name, ns, ns*4) + p("var %sSparseValues = [%d]valueRange {", c.name, ns) + for i, b := range c.sparseBlocks { + p("\n// Block %#x, offset %#x", i, c.sparseOffset[i]) + var v int + stride := mostFrequentStride(b) + n := countSparseEntries(b) + p("\n{value:%#04x,lo:%#02x},", stride, uint8(n)) + for i, nv := range b { + if int(nv)-v != stride { + if v != 0 { + p(",hi:%#02x},", 0x80+i-1) + } + if nv != 0 { + p("\n{value:%#04x,lo:%#02x", nv, 0x80+i) + } + } + v = int(nv) + } + if v != 0 { + p(",hi:%#02x},", 0x80+len(b)-1) + } + } + p("\n}\n\n") + return +} diff --git a/vendor/golang.org/x/text/width/gen.go b/vendor/golang.org/x/text/width/gen.go new file mode 100644 index 000000000..092277e1f --- /dev/null +++ b/vendor/golang.org/x/text/width/gen.go @@ -0,0 +1,115 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +// This program generates the trie for width operations. The generated table +// includes width category information as well as the normalization mappings. +package main + +import ( + "bytes" + "fmt" + "io" + "log" + "math" + "unicode/utf8" + + "golang.org/x/text/internal/gen" + "golang.org/x/text/internal/triegen" +) + +// See gen_common.go for flags. + +func main() { + gen.Init() + genTables() + genTests() + gen.Repackage("gen_trieval.go", "trieval.go", "width") + gen.Repackage("gen_common.go", "common_test.go", "width") +} + +func genTables() { + t := triegen.NewTrie("width") + // fold and inverse mappings. See mapComment for a description of the format + // of each entry. Add dummy value to make an index of 0 mean no mapping. + inverse := [][4]byte{{}} + mapping := map[[4]byte]int{[4]byte{}: 0} + + getWidthData(func(r rune, tag elem, alt rune) { + idx := 0 + if alt != 0 { + var buf [4]byte + buf[0] = byte(utf8.EncodeRune(buf[1:], alt)) + s := string(r) + buf[buf[0]] ^= s[len(s)-1] + var ok bool + if idx, ok = mapping[buf]; !ok { + idx = len(mapping) + if idx > math.MaxUint8 { + log.Fatalf("Index %d does not fit in a byte.", idx) + } + mapping[buf] = idx + inverse = append(inverse, buf) + } + } + t.Insert(r, uint64(tag|elem(idx))) + }) + + w := &bytes.Buffer{} + gen.WriteUnicodeVersion(w) + + sz, err := t.Gen(w) + if err != nil { + log.Fatal(err) + } + + sz += writeMappings(w, inverse) + + fmt.Fprintf(w, "// Total table size %d bytes (%dKiB)\n", sz, sz/1024) + + gen.WriteVersionedGoFile(*outputFile, "width", w.Bytes()) +} + +const inverseDataComment = ` +// inverseData contains 4-byte entries of the following format: +// <0 padding> +// The last byte of the UTF-8-encoded rune is xor-ed with the last byte of the +// UTF-8 encoding of the original rune. Mappings often have the following +// pattern: +// Ａ -> A (U+FF21 -> U+0041) +// Ｂ -> B (U+FF22 -> U+0042) +// ... +// By xor-ing the last byte the same entry can be shared by many mappings. This +// reduces the total number of distinct entries by about two thirds. +// The resulting entry for the aforementioned mappings is +// { 0x01, 0xE0, 0x00, 0x00 } +// Using this entry to map U+FF21 (UTF-8 [EF BC A1]), we get +// E0 ^ A1 = 41. +// Similarly, for U+FF22 (UTF-8 [EF BC A2]), we get +// E0 ^ A2 = 42. +// Note that because of the xor-ing, the byte sequence stored in the entry is +// not valid UTF-8.` + +func writeMappings(w io.Writer, data [][4]byte) int { + fmt.Fprintln(w, inverseDataComment) + fmt.Fprintf(w, "var inverseData = [%d][4]byte{\n", len(data)) + for _, x := range data { + fmt.Fprintf(w, "{ 0x%02x, 0x%02x, 0x%02x, 0x%02x },\n", x[0], x[1], x[2], x[3]) + } + fmt.Fprintln(w, "}") + return len(data) * 4 +} + +func genTests() { + w := &bytes.Buffer{} + fmt.Fprintf(w, "\nvar mapRunes = map[rune]struct{r rune; e elem}{\n") + getWidthData(func(r rune, tag elem, alt rune) { + if alt != 0 { + fmt.Fprintf(w, "\t0x%X: {0x%X, 0x%X},\n", r, alt, tag) + } + }) + fmt.Fprintln(w, "}") + gen.WriteGoFile("runes_test.go", "width", w.Bytes()) +} diff --git a/vendor/golang.org/x/text/width/gen_common.go b/vendor/golang.org/x/text/width/gen_common.go new file mode 100644 index 000000000..601e75268 --- /dev/null +++ b/vendor/golang.org/x/text/width/gen_common.go @@ -0,0 +1,96 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +package main + +// This code is shared between the main code generator and the test code. + +import ( + "flag" + "log" + "strconv" + "strings" + + "golang.org/x/text/internal/gen" + "golang.org/x/text/internal/ucd" +) + +var ( + outputFile = flag.String("out", "tables.go", "output file") +) + +var typeMap = map[string]elem{ + "A": tagAmbiguous, + "N": tagNeutral, + "Na": tagNarrow, + "W": tagWide, + "F": tagFullwidth, + "H": tagHalfwidth, +} + +// getWidthData calls f for every entry for which it is defined. +// +// f may be called multiple times for the same rune. The last call to f is the +// correct value. f is not called for all runes. The default tag type is +// Neutral. +func getWidthData(f func(r rune, tag elem, alt rune)) { + // Set the default values for Unified Ideographs. In line with Annex 11, + // we encode full ranges instead of the defined runes in Unified_Ideograph. + for _, b := range []struct{ lo, hi rune }{ + {0x4E00, 0x9FFF}, // the CJK Unified Ideographs block, + {0x3400, 0x4DBF}, // the CJK Unified Ideographs Externsion A block, + {0xF900, 0xFAFF}, // the CJK Compatibility Ideographs block, + {0x20000, 0x2FFFF}, // the Supplementary Ideographic Plane, + {0x30000, 0x3FFFF}, // the Tertiary Ideographic Plane, + } { + for r := b.lo; r <= b.hi; r++ { + f(r, tagWide, 0) + } + } + + inverse := map[rune]rune{} + maps := map[string]bool{ + "": true, + "": true, + } + + // We cannot reuse package norm's decomposition, as we need an unexpanded + // decomposition. We make use of the opportunity to verify that the + // decomposition type is as expected. + ucd.Parse(gen.OpenUCDFile("UnicodeData.txt"), func(p *ucd.Parser) { + r := p.Rune(0) + s := strings.SplitN(p.String(ucd.DecompMapping), " ", 2) + if !maps[s[0]] { + return + } + x, err := strconv.ParseUint(s[1], 16, 32) + if err != nil { + log.Fatalf("Error parsing rune %q", s[1]) + } + if inverse[r] != 0 || inverse[rune(x)] != 0 { + log.Fatalf("Circular dependency in mapping between %U and %U", r, x) + } + inverse[r] = rune(x) + inverse[rune(x)] = r + }) + + // ; + ucd.Parse(gen.OpenUCDFile("EastAsianWidth.txt"), func(p *ucd.Parser) { + tag, ok := typeMap[p.String(1)] + if !ok { + log.Fatalf("Unknown width type %q", p.String(1)) + } + r := p.Rune(0) + alt, ok := inverse[r] + if tag == tagFullwidth || tag == tagHalfwidth && r != wonSign { + tag |= tagNeedsFold + if !ok { + log.Fatalf("Narrow or wide rune %U has no decomposition", r) + } + } + f(r, tag, alt) + }) +} diff --git a/vendor/golang.org/x/text/width/gen_trieval.go b/vendor/golang.org/x/text/width/gen_trieval.go new file mode 100644 index 000000000..c17334aa6 --- /dev/null +++ b/vendor/golang.org/x/text/width/gen_trieval.go @@ -0,0 +1,34 @@ +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +package main + +// elem is an entry of the width trie. The high byte is used to encode the type +// of the rune. The low byte is used to store the index to a mapping entry in +// the inverseData array. +type elem uint16 + +const ( + tagNeutral elem = iota << typeShift + tagAmbiguous + tagWide + tagNarrow + tagFullwidth + tagHalfwidth +) + +const ( + numTypeBits = 3 + typeShift = 16 - numTypeBits + + // tagNeedsFold is true for all fullwidth and halfwidth runes except for + // the Won sign U+20A9. + tagNeedsFold = 0x1000 + + // The Korean Won sign is halfwidth, but SHOULD NOT be mapped to a wide + // variant. + wonSign rune = 0x20A9 +) diff --git a/vendor/golang.org/x/tools/go/gcexportdata/main.go b/vendor/golang.org/x/tools/go/gcexportdata/main.go new file mode 100644 index 000000000..2713dce64 --- /dev/null +++ b/vendor/golang.org/x/tools/go/gcexportdata/main.go @@ -0,0 +1,99 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build ignore + +// The gcexportdata command is a diagnostic tool that displays the +// contents of gc export data files. +package main + +import ( + "flag" + "fmt" + "go/token" + "go/types" + "log" + "os" + + "golang.org/x/tools/go/gcexportdata" + "golang.org/x/tools/go/types/typeutil" +) + +var packageFlag = flag.String("package", "", "alternative package to print") + +func main() { + log.SetPrefix("gcexportdata: ") + log.SetFlags(0) + flag.Usage = func() { + fmt.Fprintln(os.Stderr, "usage: gcexportdata [-package path] file.a") + } + flag.Parse() + if flag.NArg() != 1 { + flag.Usage() + os.Exit(2) + } + filename := flag.Args()[0] + + f, err := os.Open(filename) + if err != nil { + log.Fatal(err) + } + + r, err := gcexportdata.NewReader(f) + if err != nil { + log.Fatalf("%s: %s", filename, err) + } + + // Decode the package. + const primary = "" + imports := make(map[string]*types.Package) + fset := token.NewFileSet() + pkg, err := gcexportdata.Read(r, fset, imports, primary) + if err != nil { + log.Fatalf("%s: %s", filename, err) + } + + // Optionally select an indirectly mentioned package. + if *packageFlag != "" { + pkg = imports[*packageFlag] + if pkg == nil { + fmt.Fprintf(os.Stderr, "export data file %s does not mention %s; has:\n", + filename, *packageFlag) + for p := range imports { + if p != primary { + fmt.Fprintf(os.Stderr, "\t%s\n", p) + } + } + os.Exit(1) + } + } + + // Print all package-level declarations, including non-exported ones. + fmt.Printf("package %s\n", pkg.Name()) + for _, imp := range pkg.Imports() { + fmt.Printf("import %q\n", imp.Path()) + } + qual := func(p *types.Package) string { + if pkg == p { + return "" + } + return p.Name() + } + scope := pkg.Scope() + for _, name := range scope.Names() { + obj := scope.Lookup(name) + fmt.Printf("%s: %s\n", + fset.Position(obj.Pos()), + types.ObjectString(obj, qual)) + + // For types, print each method. + if _, ok := obj.(*types.TypeName); ok { + for _, method := range typeutil.IntuitiveMethodSet(obj.Type(), nil) { + fmt.Printf("%s: %s\n", + fset.Position(method.Obj().Pos()), + types.SelectionString(method, qual)) + } + } + } +} diff --git a/vendor/golang.org/x/tools/internal/imports/mkindex.go b/vendor/golang.org/x/tools/internal/imports/mkindex.go new file mode 100644 index 000000000..ef8c0d287 --- /dev/null +++ b/vendor/golang.org/x/tools/internal/imports/mkindex.go @@ -0,0 +1,173 @@ +// +build ignore + +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Command mkindex creates the file "pkgindex.go" containing an index of the Go +// standard library. The file is intended to be built as part of the imports +// package, so that the package may be used in environments where a GOROOT is +// not available (such as App Engine). +package imports + +import ( + "bytes" + "fmt" + "go/ast" + "go/build" + "go/format" + "go/parser" + "go/token" + "io/ioutil" + "log" + "os" + "path" + "path/filepath" + "strings" +) + +var ( + pkgIndex = make(map[string][]pkg) + exports = make(map[string]map[string]bool) +) + +func main() { + // Don't use GOPATH. + ctx := build.Default + ctx.GOPATH = "" + + // Populate pkgIndex global from GOROOT. + for _, path := range ctx.SrcDirs() { + f, err := os.Open(path) + if err != nil { + log.Print(err) + continue + } + children, err := f.Readdir(-1) + f.Close() + if err != nil { + log.Print(err) + continue + } + for _, child := range children { + if child.IsDir() { + loadPkg(path, child.Name()) + } + } + } + // Populate exports global. + for _, ps := range pkgIndex { + for _, p := range ps { + e := loadExports(p.dir) + if e != nil { + exports[p.dir] = e + } + } + } + + // Construct source file. + var buf bytes.Buffer + fmt.Fprint(&buf, pkgIndexHead) + fmt.Fprintf(&buf, "var pkgIndexMaster = %#v\n", pkgIndex) + fmt.Fprintf(&buf, "var exportsMaster = %#v\n", exports) + src := buf.Bytes() + + // Replace main.pkg type name with pkg. + src = bytes.Replace(src, []byte("main.pkg"), []byte("pkg"), -1) + // Replace actual GOROOT with "/go". + src = bytes.Replace(src, []byte(ctx.GOROOT), []byte("/go"), -1) + // Add some line wrapping. + src = bytes.Replace(src, []byte("}, "), []byte("},\n"), -1) + src = bytes.Replace(src, []byte("true, "), []byte("true,\n"), -1) + + var err error + src, err = format.Source(src) + if err != nil { + log.Fatal(err) + } + + // Write out source file. + err = ioutil.WriteFile("pkgindex.go", src, 0644) + if err != nil { + log.Fatal(err) + } +} + +const pkgIndexHead = `package imports + +func init() { + pkgIndexOnce.Do(func() { + pkgIndex.m = pkgIndexMaster + }) + loadExports = func(dir string) map[string]bool { + return exportsMaster[dir] + } +} +` + +type pkg struct { + importpath string // full pkg import path, e.g. "net/http" + dir string // absolute file path to pkg directory e.g. "/usr/lib/go/src/fmt" +} + +var fset = token.NewFileSet() + +func loadPkg(root, importpath string) { + shortName := path.Base(importpath) + if shortName == "testdata" { + return + } + + dir := filepath.Join(root, importpath) + pkgIndex[shortName] = append(pkgIndex[shortName], pkg{ + importpath: importpath, + dir: dir, + }) + + pkgDir, err := os.Open(dir) + if err != nil { + return + } + children, err := pkgDir.Readdir(-1) + pkgDir.Close() + if err != nil { + return + } + for _, child := range children { + name := child.Name() + if name == "" { + continue + } + if c := name[0]; c == '.' || ('0' <= c && c <= '9') { + continue + } + if child.IsDir() { + loadPkg(root, filepath.Join(importpath, name)) + } + } +} + +func loadExports(dir string) map[string]bool { + exports := make(map[string]bool) + buildPkg, err := build.ImportDir(dir, 0) + if err != nil { + if strings.Contains(err.Error(), "no buildable Go source files in") { + return nil + } + log.Printf("could not import %q: %v", dir, err) + return nil + } + for _, file := range buildPkg.GoFiles { + f, err := parser.ParseFile(fset, filepath.Join(dir, file), nil, 0) + if err != nil { + log.Printf("could not parse %q: %v", file, err) + continue + } + for name := range f.Scope.Objects { + if ast.IsExported(name) { + exports[name] = true + } + } + } + return exports +} diff --git a/vendor/golang.org/x/tools/internal/imports/mkstdlib.go b/vendor/golang.org/x/tools/internal/imports/mkstdlib.go new file mode 100644 index 000000000..f67b5c1ed --- /dev/null +++ b/vendor/golang.org/x/tools/internal/imports/mkstdlib.go @@ -0,0 +1,132 @@ +// +build ignore + +// mkstdlib generates the zstdlib.go file, containing the Go standard +// library API symbols. It's baked into the binary to avoid scanning +// GOPATH in the common case. +package imports + +import ( + "bufio" + "bytes" + "fmt" + "go/format" + "io" + "io/ioutil" + "log" + "os" + "os/exec" + "path/filepath" + "regexp" + "runtime" + "sort" + "strings" +) + +func mustOpen(name string) io.Reader { + f, err := os.Open(name) + if err != nil { + log.Fatal(err) + } + return f +} + +func api(base string) string { + return filepath.Join(runtime.GOROOT(), "api", base) +} + +var sym = regexp.MustCompile(`^pkg (\S+).*?, (?:var|func|type|const) ([A-Z]\w*)`) + +var unsafeSyms = map[string]bool{"Alignof": true, "ArbitraryType": true, "Offsetof": true, "Pointer": true, "Sizeof": true} + +func main() { + var buf bytes.Buffer + outf := func(format string, args ...interface{}) { + fmt.Fprintf(&buf, format, args...) + } + outf("// Code generated by mkstdlib.go. DO NOT EDIT.\n\n") + outf("package imports\n") + outf("var stdlib = map[string]map[string]bool{\n") + f := io.MultiReader( + mustOpen(api("go1.txt")), + mustOpen(api("go1.1.txt")), + mustOpen(api("go1.2.txt")), + mustOpen(api("go1.3.txt")), + mustOpen(api("go1.4.txt")), + mustOpen(api("go1.5.txt")), + mustOpen(api("go1.6.txt")), + mustOpen(api("go1.7.txt")), + mustOpen(api("go1.8.txt")), + mustOpen(api("go1.9.txt")), + mustOpen(api("go1.10.txt")), + mustOpen(api("go1.11.txt")), + mustOpen(api("go1.12.txt")), + + // The API of the syscall/js package needs to be computed explicitly, + // because it's not included in the GOROOT/api/go1.*.txt files at this time. + syscallJSAPI(), + ) + sc := bufio.NewScanner(f) + + pkgs := map[string]map[string]bool{ + "unsafe": unsafeSyms, + } + paths := []string{"unsafe"} + + for sc.Scan() { + l := sc.Text() + has := func(v string) bool { return strings.Contains(l, v) } + if has("struct, ") || has("interface, ") || has(", method (") { + continue + } + if m := sym.FindStringSubmatch(l); m != nil { + path, sym := m[1], m[2] + + if _, ok := pkgs[path]; !ok { + pkgs[path] = map[string]bool{} + paths = append(paths, path) + } + pkgs[path][sym] = true + } + } + if err := sc.Err(); err != nil { + log.Fatal(err) + } + sort.Strings(paths) + for _, path := range paths { + outf("\t%q: map[string]bool{\n", path) + pkg := pkgs[path] + var syms []string + for sym := range pkg { + syms = append(syms, sym) + } + sort.Strings(syms) + for _, sym := range syms { + outf("\t\t%q: true,\n", sym) + } + outf("},\n") + } + outf("}\n") + fmtbuf, err := format.Source(buf.Bytes()) + if err != nil { + log.Fatal(err) + } + err = ioutil.WriteFile("zstdlib.go", fmtbuf, 0666) + if err != nil { + log.Fatal(err) + } +} + +// syscallJSAPI returns the API of the syscall/js package. +// It's computed from the contents of $(go env GOROOT)/src/syscall/js. +func syscallJSAPI() io.Reader { + var exeSuffix string + if runtime.GOOS == "windows" { + exeSuffix = ".exe" + } + cmd := exec.Command("go"+exeSuffix, "run", "cmd/api", "-contexts", "js-wasm", "syscall/js") + out, err := cmd.Output() + if err != nil { + log.Fatalln(err) + } + return bytes.NewReader(out) +} diff --git a/vendor/modules.txt b/vendor/modules.txt index a611599d3..02c080dca 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -22,6 +22,8 @@ github.com/BurntSushi/toml github.com/Microsoft/go-winio # github.com/NYTimes/gziphandler v1.1.1 => github.com/NYTimes/gziphandler v1.1.1 github.com/NYTimes/gziphandler +# github.com/OneOfOne/xxhash v1.2.7 => github.com/OneOfOne/xxhash v1.2.2 +github.com/OneOfOne/xxhash # github.com/PuerkitoBio/goquery v1.5.0 => github.com/PuerkitoBio/goquery v1.5.0 github.com/PuerkitoBio/goquery # github.com/PuerkitoBio/purell v1.1.1 => github.com/PuerkitoBio/purell v1.1.1 @@ -247,6 +249,15 @@ github.com/go-redis/redis/internal/util github.com/go-sql-driver/mysql # github.com/gobuffalo/flect v0.1.5 => github.com/gobuffalo/flect v0.1.5 github.com/gobuffalo/flect +# github.com/gobwas/glob v0.2.3 => github.com/gobwas/glob v0.2.3 +github.com/gobwas/glob +github.com/gobwas/glob/compiler +github.com/gobwas/glob/match +github.com/gobwas/glob/syntax +github.com/gobwas/glob/syntax/ast +github.com/gobwas/glob/syntax/lexer +github.com/gobwas/glob/util/runes +github.com/gobwas/glob/util/strings # github.com/gocraft/dbr v0.0.0-20180507214907-a0fd650918f6 => github.com/gocraft/dbr v0.0.0-20180507214907-a0fd650918f6 github.com/gocraft/dbr github.com/gocraft/dbr/dialect @@ -526,6 +537,41 @@ github.com/onsi/gomega/matchers/support/goraph/edge github.com/onsi/gomega/matchers/support/goraph/node github.com/onsi/gomega/matchers/support/goraph/util github.com/onsi/gomega/types +# github.com/open-policy-agent/opa v0.18.0 => github.com/open-policy-agent/opa v0.18.0 +github.com/open-policy-agent/opa/ast +github.com/open-policy-agent/opa/bundle +github.com/open-policy-agent/opa/internal/compiler/wasm +github.com/open-policy-agent/opa/internal/compiler/wasm/opa +github.com/open-policy-agent/opa/internal/file/archive +github.com/open-policy-agent/opa/internal/file/url +github.com/open-policy-agent/opa/internal/ir +github.com/open-policy-agent/opa/internal/leb128 +github.com/open-policy-agent/opa/internal/merge +github.com/open-policy-agent/opa/internal/planner +github.com/open-policy-agent/opa/internal/version +github.com/open-policy-agent/opa/internal/wasm/constant +github.com/open-policy-agent/opa/internal/wasm/encoding +github.com/open-policy-agent/opa/internal/wasm/instruction +github.com/open-policy-agent/opa/internal/wasm/module +github.com/open-policy-agent/opa/internal/wasm/opcode +github.com/open-policy-agent/opa/internal/wasm/types +github.com/open-policy-agent/opa/loader +github.com/open-policy-agent/opa/metrics +github.com/open-policy-agent/opa/rego +github.com/open-policy-agent/opa/storage +github.com/open-policy-agent/opa/storage/inmem +github.com/open-policy-agent/opa/topdown +github.com/open-policy-agent/opa/topdown/builtins +github.com/open-policy-agent/opa/topdown/copypropagation +github.com/open-policy-agent/opa/topdown/internal/jwx/buffer +github.com/open-policy-agent/opa/topdown/internal/jwx/jwa +github.com/open-policy-agent/opa/topdown/internal/jwx/jwk +github.com/open-policy-agent/opa/topdown/internal/jwx/jws +github.com/open-policy-agent/opa/topdown/internal/jwx/jws/sign +github.com/open-policy-agent/opa/topdown/internal/jwx/jws/verify +github.com/open-policy-agent/opa/types +github.com/open-policy-agent/opa/util +github.com/open-policy-agent/opa/version # github.com/opencontainers/go-digest v1.0.0-rc1 => github.com/opencontainers/go-digest v1.0.0-rc1 github.com/opencontainers/go-digest # github.com/opencontainers/image-spec v1.0.1 => github.com/opencontainers/image-spec v1.0.1 @@ -595,6 +641,8 @@ github.com/prometheus/common/model # github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084 => github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084 github.com/prometheus/procfs github.com/prometheus/procfs/internal/fs +# github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a => github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a +github.com/rcrowley/go-metrics # github.com/russross/blackfriday v1.5.2 => github.com/russross/blackfriday v1.5.2 github.com/russross/blackfriday # github.com/satori/go.uuid v1.2.0 => github.com/satori/go.uuid v1.2.0 @@ -632,6 +680,8 @@ github.com/stretchr/testify/assert github.com/stretchr/testify/mock # github.com/xanzy/ssh-agent v0.2.1 => github.com/xanzy/ssh-agent v0.2.1 github.com/xanzy/ssh-agent +# github.com/yashtewari/glob-intersection v0.0.0-20180916065949-5c77d914dd0b => github.com/yashtewari/glob-intersection v0.0.0-20180916065949-5c77d914dd0b +github.com/yashtewari/glob-intersection # go.uber.org/atomic v1.4.0 => go.uber.org/atomic v1.4.0 go.uber.org/atomic # go.uber.org/multierr v1.1.0 => go.uber.org/multierr v1.1.0 @@ -1389,6 +1439,9 @@ k8s.io/utils/path k8s.io/utils/pointer k8s.io/utils/trace # openpitrix.io/openpitrix v0.4.1-0.20190920134345-4d2be6e4965c => openpitrix.io/openpitrix v0.4.1-0.20190920134345-4d2be6e4965c +openpitrix.io/openpitrix/pkg/client +openpitrix.io/openpitrix/pkg/client/access +openpitrix.io/openpitrix/pkg/client/account openpitrix.io/openpitrix/pkg/config openpitrix.io/openpitrix/pkg/constants openpitrix.io/openpitrix/pkg/db diff --git a/vendor/openpitrix.io/openpitrix/pkg/client/access/client.go b/vendor/openpitrix.io/openpitrix/pkg/client/access/client.go new file mode 100644 index 000000000..aa69fed00 --- /dev/null +++ b/vendor/openpitrix.io/openpitrix/pkg/client/access/client.go @@ -0,0 +1,73 @@ +// Copyright 2018 The OpenPitrix Authors. All rights reserved. +// Use of this source code is governed by a Apache license +// that can be found in the LICENSE file. + +package access + +import ( + "context" + + accountclient "openpitrix.io/openpitrix/pkg/client/account" + "openpitrix.io/openpitrix/pkg/constants" + "openpitrix.io/openpitrix/pkg/logger" + "openpitrix.io/openpitrix/pkg/manager" + "openpitrix.io/openpitrix/pkg/pb" +) + +type Client struct { + pb.AccessManagerClient +} + +func NewClient() (*Client, error) { + conn, err := manager.NewClient(constants.AccountServiceHost, constants.AccountServicePort) + if err != nil { + return nil, err + } + return &Client{ + AccessManagerClient: pb.NewAccessManagerClient(conn), + }, nil +} + +func (c *Client) CheckActionBundleUser(ctx context.Context, actionBundleIds []string, userId string) bool { + users, err := c.GetActionBundleUsers(ctx, actionBundleIds) + if err != nil { + return false + } + for _, user := range users { + if user.GetUserId().GetValue() == userId { + return true + } + } + return false +} + +func (c *Client) GetActionBundleRoles(ctx context.Context, actionBundleIds []string) ([]*pb.Role, error) { + response, err := c.DescribeRoles(ctx, &pb.DescribeRolesRequest{ + ActionBundleId: actionBundleIds, + Status: []string{constants.StatusActive}, + }) + if err != nil { + logger.Error(ctx, "Describe roles failed: %+v", err) + return nil, err + } + + return response.RoleSet, nil +} + +func (c *Client) GetActionBundleUsers(ctx context.Context, actionBundleIds []string) ([]*pb.User, error) { + roles, err := c.GetActionBundleRoles(ctx, actionBundleIds) + if err != nil { + return nil, err + } + var roleIds []string + for _, role := range roles { + roleIds = append(roleIds, role.RoleId) + } + + accountClient, err := accountclient.NewClient() + if err != nil { + logger.Error(ctx, "Get account manager client failed: %+v", err) + return nil, err + } + return accountClient.GetRoleUsers(ctx, roleIds) +} diff --git a/vendor/openpitrix.io/openpitrix/pkg/client/account/client.go b/vendor/openpitrix.io/openpitrix/pkg/client/account/client.go new file mode 100644 index 000000000..b1bc16d33 --- /dev/null +++ b/vendor/openpitrix.io/openpitrix/pkg/client/account/client.go @@ -0,0 +1,142 @@ +// Copyright 2018 The OpenPitrix Authors. All rights reserved. +// Use of this source code is governed by a Apache license +// that can be found in the LICENSE file. + +package account + +import ( + "context" + "fmt" + "math" + "strings" + + "openpitrix.io/openpitrix/pkg/constants" + "openpitrix.io/openpitrix/pkg/logger" + "openpitrix.io/openpitrix/pkg/manager" + "openpitrix.io/openpitrix/pkg/pb" + "openpitrix.io/openpitrix/pkg/util/pbutil" + "openpitrix.io/openpitrix/pkg/util/stringutil" +) + +type Client struct { + pb.AccountManagerClient +} + +func NewClient() (*Client, error) { + conn, err := manager.NewClient(constants.AccountServiceHost, constants.AccountServicePort) + if err != nil { + return nil, err + } + return &Client{ + AccountManagerClient: pb.NewAccountManagerClient(conn), + }, nil +} + +func (c *Client) GetUsers(ctx context.Context, userIds []string) ([]*pb.User, error) { + var internalUsers []*pb.User + var noInternalUserIds []string + for _, userId := range userIds { + if stringutil.StringIn(userId, constants.InternalUsers) { + internalUsers = append(internalUsers, &pb.User{ + UserId: pbutil.ToProtoString(userId), + }) + } else { + noInternalUserIds = append(noInternalUserIds, userId) + } + } + + if len(noInternalUserIds) == 0 { + return internalUsers, nil + } + + response, err := c.DescribeUsers(ctx, &pb.DescribeUsersRequest{ + UserId: noInternalUserIds, + }) + if err != nil { + logger.Error(ctx, "Describe users %s failed: %+v", noInternalUserIds, err) + return nil, err + } + if len(response.UserSet) != len(noInternalUserIds) { + logger.Error(ctx, "Describe users %s with return count [%d]", userIds, len(response.UserSet)+len(internalUsers)) + return nil, fmt.Errorf("describe users %s with return count [%d]", userIds, len(response.UserSet)+len(internalUsers)) + } + response.UserSet = append(response.UserSet, internalUsers...) + return response.UserSet, nil +} + +func (c *Client) GetUser(ctx context.Context, userId string) (*pb.User, error) { + users, err := c.GetUsers(ctx, []string{userId}) + if err != nil { + return nil, err + } + if len(users) == 0 { + return nil, fmt.Errorf("not found user [%s]", userId) + } + return users[0], nil +} + +func (c *Client) GetUserGroupPath(ctx context.Context, userId string) (string, error) { + var userGroupPath string + + response, err := c.DescribeUsersDetail(ctx, &pb.DescribeUsersRequest{ + UserId: []string{userId}, + }) + if err != nil || len(response.UserDetailSet) == 0 { + logger.Error(ctx, "Describe user [%s] failed: %+v", userId, err) + return "", err + } + + groups := response.UserDetailSet[0].GroupSet + + //If one user under different groups, get the highest group path. + minLevel := math.MaxInt32 + for _, group := range groups { + level := len(strings.Split(group.GroupPath.GetValue(), ".")) + if level < minLevel { + minLevel = level + userGroupPath = group.GetGroupPath().GetValue() + } + } + + return userGroupPath, nil + +} + +func (c *Client) GetRoleUsers(ctx context.Context, roleIds []string) ([]*pb.User, error) { + response, err := c.DescribeUsers(ctx, &pb.DescribeUsersRequest{ + RoleId: roleIds, + Status: []string{constants.StatusActive}, + }) + if err != nil { + logger.Error(ctx, "Describe users failed: %+v", err) + return nil, err + } + + return response.UserSet, nil +} + +func (c *Client) GetIsvFromUser(ctx context.Context, userId string) (*pb.User, error) { + groupPath, err := c.GetUserGroupPath(ctx, userId) + if err != nil { + return nil, err + } + + rootGroupId := strings.Split(groupPath, ".")[0] + + describeUsersResponse, err := c.DescribeUsers(ctx, &pb.DescribeUsersRequest{ + RootGroupId: []string{rootGroupId}, + Status: []string{constants.StatusActive}, + RoleId: []string{constants.RoleIsv}, + }) + if err != nil { + logger.Error(ctx, "Failed to describe users: %+v", err) + return nil, err + } + + if len(describeUsersResponse.UserSet) == 0 { + logger.Error(ctx, "Isv not exist with root group id [%s]", rootGroupId) + return nil, fmt.Errorf("isv not exist") + } + + return describeUsersResponse.UserSet[0], nil +} diff --git a/vendor/openpitrix.io/openpitrix/pkg/client/client.go b/vendor/openpitrix.io/openpitrix/pkg/client/client.go new file mode 100644 index 000000000..8c00d87c1 --- /dev/null +++ b/vendor/openpitrix.io/openpitrix/pkg/client/client.go @@ -0,0 +1,35 @@ +// Copyright 2018 The OpenPitrix Authors. All rights reserved. +// Use of this source code is governed by a Apache license +// that can be found in the LICENSE file. + +package client + +import ( + "context" + + accessclient "openpitrix.io/openpitrix/pkg/client/access" + "openpitrix.io/openpitrix/pkg/pb" + "openpitrix.io/openpitrix/pkg/sender" + "openpitrix.io/openpitrix/pkg/util/ctxutil" +) + +func SetSystemUserToContext(ctx context.Context) context.Context { + return ctxutil.ContextWithSender(ctx, sender.GetSystemSender()) +} + +func SetUserToContext(ctx context.Context, userId, apiMethod string) (context.Context, error) { + accessClient, err := accessclient.NewClient() + if err != nil { + return nil, err + } + response, err := accessClient.CanDo(ctx, &pb.CanDoRequest{ + UserId: userId, + ApiMethod: apiMethod, + }) + if err != nil { + return nil, err + } + + userSender := sender.New(response.UserId, sender.OwnerPath(response.OwnerPath), sender.OwnerPath(response.AccessPath)) + return ctxutil.ContextWithSender(ctx, userSender), nil +} diff --git a/vendor/openpitrix.io/openpitrix/pkg/version/gen_helper.go b/vendor/openpitrix.io/openpitrix/pkg/version/gen_helper.go new file mode 100644 index 000000000..01af5342f --- /dev/null +++ b/vendor/openpitrix.io/openpitrix/pkg/version/gen_helper.go @@ -0,0 +1,74 @@ +// Copyright 2018 The OpenPitrix Authors. All rights reserved. +// Use of this source code is governed by a Apache license +// that can be found in the LICENSE file. + +// +build ignore + +package main + +import ( + "fmt" + "io/ioutil" + "log" + "os/exec" + "strings" + "time" +) + +func main() { + version, gitSha1Version := getAppVersion() + buildTime := time.Now().Format("2006-01-02 15:04:05") + + data := make_update_version_go_file(version, gitSha1Version, buildTime) + + err := ioutil.WriteFile("./z_update_version.go", []byte(data), 0666) + if err != nil { + log.Fatalf("ioutil.WriteFile: err = %v", err) + } + + fmt.Printf("%s (%s)\n", version, gitSha1Version) + fmt.Println(buildTime) +} + +func make_update_version_go_file(version, gitSha1Version, buildTime string) string { + return fmt.Sprintf(`// Copyright 2018 The OpenPitrix Authors. All rights reserved. +// Use of this source code is governed by a Apache license +// that can be found in the LICENSE file. + +// Auto generated by 'go run gen_helper.go', DO NOT EDIT. + +package version + +func init() { + ShortVersion = "%s" + GitSha1Version = "%s" + BuildDate = "%s" +} +`, + version, + gitSha1Version, + buildTime, + ) +} + +func getAppVersion() (version, gitSha1Version string) { + // VERSION=`git describe --tags --always --dirty="-dev"` + versionData, err := exec.Command( + `git`, `describe`, `--tags`, `--always`, `--dirty=-dev`, + ).CombinedOutput() + if err != nil { + log.Fatal(err) + } + + // GIT_SHA1=`git show --quiet --pretty=format:%H` + gitSha1VersionData, err := exec.Command( + `git`, `show`, `--quiet`, `--pretty=format:%H`, + ).CombinedOutput() + if err != nil { + log.Fatal(err) + } + + version = strings.TrimSpace(string(versionData)) + gitSha1Version = strings.TrimSpace(string(gitSha1VersionData)) + return +}