Update dependencies (#5518)

vendor/github.com/open-policy-agent/opa/topdown/http.go

@@ -11,7 +11,7 @@ import (
 	"crypto/x509"
 	"encoding/json"
 	"fmt"
-	"io/ioutil"
+	"io"
 	"math"
 	"net"
 	"net/http"
@@ -69,8 +69,24 @@ var allowedKeyNames = [...]string{
 	"caching_mode",
 }
 
+// ref: https://www.rfc-editor.org/rfc/rfc7231#section-6.1
+var cacheableHTTPStatusCodes = [...]int{
+	http.StatusOK,
+	http.StatusNonAuthoritativeInfo,
+	http.StatusNoContent,
+	http.StatusPartialContent,
+	http.StatusMultipleChoices,
+	http.StatusMovedPermanently,
+	http.StatusNotFound,
+	http.StatusMethodNotAllowed,
+	http.StatusGone,
+	http.StatusRequestURITooLong,
+	http.StatusNotImplemented,
+}
+
 var (
 	allowedKeys                 = ast.NewSet()
+	cacheableCodes              = ast.NewSet()
 	requiredKeys                = ast.NewSet(ast.StringTerm("method"), ast.StringTerm("url"))
 	httpSendLatencyMetricKey    = "rego_builtin_" + strings.ReplaceAll(ast.HTTPSend.Name, ".", "_")
 	httpSendInterQueryCacheHits = httpSendLatencyMetricKey + "_interquery_cache_hits"
@@ -90,8 +106,8 @@ const (
 	HTTPSendNetworkErr string = "eval_http_send_network_error"
 )
 
-func builtinHTTPSend(bctx BuiltinContext, args []*ast.Term, iter func(*ast.Term) error) error {
-	req, err := validateHTTPRequestOperand(args[0], 1)
+func builtinHTTPSend(bctx BuiltinContext, operands []*ast.Term, iter func(*ast.Term) error) error {
+	req, err := validateHTTPRequestOperand(operands[0], 1)
 	if err != nil {
 		return handleBuiltinErr(ast.HTTPSend.Name, bctx.Location, err)
 	}
@@ -162,6 +178,7 @@ func getHTTPResponse(bctx BuiltinContext, req ast.Object) (*ast.Term, error) {
 
 func init() {
 	createAllowedKeys()
+	createCacheableHTTPStatusCodes()
 	initDefaults()
 	RegisterBuiltinFunc(ast.HTTPSend.Name, builtinHTTPSend)
 }
@@ -237,7 +254,7 @@ func canonicalizeHeaders(headers map[string]interface{}) map[string]interface{}
 // useSocket examines the url for "unix://" and returns a *http.Transport with
 // a DialContext that opens a socket (specified in the http call).
 // The url is expected to contain socket=/path/to/socket (url encoded)
-// Ex. "unix:localhost/end/point?socket=%2Ftmp%2Fhttp.sock"
+// Ex. "unix://localhost/end/point?socket=%2Ftmp%2Fhttp.sock"
 func useSocket(rawURL string, tlsConfig *tls.Config) (bool, string, *http.Transport) {
 	u, err := url.Parse(rawURL)
 	if err != nil {
@@ -248,21 +265,29 @@ func useSocket(rawURL string, tlsConfig *tls.Config) (bool, string, *http.Transp
 		return false, rawURL, nil
 	}
 
-	// Get the path to the socket
 	v, err := url.ParseQuery(u.RawQuery)
 	if err != nil {
 		return false, rawURL, nil
 	}
 
+	// Rewrite URL targeting the UNIX domain socket.
+	u.Scheme = "http"
+
+	// Extract the path to the socket.
+	// Only retrieve the first value. Subsequent values are ignored and removed
+	// to prevent HTTP parameter pollution.
+	socket := v.Get("socket")
+	v.Del("socket")
+	u.RawQuery = v.Encode()
+
 	tr := http.DefaultTransport.(*http.Transport).Clone()
 	tr.DialContext = func(ctx context.Context, network, addr string) (net.Conn, error) {
-		return http.DefaultTransport.(*http.Transport).DialContext(ctx, u.Scheme, v.Get("socket"))
+		return http.DefaultTransport.(*http.Transport).DialContext(ctx, "unix", socket)
 	}
 	tr.TLSClientConfig = tlsConfig
 	tr.DisableKeepAlives = true
 
-	rawURL = strings.Replace(rawURL, "unix:", "http:", 1)
-	return true, rawURL, tr
+	return true, u.String(), tr
 }
 
 func verifyHost(bctx BuiltinContext, host string) error {
@@ -444,6 +469,9 @@ func createHTTPRequest(bctx BuiltinContext, obj ast.Object) (*http.Request, *htt
 	isTLS := false
 	client := &http.Client{
 		Timeout: timeout,
+		CheckRedirect: func(*http.Request, []*http.Request) error {
+			return http.ErrUseLastResponse
+		},
 	}
 
 	if tlsInsecureSkipVerify {
@@ -505,7 +533,7 @@ func createHTTPRequest(bctx BuiltinContext, obj ast.Object) (*http.Request, *htt
 
 	if len(tlsCaCert) != 0 {
 		tlsCaCert = bytes.Replace(tlsCaCert, []byte("\\n"), []byte("\n"), -1)
-		pool, err := addCACertsFromBytes(tlsConfig.RootCAs, []byte(tlsCaCert))
+		pool, err := addCACertsFromBytes(tlsConfig.RootCAs, tlsCaCert)
 		if err != nil {
 			return nil, nil, err
 		}
@@ -552,9 +580,9 @@ func createHTTPRequest(bctx BuiltinContext, obj ast.Object) (*http.Request, *htt
 	}
 
 	// check if redirects are enabled
-	if !enableRedirect {
-		client.CheckRedirect = func(*http.Request, []*http.Request) error {
-			return http.ErrUseLastResponse
+	if enableRedirect {
+		client.CheckRedirect = func(req *http.Request, _ []*http.Request) error {
+			return verifyURLHost(bctx, req.URL.String())
 		}
 	}
 
@@ -692,21 +720,21 @@ func (c *interQueryCache) checkHTTPSendInterQueryCache() (ast.Value, error) {
 		return nil, nil
 	}
 
-	headers, err := parseResponseHeaders(cachedRespData.Headers)
-	if err != nil {
-		return nil, err
-	}
-
-	// check the freshness of the cached response
-	if isCachedResponseFresh(c.bctx, headers, c.forceCacheParams) {
+	if getCurrentTime(c.bctx).Before(cachedRespData.ExpiresAt) {
 		return cachedRespData.formatToAST(c.forceJSONDecode, c.forceYAMLDecode)
 	}
 
+	var err error
 	c.httpReq, c.httpClient, err = createHTTPRequest(c.bctx, c.key)
 	if err != nil {
 		return nil, handleHTTPSendErr(c.bctx, err)
 	}
 
+	headers, err := parseResponseHeaders(cachedRespData.Headers)
+	if err != nil {
+		return nil, err
+	}
+
 	// check with the server if the stale response is still up-to-date.
 	// If server returns a new response (ie. status_code=200), update the cache with the new response
 	// If server returns an unmodified response (ie. status_code=304), update the headers for the existing response
@@ -727,6 +755,12 @@ func (c *interQueryCache) checkHTTPSendInterQueryCache() (ast.Value, error) {
 			}
 		}
 
+		expiresAt, err := expiryFromHeaders(result.Header)
+		if err != nil {
+			return nil, err
+		}
+		cachedRespData.ExpiresAt = expiresAt
+
 		cachingMode, err := getCachingMode(c.key)
 		if err != nil {
 			return nil, err
@@ -753,7 +787,7 @@ func (c *interQueryCache) checkHTTPSendInterQueryCache() (ast.Value, error) {
 		return nil, err
 	}
 
-	if err := insertIntoHTTPSendInterQueryCache(c.bctx, c.key, result, respBody, c.forceCacheParams != nil); err != nil {
+	if err := insertIntoHTTPSendInterQueryCache(c.bctx, c.key, result, respBody, c.forceCacheParams); err != nil {
 		return nil, err
 	}
 
@@ -761,8 +795,8 @@ func (c *interQueryCache) checkHTTPSendInterQueryCache() (ast.Value, error) {
 }
 
 // insertIntoHTTPSendInterQueryCache inserts given key and value in the inter-query cache
-func insertIntoHTTPSendInterQueryCache(bctx BuiltinContext, key ast.Value, resp *http.Response, respBody []byte, force bool) error {
-	if resp == nil || (!force && !canStore(resp.Header)) {
+func insertIntoHTTPSendInterQueryCache(bctx BuiltinContext, key ast.Value, resp *http.Response, respBody []byte, cacheParams *forceCacheParams) error {
+	if resp == nil || (!forceCaching(cacheParams) && !canStore(resp.Header)) || !cacheableCodes.Contains(ast.IntNumberTerm(resp.StatusCode)) {
 		return nil
 	}
 
@@ -781,9 +815,9 @@ func insertIntoHTTPSendInterQueryCache(bctx BuiltinContext, key ast.Value, resp
 	var pcv cache.InterQueryCacheValue
 
 	if cachingMode == defaultCachingMode {
-		pcv, err = newInterQueryCacheValue(resp, respBody)
+		pcv, err = newInterQueryCacheValue(bctx, resp, respBody, cacheParams)
 	} else {
-		pcv, err = newInterQueryCacheData(resp, respBody)
+		pcv, err = newInterQueryCacheData(bctx, resp, respBody, cacheParams)
 	}
 
 	if err != nil {
@@ -800,6 +834,12 @@ func createAllowedKeys() {
 	}
 }
 
+func createCacheableHTTPStatusCodes() {
+	for _, element := range cacheableHTTPStatusCodes {
+		cacheableCodes.Add(ast.IntNumberTerm(element))
+	}
+}
+
 func parseTimeout(timeoutVal ast.Value) (time.Duration, error) {
 	var timeout time.Duration
 	switch t := timeoutVal.(type) {
@@ -855,15 +895,15 @@ func getCachingMode(req ast.Object) (cachingMode, error) {
 			return "", fmt.Errorf("invalid value specified for %v field: %v", key.String(), string(s))
 		}
 	}
-	return cachingMode(defaultCachingMode), nil
+	return defaultCachingMode, nil
 }
 
 type interQueryCacheValue struct {
 	Data []byte
 }
 
-func newInterQueryCacheValue(resp *http.Response, respBody []byte) (*interQueryCacheValue, error) {
-	data, err := newInterQueryCacheData(resp, respBody)
+func newInterQueryCacheValue(bctx BuiltinContext, resp *http.Response, respBody []byte, cacheParams *forceCacheParams) (*interQueryCacheValue, error) {
+	data, err := newInterQueryCacheData(bctx, resp, respBody, cacheParams)
 	if err != nil {
 		return nil, err
 	}
@@ -893,15 +933,48 @@ type interQueryCacheData struct {
 	Status     string
 	StatusCode int
 	Headers    http.Header
+	ExpiresAt  time.Time
 }
 
-func newInterQueryCacheData(resp *http.Response, respBody []byte) (*interQueryCacheData, error) {
-	_, err := parseResponseHeaders(resp.Header)
+func forceCaching(cacheParams *forceCacheParams) bool {
+	return cacheParams != nil && cacheParams.forceCacheDurationSeconds > 0
+}
+
+func expiryFromHeaders(headers http.Header) (time.Time, error) {
+	var expiresAt time.Time
+	maxAge, err := parseMaxAgeCacheDirective(parseCacheControlHeader(headers))
 	if err != nil {
-		return nil, err
+		return time.Time{}, err
+	}
+	if maxAge != -1 {
+		createdAt, err := getResponseHeaderDate(headers)
+		if err != nil {
+			return time.Time{}, err
+		}
+		expiresAt = createdAt.Add(time.Second * time.Duration(maxAge))
+	} else {
+		expiresAt = getResponseHeaderExpires(headers)
+	}
+	return expiresAt, nil
+}
+
+func newInterQueryCacheData(bctx BuiltinContext, resp *http.Response, respBody []byte, cacheParams *forceCacheParams) (*interQueryCacheData, error) {
+	var expiresAt time.Time
+
+	if forceCaching(cacheParams) {
+		createdAt := getCurrentTime(bctx)
+		expiresAt = createdAt.Add(time.Second * time.Duration(cacheParams.forceCacheDurationSeconds))
+	} else {
+		var err error
+		expiresAt, err = expiryFromHeaders(resp.Header)
+		if err != nil {
+			return nil, err
+		}
 	}
 
-	cv := interQueryCacheData{RespBody: respBody,
+	cv := interQueryCacheData{
+		ExpiresAt:  expiresAt,
+		RespBody:   respBody,
 		Status:     resp.Status,
 		StatusCode: resp.StatusCode,
 		Headers:    resp.Header}
@@ -1008,58 +1081,6 @@ func canStore(headers http.Header) bool {
 	return true
 }
 
-func isCachedResponseFresh(bctx BuiltinContext, headers *responseHeaders, cacheParams *forceCacheParams) bool {
-	if headers.date.IsZero() {
-		return false
-	}
-
-	currentTime := getCurrentTime(bctx)
-	if currentTime.IsZero() {
-		return false
-	}
-
-	currentAge := currentTime.Sub(headers.date)
-
-	// The time.Sub operation uses wall clock readings and
-	// not monotonic clock readings as the parsed version of the response time
-	// does not contain monotonic clock readings. This can result in negative durations.
-	// Another scenario where a negative duration can occur, is when a server sets the Date
-	// response header. As per https://tools.ietf.org/html/rfc7231#section-7.1.1.2,
-	// an origin server MUST NOT send a Date header field if it does not
-	// have a clock capable of providing a reasonable approximation of the
-	// current instance in Coordinated Universal Time.
-	// Hence, consider the cached response as stale if a negative duration is encountered.
-	if currentAge < 0 {
-		return false
-	}
-
-	if cacheParams != nil {
-		// override the cache directives set by the server
-		maxAgeDur := time.Second * time.Duration(cacheParams.forceCacheDurationSeconds)
-		if maxAgeDur > currentAge {
-			return true
-		}
-	} else {
-		// Check "max-age" cache directive.
-		// The "max-age" response directive indicates that the response is to be
-		// considered stale after its age is greater than the specified number
-		// of seconds.
-		if headers.maxAge != -1 {
-			maxAgeDur := time.Second * time.Duration(headers.maxAge)
-			if maxAgeDur > currentAge {
-				return true
-			}
-		} else {
-			// Check "Expires" header.
-			// Note: "max-age" if set, takes precedence over "Expires"
-			if headers.expires.Sub(headers.date) > currentAge {
-				return true
-			}
-		}
-	}
-	return false
-}
-
 func getCurrentTime(bctx BuiltinContext) time.Time {
 	var current time.Time
 
@@ -1155,7 +1176,7 @@ func parseMaxAgeCacheDirective(cc map[string]string) (deltaSeconds, error) {
 
 func formatHTTPResponseToAST(resp *http.Response, forceJSONDecode, forceYAMLDecode bool) (ast.Value, []byte, error) {
 
-	resultRawBody, err := ioutil.ReadAll(resp.Body)
+	resultRawBody, err := io.ReadAll(resp.Body)
 	if err != nil {
 		return nil, nil, err
 	}
@@ -1172,7 +1193,7 @@ func prepareASTResult(headers http.Header, forceJSONDecode, forceYAMLDecode bool
 	var resultBody interface{}
 
 	// If the response body cannot be JSON/YAML decoded,
-	// an error will not be returned. Instead the "body" field
+	// an error will not be returned. Instead, the "body" field
 	// in the result will be null.
 	switch {
 	case forceJSONDecode || isContentType(headers, "application/json"):
@@ -1275,7 +1296,7 @@ func (c *interQueryCache) InsertIntoCache(value *http.Response) (ast.Value, erro
 	}
 
 	// fallback to the http send cache if error encountered while inserting response in inter-query cache
-	err = insertIntoHTTPSendInterQueryCache(c.bctx, c.key, value, respBody, c.forceCacheParams != nil)
+	err = insertIntoHTTPSendInterQueryCache(c.bctx, c.key, value, respBody, c.forceCacheParams)
 	if err != nil {
 		insertIntoHTTPSendCache(c.bctx, c.key, result)
 	}
@@ -1323,7 +1344,10 @@ func (c *intraQueryCache) InsertIntoCache(value *http.Response) (ast.Value, erro
 		return nil, handleHTTPSendErr(c.bctx, err)
 	}
 
-	insertIntoHTTPSendCache(c.bctx, c.key, result)
+	if cacheableCodes.Contains(ast.IntNumberTerm(value.StatusCode)) {
+		insertIntoHTTPSendCache(c.bctx, c.key, result)
+	}
+
 	return result, nil
 }