feat: kubesphere 4.0 (#6115)

* feat: kubesphere 4.0

Signed-off-by: ci-bot <ci-bot@kubesphere.io>

* feat: kubesphere 4.0

Signed-off-by: ci-bot <ci-bot@kubesphere.io>

---------

Signed-off-by: ci-bot <ci-bot@kubesphere.io>
Co-authored-by: ks-ci-bot <ks-ci-bot@example.com>
Co-authored-by: joyceliu <joyceliu@yunify.com>

vendor/github.com/open-policy-agent/opa/internal/planner/planner.go

@@ -25,6 +25,8 @@ type QuerySet struct {
 }
 
 type planiter func() error
+type planLocalIter func(ir.Local) error
+type stmtFactory func(ir.Local) ir.Stmt
 
 // Planner implements a query planner for Rego queries.
 type Planner struct {
@@ -147,32 +149,31 @@ func (p *Planner) buildFunctrie() error {
 		}
 
 		for _, rule := range module.Rules {
-			r := rule.Ref()
-			switch r[len(r)-1].Value.(type) {
-			case ast.String: // pass
-			default: // cut off
-				r = r[:len(r)-1]
-			}
+			r := rule.Ref().StringPrefix()
 			val := p.rules.LookupOrInsert(r)
+
+			val.rules = val.DescendantRules()
 			val.rules = append(val.rules, rule)
+			val.children = nil
 		}
 	}
 	return nil
 }
 
 func (p *Planner) planRules(rules []*ast.Rule) (string, error) {
+	// We know the rules with closer to the root (shorter static path) are ordered first.
 	pathRef := rules[0].Ref()
 
 	// figure out what our rules' collective name/path is:
 	// if we're planning both p.q.r and p.q[s], we'll name
 	// the function p.q (for the mapping table)
-	// TODO(sr): this has to change when allowing `p[v].q.r[w]` ref rules
-	// including the mapping lookup structure and lookup functions
 	pieces := len(pathRef)
 	for i := range rules {
 		r := rules[i].Ref()
-		if _, ok := r[len(r)-1].Value.(ast.String); !ok {
-			pieces = len(r) - 1
+		for j, t := range r {
+			if _, ok := t.Value.(ast.String); !ok && j > 0 && j < pieces {
+				pieces = j
+			}
 		}
 	}
 	// control if p.a = 1 is to return 1 directly; or insert 1 under key "a" into an object
@@ -236,7 +237,11 @@ func (p *Planner) planRules(rules []*ast.Rule) (string, error) {
 			fn.Blocks = append(fn.Blocks, p.blockWithStmt(&ir.MakeObjectStmt{Target: fn.Return}))
 		}
 	case ast.MultiValue:
-		fn.Blocks = append(fn.Blocks, p.blockWithStmt(&ir.MakeSetStmt{Target: fn.Return}))
+		if buildObject {
+			fn.Blocks = append(fn.Blocks, p.blockWithStmt(&ir.MakeObjectStmt{Target: fn.Return}))
+		} else {
+			fn.Blocks = append(fn.Blocks, p.blockWithStmt(&ir.MakeSetStmt{Target: fn.Return}))
+		}
 	}
 
 	// For complete document rules, allocate one local variable for output
@@ -252,6 +257,12 @@ func (p *Planner) planRules(rules []*ast.Rule) (string, error) {
 	var defaultRule *ast.Rule
 	var ruleLoc *location.Location
 
+	// We sort rules by ref length, to ensure that when merged, we can detect conflicts when one
+	// rule attempts to override values (deep and shallow) defined by another rule.
+	sort.Slice(rules, func(i, j int) bool {
+		return len(rules[i].Ref()) > len(rules[j].Ref())
+	})
+
 	// Generate function blocks for rules.
 	for i := range rules {
 
@@ -320,18 +331,19 @@ func (p *Planner) planRules(rules []*ast.Rule) (string, error) {
 					switch rule.Head.RuleKind() {
 					case ast.SingleValue:
 						if buildObject {
-							ref := rule.Head.Ref()
-							last := ref[len(ref)-1]
-							return p.planTerm(last, func() error {
-								key := p.ltarget
-								return p.planTerm(rule.Head.Value, func() error {
-									value := p.ltarget
-									p.appendStmt(&ir.ObjectInsertOnceStmt{
-										Object: fn.Return,
-										Key:    key,
-										Value:  value,
+							ref := rule.Ref()
+							return p.planTerm(rule.Head.Value, func() error {
+								value := p.ltarget
+								return p.planNestedObjects(fn.Return, ref[pieces:len(ref)-1], func(obj ir.Local) error {
+									return p.planTerm(ref[len(ref)-1], func() error {
+										key := p.ltarget
+										p.appendStmt(&ir.ObjectInsertOnceStmt{
+											Object: obj,
+											Key:    key,
+											Value:  value,
+										})
+										return nil
 									})
-									return nil
 								})
 							})
 						}
@@ -343,6 +355,28 @@ func (p *Planner) planRules(rules []*ast.Rule) (string, error) {
 							return nil
 						})
 					case ast.MultiValue:
+						if buildObject {
+							ref := rule.Ref()
+							// we drop the trailing set key from the ref
+							return p.planNestedObjects(fn.Return, ref[pieces:len(ref)-1], func(obj ir.Local) error {
+								// Last term on rule ref is the key an which the set is assigned in the deepest nested object
+								return p.planTerm(ref[len(ref)-1], func() error {
+									key := p.ltarget
+									return p.planTerm(rule.Head.Key, func() error {
+										value := p.ltarget
+										factory := func(v ir.Local) ir.Stmt { return &ir.MakeSetStmt{Target: v} }
+										return p.planDotOr(obj, key, factory, func(set ir.Local) error {
+											p.appendStmt(&ir.SetAddStmt{
+												Set:   set,
+												Value: value,
+											})
+											p.appendStmt(&ir.ObjectInsertStmt{Key: key, Value: op(set), Object: obj})
+											return nil
+										})
+									})
+								})
+							})
+						}
 						return p.planTerm(rule.Head.Key, func() error {
 							p.appendStmt(&ir.SetAddStmt{
 								Set:   fn.Return,
@@ -422,6 +456,63 @@ func (p *Planner) planRules(rules []*ast.Rule) (string, error) {
 	return fn.Name, nil
 }
 
+func (p *Planner) planDotOr(obj ir.Local, key ir.Operand, or stmtFactory, iter planLocalIter) error {
+	// We're constructing the following plan:
+	//
+	// | block a
+	// | | block b
+	// | | | dot &{Source:Local<obj> Key:{Value:Local<key>} Target:Local<val>}
+	// | | | break 1
+	// | | or &{Target:Local<val>}
+	// | iter &{Target:Local<val>} # may update Local<val>.
+	// | *ir.ObjectInsertStmt &{Key:{Value:Local<key>} Value:{Value:Local<val>} Object:Local<obj>}
+
+	prev := p.curr
+	dotBlock := &ir.Block{}
+	p.curr = dotBlock
+
+	val := p.newLocal()
+	p.appendStmt(&ir.DotStmt{
+		Source: op(obj),
+		Key:    key,
+		Target: val,
+	})
+	p.appendStmt(&ir.BreakStmt{Index: 1})
+
+	outerBlock := &ir.Block{
+		Stmts: []ir.Stmt{
+			&ir.BlockStmt{Blocks: []*ir.Block{dotBlock}}, // FIXME: Set Location
+			or(val),
+		},
+	}
+
+	p.curr = prev
+	p.appendStmt(&ir.BlockStmt{Blocks: []*ir.Block{outerBlock}})
+	if err := iter(val); err != nil {
+		return err
+	}
+	p.appendStmt(&ir.ObjectInsertStmt{Key: key, Value: op(val), Object: obj})
+	return nil
+}
+
+func (p *Planner) planNestedObjects(obj ir.Local, ref ast.Ref, iter planLocalIter) error {
+	if len(ref) == 0 {
+		//return fmt.Errorf("nested object construction didn't create object")
+		return iter(obj)
+	}
+
+	t := ref[0]
+
+	return p.planTerm(t, func() error {
+		key := p.ltarget
+
+		factory := func(v ir.Local) ir.Stmt { return &ir.MakeObjectStmt{Target: v} }
+		return p.planDotOr(obj, key, factory, func(childObj ir.Local) error {
+			return p.planNestedObjects(childObj, ref[1:], iter)
+		})
+	})
+}
+
 func (p *Planner) planFuncParams(params []ir.Local, args ast.Args, idx int, iter planiter) error {
 	if idx >= len(args) {
 		return iter()
@@ -754,13 +845,29 @@ func (p *Planner) dataRefsShadowRuletrie(refs []ast.Ref) bool {
 }
 
 func (p *Planner) planExprTerm(e *ast.Expr, iter planiter) error {
-	return p.planTerm(e.Terms.(*ast.Term), func() error {
-		p.appendStmt(&ir.NotEqualStmt{
-			A: p.ltarget,
-			B: op(ir.Bool(false)),
+	// NOTE(sr): There are only three cases to deal with when we see a naked term
+	// in a rule body:
+	//  1. it's `false` -- so we can stop, emit a break stmt
+	//  2. it's a var or a ref, like `input` or `data.foo.bar`, where we need to
+	//     check what it ends up being (at run time) to determine if it's not false
+	//  3. it's any other term -- `true`, a string, a number, whatever. We can skip
+	//     that, since it's true-ish enough for evaluating the rule body.
+	switch t := e.Terms.(*ast.Term).Value.(type) {
+	case ast.Boolean:
+		if !bool(t) { // We know this cannot hold, break unconditionally
+			p.appendStmt(&ir.BreakStmt{})
+			return iter()
+		}
+	case ast.Ref, ast.Var: // We don't know these at plan-time
+		return p.planTerm(e.Terms.(*ast.Term), func() error {
+			p.appendStmt(&ir.NotEqualStmt{
+				A: p.ltarget,
+				B: op(ir.Bool(false)),
+			})
+			return iter()
 		})
-		return iter()
-	})
+	}
+	return iter()
 }
 
 func (p *Planner) planExprEvery(e *ast.Expr, iter planiter) error {
@@ -1116,6 +1223,24 @@ func (p *Planner) planUnifyVar(a ast.Var, b *ast.Term, iter planiter) error {
 }
 
 func (p *Planner) planUnifyLocal(a ir.Operand, b *ast.Term, iter planiter) error {
+	// special cases: when a is StringIndex or Bool, and b is a string, or a bool, we can shortcut
+	switch va := a.Value.(type) {
+	case ir.StringIndex:
+		if vb, ok := b.Value.(ast.String); ok {
+			if va != ir.StringIndex(p.getStringConst(string(vb))) {
+				p.appendStmt(&ir.BreakStmt{})
+			}
+			return iter() // Don't plan EqualStmt{A: "foo", B: "foo"}
+		}
+	case ir.Bool:
+		if vb, ok := b.Value.(ast.Boolean); ok {
+			if va != ir.Bool(vb) {
+				p.appendStmt(&ir.BreakStmt{})
+			}
+			return iter() // Don't plan EqualStmt{A: true, B: true}
+		}
+	}
+
 	switch vb := b.Value.(type) {
 	case ast.Null, ast.Boolean, ast.Number, ast.String, ast.Ref, ast.Set, *ast.SetComprehension, *ast.ArrayComprehension, *ast.ObjectComprehension:
 		return p.planTerm(b, func() error {
@@ -1565,16 +1690,14 @@ func (p *Planner) planComprehension(body ast.Body, closureIter planiter, target
 	// below.
 	p.vars.Push(map[ast.Var]ir.Local{})
 	prev := p.curr
-	p.curr = &ir.Block{}
+	block := &ir.Block{}
+	p.curr = block
 	ploc := p.loc
 
-	if err := p.planQuery(body, 0, func() error {
-		return closureIter()
-	}); err != nil {
+	if err := p.planQuery(body, 0, closureIter); err != nil {
 		return err
 	}
 
-	block := p.curr
 	p.curr = prev
 	p.loc = ploc
 	p.vars.Pop()
@@ -1737,11 +1860,12 @@ func (p *Planner) planRefData(virtual *ruletrie, base *baseptr, ref ast.Ref, ind
 					}},
 				}}
 				p.curr = outerBlock
-				return p.planRefRec(ref, index+1, func() error { // rest of the ref
-					p.curr = prev
-					p.appendStmt(&ir.BlockStmt{Blocks: []*ir.Block{outerBlock}})
-					return iter()
-				})
+				if err := p.planRefRec(ref, index+1, iter); err != nil { // rest of the ref
+					return err
+				}
+				p.curr = prev
+				p.appendStmt(&ir.BlockStmt{Blocks: []*ir.Block{outerBlock}})
+				return nil
 			})
 		}
 	}
@@ -2236,6 +2360,7 @@ func (p *Planner) optimizeLookup(t *ruletrie, ref ast.Ref) ([][]*ast.Rule, []ir.
 	var index int
 
 	// ref[0] is data, ignore
+outer:
 	for i := 1; i < len(ref); i++ {
 		index = i
 		r := ref[i]
@@ -2259,7 +2384,7 @@ func (p *Planner) optimizeLookup(t *ruletrie, ref ast.Ref) ([][]*ast.Rule, []ir.
 				}
 			}
 		case ast.String:
-			// take matching children
+			// take all children that either match or have a var key
 			for _, node := range nodes {
 				if node := node.Get(r); node != nil {
 					nextNodes = append(nextNodes, node)
@@ -2272,15 +2397,44 @@ func (p *Planner) optimizeLookup(t *ruletrie, ref ast.Ref) ([][]*ast.Rule, []ir.
 
 		nodes = nextNodes
 
-		// if all nodes have 0 children, abort ref check and optimize
-		all := true
+		// if all nodes have rules() > 0, abort ref check and optimize
+		// NOTE(sr): for a.b[c] = ... and a.b.d = ..., we stop at a.b, as its rules()
+		// will collect the children rules
+		// We keep the "all nodes have 0 children" check since it's cheaper and might
+		// let us break, too.
+		all := 0
 		for _, node := range nodes {
-			all = all && len(node.Children()) == 0
+			all += node.ChildrenCount()
 		}
-		if all {
+		if all == 0 {
 			p.debugf("ref %s: all nodes have 0 children, break", ref[0:index+1])
 			break
 		}
+
+		// NOTE(sr): we only need this check for the penultimate part:
+		// When planning the ref data.pkg.a[input.x][input.y],
+		// We want to capture this situation:
+		//   a.b[c] := "x" if c := "c"
+		//   a.b.d := "y"
+		//
+		// Not this:
+		//   a.b[c] := "x" if c := "c"
+		//   a.d := "y"
+		// since the length doesn't add up. Even if input.x was "d", the second
+		// rule (a.d) wouldn't contribute anything to the result, since we cannot
+		// "dot it".
+		if index == len(ref)-2 {
+			for _, node := range nodes {
+				anyNonGround := false
+				for _, r := range node.Rules() {
+					anyNonGround = anyNonGround || !r.Ref().IsGround()
+				}
+				if anyNonGround {
+					p.debugf("ref %s: at least one node has 1+ non-ground ref rules, break", ref[0:index+1])
+					break outer
+				}
+			}
+		}
 	}
 
 	var res [][]*ast.Rule
@@ -2295,7 +2449,7 @@ func (p *Planner) optimizeLookup(t *ruletrie, ref ast.Ref) ([][]*ast.Rule, []ir.
 	for _, node := range nodes {
 		// we're done with ref, check if there's only ruleset leaves; collect rules
 		if index == len(ref)-1 {
-			if len(node.Rules()) == 0 && len(node.Children()) > 0 {
+			if len(node.Rules()) == 0 && node.ChildrenCount() > 0 {
 				p.debugf("no optimization of %s: unbalanced ruletrie", ref)
 				return dont()
 			}

vendor/github.com/open-policy-agent/opa/internal/planner/rules.go

@@ -98,6 +98,7 @@ func (t *ruletrie) Rules() []*ast.Rule {
 		//
 		// and we're retrieving a.b, we want Rules() to include the rule body
 		// of a.b.c.
+		// FIXME: We need to go deeper than just immediate children (?)
 		for _, rs := range t.children {
 			if r := rs[len(rs)-1].rules; r != nil {
 				rules = append(rules, r...)
@@ -157,13 +158,50 @@ func (t *ruletrie) Lookup(key ast.Ref) *ruletrie {
 	return node
 }
 
+func (t *ruletrie) LookupShallowest(key ast.Ref) *ruletrie {
+	node := t
+	for _, elem := range key {
+		node = node.Get(elem.Value)
+		if node == nil {
+			return nil
+		}
+		if len(node.rules) > 0 {
+			return node
+		}
+	}
+	return node
+}
+
+// TODO: Collapse rules with overlapping extent to same node(?)
 func (t *ruletrie) LookupOrInsert(key ast.Ref) *ruletrie {
-	if val := t.Lookup(key); val != nil {
+	if val := t.LookupShallowest(key); val != nil {
+
 		return val
 	}
 	return t.Insert(key)
 }
 
+func (t *ruletrie) DescendantRules() []*ast.Rule {
+	if len(t.children) == 0 {
+		return t.rules
+	}
+
+	rules := make([]*ast.Rule, len(t.rules), len(t.rules)+len(t.children)) // could be too little
+	copy(rules, t.rules)
+
+	for _, cs := range t.children {
+		for _, c := range cs {
+			rules = append(rules, c.DescendantRules()...)
+		}
+	}
+
+	return rules
+}
+
+func (t *ruletrie) ChildrenCount() int {
+	return len(t.children)
+}
+
 func (t *ruletrie) Children() []ast.Value {
 	if t == nil {
 		return nil