/* Copyright 2019 The Kubernetes Authors. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. */ package markers import ( "go/ast" "go/token" "strings" "sync" "sigs.k8s.io/controller-tools/pkg/loader" ) // Collector collects and parses marker comments defined in the registry // from package source code. If no registry is provided, an empty one will // be initialized on the first call to MarkersInPackage. type Collector struct { *Registry byPackage map[string]map[ast.Node]MarkerValues mu sync.Mutex } // MarkerValues are all the values for some set of markers. type MarkerValues map[string][]interface{} // Get fetches the first value that for the given marker, returning // nil if no values are available. func (v MarkerValues) Get(name string) interface{} { vals := v[name] if len(vals) == 0 { return nil } return vals[0] } func (c *Collector) init() { if c.Registry == nil { c.Registry = &Registry{} } if c.byPackage == nil { c.byPackage = make(map[string]map[ast.Node]MarkerValues) } } // MarkersInPackage computes the marker values by node for the given package. Results // are cached by package ID, so this is safe to call repeatedly from different functions. // Each file in the package is treated as a distinct node. // // We consider a marker to be associated with a given AST node if either of the following are true: // // - it's in the Godoc for that AST node // // - it's in the closest non-godoc comment group above that node, // *and* that node is a type or field node, *and* [it's either // registered as type-level *or* it's not registered as being // package-level] // // - it's not in the Godoc of a node, doesn't meet the above criteria, and // isn't in a struct definition (in which case it's package-level) func (c *Collector) MarkersInPackage(pkg *loader.Package) (map[ast.Node]MarkerValues, error) { c.mu.Lock() c.init() if markers, exist := c.byPackage[pkg.ID]; exist { c.mu.Unlock() return markers, nil } // unlock early, it's ok if we do a bit extra work rather than locking while we're working c.mu.Unlock() pkg.NeedSyntax() nodeMarkersRaw := c.associatePkgMarkers(pkg) markers, err := c.parseMarkersInPackage(nodeMarkersRaw) if err != nil { return nil, err } c.mu.Lock() defer c.mu.Unlock() c.byPackage[pkg.ID] = markers return markers, nil } // parseMarkersInPackage parses the given raw marker comments into output values using the registry. func (c *Collector) parseMarkersInPackage(nodeMarkersRaw map[ast.Node][]markerComment) (map[ast.Node]MarkerValues, error) { var errors []error nodeMarkerValues := make(map[ast.Node]MarkerValues) for node, markersRaw := range nodeMarkersRaw { var target TargetType switch node.(type) { case *ast.File: target = DescribesPackage case *ast.Field: target = DescribesField default: target = DescribesType } markerVals := make(map[string][]interface{}) for _, markerRaw := range markersRaw { markerText := markerRaw.Text() def := c.Registry.Lookup(markerText, target) if def == nil { continue } val, err := def.Parse(markerText) if err != nil { errors = append(errors, loader.ErrFromNode(err, markerRaw)) continue } markerVals[def.Name] = append(markerVals[def.Name], val) } nodeMarkerValues[node] = markerVals } return nodeMarkerValues, loader.MaybeErrList(errors) } // associatePkgMarkers associates markers with AST nodes in the given package. func (c *Collector) associatePkgMarkers(pkg *loader.Package) map[ast.Node][]markerComment { nodeMarkers := make(map[ast.Node][]markerComment) for _, file := range pkg.Syntax { fileNodeMarkers := c.associateFileMarkers(file) for node, markers := range fileNodeMarkers { nodeMarkers[node] = append(nodeMarkers[node], markers...) } } return nodeMarkers } // associateFileMarkers associates markers with AST nodes in the given file. func (c *Collector) associateFileMarkers(file *ast.File) map[ast.Node][]markerComment { // grab all the raw marker comments by node visitor := markerSubVisitor{ collectPackageLevel: true, markerVisitor: &markerVisitor{ nodeMarkers: make(map[ast.Node][]markerComment), allComments: file.Comments, }, } ast.Walk(visitor, file) // grab the last package-level comments at the end of the file (if any) lastFileMarkers := visitor.markersBetween(false, visitor.commentInd, len(visitor.allComments)) visitor.pkgMarkers = append(visitor.pkgMarkers, lastFileMarkers...) // figure out if any type-level markers are actually package-level markers for node, markers := range visitor.nodeMarkers { _, isType := node.(*ast.TypeSpec) if !isType { continue } endOfMarkers := 0 for _, marker := range markers { if marker.fromGodoc { // markers from godoc are never package level markers[endOfMarkers] = marker endOfMarkers++ continue } markerText := marker.Text() typeDef := c.Registry.Lookup(markerText, DescribesType) if typeDef != nil { // prefer assuming type-level markers markers[endOfMarkers] = marker endOfMarkers++ continue } def := c.Registry.Lookup(markerText, DescribesPackage) if def == nil { // assume type-level unless proven otherwise markers[endOfMarkers] = marker endOfMarkers++ continue } // it's package-level, since a package-level definition exists visitor.pkgMarkers = append(visitor.pkgMarkers, marker) } visitor.nodeMarkers[node] = markers[:endOfMarkers] // re-set after trimming the package markers } visitor.nodeMarkers[file] = visitor.pkgMarkers return visitor.nodeMarkers } // markerComment is an AST comment that contains a marker. // It may or may not be from a Godoc comment, which affects // marker re-associated (from type-level to package-level) type markerComment struct { *ast.Comment fromGodoc bool } // Text returns the text of the marker, stripped of the comment // marker and leading spaces, as should be passed to Registry.Lookup // and Registry.Parse. func (c markerComment) Text() string { return strings.TrimSpace(c.Comment.Text[2:]) } // markerVisistor visits AST nodes, recording markers associated with each node. type markerVisitor struct { allComments []*ast.CommentGroup commentInd int declComments []markerComment lastLineCommentGroup *ast.CommentGroup pkgMarkers []markerComment nodeMarkers map[ast.Node][]markerComment } // isMarkerComment checks that the given comment is a single-line (`//`) // comment and it's first non-space content is `+`. func isMarkerComment(comment string) bool { if comment[0:2] != "//" { return false } stripped := strings.TrimSpace(comment[2:]) if len(stripped) < 1 || stripped[0] != '+' { return false } return true } // markersBetween grabs the markers between the given indicies in the list of all comments. func (v *markerVisitor) markersBetween(fromGodoc bool, start, end int) []markerComment { if start < 0 || end < 0 { return nil } var res []markerComment for i := start; i < end; i++ { commentGroup := v.allComments[i] for _, comment := range commentGroup.List { if !isMarkerComment(comment.Text) { continue } res = append(res, markerComment{Comment: comment, fromGodoc: fromGodoc}) } } return res } type markerSubVisitor struct { *markerVisitor node ast.Node collectPackageLevel bool } // Visit collects markers for each node in the AST, optionally // collecting unassociated markers as package-level. func (v markerSubVisitor) Visit(node ast.Node) ast.Visitor { if node == nil { // end of the node, so we might need to advance comments beyond the end // of the block if we don't want to collect package-level markers in // this block. if !v.collectPackageLevel { if v.commentInd < len(v.allComments) { lastCommentInd := v.commentInd nextGroup := v.allComments[lastCommentInd] for nextGroup.Pos() < v.node.End() { lastCommentInd++ if lastCommentInd >= len(v.allComments) { // after the increment so our decrement below still makes sense break } nextGroup = v.allComments[lastCommentInd] } v.commentInd = lastCommentInd } } return nil } // skip comments on the same line as the previous node // making sure to double-check for the case where we've gone past the end of the comments // but still have to finish up typespec-gendecl association (see below). if v.lastLineCommentGroup != nil && v.commentInd < len(v.allComments) && v.lastLineCommentGroup.Pos() == v.allComments[v.commentInd].Pos() { v.commentInd++ } // stop visiting if there are no more comments in the file // NB(directxman12): we can't just stop immediately, because we // still need to check if there are typespecs associated with gendecls. var markerCommentBlock []markerComment var docCommentBlock []markerComment lastCommentInd := v.commentInd if v.commentInd < len(v.allComments) { // figure out the first comment after the node in question... nextGroup := v.allComments[lastCommentInd] for nextGroup.Pos() < node.Pos() { lastCommentInd++ if lastCommentInd >= len(v.allComments) { // after the increment so our decrement below still makes sense break } nextGroup = v.allComments[lastCommentInd] } lastCommentInd-- // ...then decrement to get the last comment before the node in question // figure out the godoc comment so we can deal with it separately var docGroup *ast.CommentGroup docGroup, v.lastLineCommentGroup = associatedCommentsFor(node) // find the last comment group that's not godoc markerCommentInd := lastCommentInd if docGroup != nil && v.allComments[markerCommentInd].Pos() == docGroup.Pos() { markerCommentInd-- } // check if we have freestanding package markers, // and find the markers in our "closest non-godoc" comment block, // plus our godoc comment block if markerCommentInd >= v.commentInd { if v.collectPackageLevel { // assume anything between the comment ind and the marker ind (not including it) // are package-level v.pkgMarkers = append(v.pkgMarkers, v.markersBetween(false, v.commentInd, markerCommentInd)...) } markerCommentBlock = v.markersBetween(false, markerCommentInd, markerCommentInd+1) docCommentBlock = v.markersBetween(true, markerCommentInd+1, lastCommentInd+1) } else { docCommentBlock = v.markersBetween(true, markerCommentInd+1, lastCommentInd+1) } } resVisitor := markerSubVisitor{ collectPackageLevel: false, // don't collect package level by default markerVisitor: v.markerVisitor, node: node, } // associate those markers with a node switch typedNode := node.(type) { case *ast.GenDecl: // save the comments associated with the gen-decl if it's a single-line type decl if typedNode.Lparen != token.NoPos || typedNode.Tok != token.TYPE { // not a single-line type spec, treat them as free comments v.pkgMarkers = append(v.pkgMarkers, markerCommentBlock...) break } // save these, we'll need them when we encounter the actual type spec v.declComments = append(v.declComments, markerCommentBlock...) v.declComments = append(v.declComments, docCommentBlock...) case *ast.TypeSpec: // add in comments attributed to the gen-decl, if any, // as well as comments associated with the actual type v.nodeMarkers[node] = append(v.nodeMarkers[node], v.declComments...) v.nodeMarkers[node] = append(v.nodeMarkers[node], markerCommentBlock...) v.nodeMarkers[node] = append(v.nodeMarkers[node], docCommentBlock...) v.declComments = nil v.collectPackageLevel = false // don't collect package-level inside type structs case *ast.Field: v.nodeMarkers[node] = append(v.nodeMarkers[node], markerCommentBlock...) v.nodeMarkers[node] = append(v.nodeMarkers[node], docCommentBlock...) case *ast.File: v.pkgMarkers = append(v.pkgMarkers, markerCommentBlock...) v.pkgMarkers = append(v.pkgMarkers, docCommentBlock...) // collect markers in root file scope resVisitor.collectPackageLevel = true default: // assume markers before anything else are package-level markers, // *but* don't include any markers in godoc if v.collectPackageLevel { v.pkgMarkers = append(v.pkgMarkers, markerCommentBlock...) } } // increment the comment ind so that we start at the right place for the next node v.commentInd = lastCommentInd + 1 return resVisitor } // associatedCommentsFor returns the doc comment group (if relevant and present) and end-of-line comment // (again if relevant and present) for the given AST node. func associatedCommentsFor(node ast.Node) (docGroup *ast.CommentGroup, lastLineCommentGroup *ast.CommentGroup) { switch typedNode := node.(type) { case *ast.Field: docGroup = typedNode.Doc lastLineCommentGroup = typedNode.Comment case *ast.File: docGroup = typedNode.Doc case *ast.FuncDecl: docGroup = typedNode.Doc case *ast.GenDecl: docGroup = typedNode.Doc case *ast.ImportSpec: docGroup = typedNode.Doc lastLineCommentGroup = typedNode.Comment case *ast.TypeSpec: docGroup = typedNode.Doc lastLineCommentGroup = typedNode.Comment case *ast.ValueSpec: docGroup = typedNode.Doc lastLineCommentGroup = typedNode.Comment default: lastLineCommentGroup = nil } return docGroup, lastLineCommentGroup }