update package s2ioperator

Signed-off-by: soulseen <sunzhu@yunify.com>
This commit is contained in:
soulseen
2019-08-30 10:37:09 +08:00
parent f22e8ea90e
commit bbd151e7aa
300 changed files with 18100 additions and 52462 deletions

View File

@@ -16,14 +16,29 @@ import (
"strings"
)
// Driver
// The Driver Protocol
//
// The driver, given the inputs to a call to Load, returns metadata about the packages specified.
// This allows for different build systems to support go/packages by telling go/packages how the
// packages' source is organized.
// The driver is a binary, either specified by the GOPACKAGESDRIVER environment variable or in
// the path as gopackagesdriver. It's given the inputs to load in its argv. See the package
// documentation in doc.go for the full description of the patterns that need to be supported.
// A driver receives as a JSON-serialized driverRequest struct in standard input and will
// produce a JSON-serialized driverResponse (see definition in packages.go) in its standard output.
// driverRequest is used to provide the portion of Load's Config that is needed by a driver.
type driverRequest struct {
Command string `json:"command"`
Mode LoadMode `json:"mode"`
Env []string `json:"env"`
BuildFlags []string `json:"build_flags"`
Tests bool `json:"tests"`
Overlay map[string][]byte `json:"overlay"`
Mode LoadMode `json:"mode"`
// Env specifies the environment the underlying build system should be run in.
Env []string `json:"env"`
// BuildFlags are flags that should be passed to the underlying build system.
BuildFlags []string `json:"build_flags"`
// Tests specifies whether the patterns should also return test packages.
Tests bool `json:"tests"`
// Overlay maps file paths (relative to the driver's working directory) to the byte contents
// of overlay files.
Overlay map[string][]byte `json:"overlay"`
}
// findExternalDriver returns the file path of a tool that supplies

View File

@@ -158,7 +158,7 @@ extractQueries:
}
}
modifiedPkgs, needPkgs, err := processGolistOverlay(cfg, response.dr)
modifiedPkgs, needPkgs, err := processGolistOverlay(cfg, response)
if err != nil {
return nil, err
}
@@ -209,11 +209,13 @@ func addNeededOverlayPackages(cfg *Config, driver driver, response *responseDedu
for _, pkg := range dr.Packages {
response.addPackage(pkg)
}
_, needPkgs, err := processGolistOverlay(cfg, response.dr)
_, needPkgs, err := processGolistOverlay(cfg, response)
if err != nil {
return err
}
addNeededOverlayPackages(cfg, driver, response, needPkgs)
if err := addNeededOverlayPackages(cfg, driver, response, needPkgs); err != nil {
return err
}
return nil
}
@@ -228,8 +230,16 @@ func runContainsQueries(cfg *Config, driver driver, response *responseDeduper, q
return fmt.Errorf("could not determine absolute path of file= query path %q: %v", query, err)
}
dirResponse, err := driver(cfg, pattern)
if err != nil {
return err
if err != nil || (len(dirResponse.Packages) == 1 && len(dirResponse.Packages[0].Errors) == 1) {
// There was an error loading the package. Try to load the file as an ad-hoc package.
// Usually the error will appear in a returned package, but may not if we're in modules mode
// and the ad-hoc is located outside a module.
var queryErr error
dirResponse, queryErr = driver(cfg, query)
if queryErr != nil {
// Return the original error if the attempt to fall back failed.
return err
}
}
isRoot := make(map[string]bool, len(dirResponse.Roots))
for _, root := range dirResponse.Roots {
@@ -693,7 +703,7 @@ func golistDriver(cfg *Config, words ...string) (*driverResponse, error) {
if p.Error != nil {
pkg.Errors = append(pkg.Errors, Error{
Pos: p.Error.Pos,
Msg: p.Error.Err,
Msg: strings.TrimSpace(p.Error.Err), // Trim to work around golang.org/issue/32363.
})
}

View File

@@ -3,6 +3,7 @@ package packages
import (
"bytes"
"encoding/json"
"fmt"
"go/parser"
"go/token"
"path"
@@ -16,16 +17,13 @@ import (
// files that don't exist on disk to an overlay. The results can be
// sometimes incorrect.
// TODO(matloob): Handle unsupported cases, including the following:
// - test files
// - adding test and non-test files to test variants of packages
// - determining the correct package to add given a new import path
// - creating packages that don't exist
func processGolistOverlay(cfg *Config, response *driverResponse) (modifiedPkgs, needPkgs []string, err error) {
func processGolistOverlay(cfg *Config, response *responseDeduper) (modifiedPkgs, needPkgs []string, err error) {
havePkgs := make(map[string]string) // importPath -> non-test package ID
needPkgsSet := make(map[string]bool)
modifiedPkgsSet := make(map[string]bool)
for _, pkg := range response.Packages {
for _, pkg := range response.dr.Packages {
// This is an approximation of import path to id. This can be
// wrong for tests, vendored packages, and a number of other cases.
havePkgs[pkg.PkgPath] = pkg.ID
@@ -34,21 +32,42 @@ func processGolistOverlay(cfg *Config, response *driverResponse) (modifiedPkgs,
var rootDirs map[string]string
var onceGetRootDirs sync.Once
// If no new imports are added, it is safe to avoid loading any needPkgs.
// Otherwise, it's hard to tell which package is actually being loaded
// (due to vendoring) and whether any modified package will show up
// in the transitive set of dependencies (because new imports are added,
// potentially modifying the transitive set of dependencies).
var overlayAddsImports bool
for opath, contents := range cfg.Overlay {
base := filepath.Base(opath)
if strings.HasSuffix(opath, "_test.go") {
// Overlays don't support adding new test files yet.
// TODO(matloob): support adding new test files.
continue
}
dir := filepath.Dir(opath)
var pkg *Package
var testVariantOf *Package // if opath is a test file, this is the package it is testing
var fileExists bool
for _, p := range response.Packages {
isTest := strings.HasSuffix(opath, "_test.go")
pkgName, ok := extractPackageName(opath, contents)
if !ok {
// Don't bother adding a file that doesn't even have a parsable package statement
// to the overlay.
continue
}
nextPackage:
for _, p := range response.dr.Packages {
if pkgName != p.Name {
continue
}
for _, f := range p.GoFiles {
if !sameFile(filepath.Dir(f), dir) {
continue
}
if isTest && !hasTestFiles(p) {
// TODO(matloob): Are there packages other than the 'production' variant
// of a package that this can match? This shouldn't match the test main package
// because the file is generated in another directory.
testVariantOf = p
continue nextPackage
}
pkg = p
if filepath.Base(f) == base {
fileExists = true
@@ -82,13 +101,16 @@ func processGolistOverlay(cfg *Config, response *driverResponse) (modifiedPkgs,
if pkgPath == "" {
continue
}
pkgName, ok := extractPackageName(opath, contents)
if !ok {
continue
isXTest := strings.HasSuffix(pkgName, "_test")
if isXTest {
pkgPath += "_test"
}
id := pkgPath
if isTest && !isXTest {
id = fmt.Sprintf("%s [%s.test]", pkgPath, pkgPath)
}
// Try to reclaim a package with the same id if it exists in the response.
for _, p := range response.Packages {
for _, p := range response.dr.Packages {
if reclaimPackage(p, id, opath, contents) {
pkg = p
break
@@ -97,9 +119,13 @@ func processGolistOverlay(cfg *Config, response *driverResponse) (modifiedPkgs,
// Otherwise, create a new package
if pkg == nil {
pkg = &Package{PkgPath: pkgPath, ID: id, Name: pkgName, Imports: make(map[string]*Package)}
// TODO(matloob): Is it okay to amend response.Packages this way?
response.Packages = append(response.Packages, pkg)
response.addPackage(pkg)
havePkgs[pkg.PkgPath] = id
// Add the production package's sources for a test variant.
if isTest && !isXTest && testVariantOf != nil {
pkg.GoFiles = append(pkg.GoFiles, testVariantOf.GoFiles...)
pkg.CompiledGoFiles = append(pkg.CompiledGoFiles, testVariantOf.CompiledGoFiles...)
}
}
}
if !fileExists {
@@ -117,6 +143,7 @@ func processGolistOverlay(cfg *Config, response *driverResponse) (modifiedPkgs,
for _, imp := range imports {
_, found := pkg.Imports[imp]
if !found {
overlayAddsImports = true
// TODO(matloob): Handle cases when the following block isn't correct.
// These include imports of test variants, imports of vendored packages, etc.
id, ok := havePkgs[imp]
@@ -143,7 +170,7 @@ func processGolistOverlay(cfg *Config, response *driverResponse) (modifiedPkgs,
// Do another pass now that new packages have been created to determine the
// set of missing packages.
for _, pkg := range response.Packages {
for _, pkg := range response.dr.Packages {
for _, imp := range pkg.Imports {
pkgPath := toPkgPath(imp.ID)
if _, ok := havePkgs[pkgPath]; !ok {
@@ -152,9 +179,11 @@ func processGolistOverlay(cfg *Config, response *driverResponse) (modifiedPkgs,
}
}
needPkgs = make([]string, 0, len(needPkgsSet))
for pkg := range needPkgsSet {
needPkgs = append(needPkgs, pkg)
if overlayAddsImports {
needPkgs = make([]string, 0, len(needPkgsSet))
for pkg := range needPkgsSet {
needPkgs = append(needPkgs, pkg)
}
}
modifiedPkgs = make([]string, 0, len(modifiedPkgsSet))
for pkg := range modifiedPkgsSet {
@@ -163,6 +192,15 @@ func processGolistOverlay(cfg *Config, response *driverResponse) (modifiedPkgs,
return modifiedPkgs, needPkgs, err
}
func hasTestFiles(p *Package) bool {
for _, f := range p.GoFiles {
if strings.HasSuffix(f, "_test.go") {
return true
}
}
return false
}
// determineRootDirs returns a mapping from directories code can be contained in to the
// corresponding import path prefixes of those directories.
// Its result is used to try to determine the import path for a package containing

View File

@@ -527,28 +527,32 @@ func (ld *loader) refine(roots []string, list ...*Package) ([]*Package, error) {
lpkg.color = grey
stack = append(stack, lpkg) // push
stubs := lpkg.Imports // the structure form has only stubs with the ID in the Imports
lpkg.Imports = make(map[string]*Package, len(stubs))
for importPath, ipkg := range stubs {
var importErr error
imp := ld.pkgs[ipkg.ID]
if imp == nil {
// (includes package "C" when DisableCgo)
importErr = fmt.Errorf("missing package: %q", ipkg.ID)
} else if imp.color == grey {
importErr = fmt.Errorf("import cycle: %s", stack)
}
if importErr != nil {
if lpkg.importErrors == nil {
lpkg.importErrors = make(map[string]error)
// If NeedImports isn't set, the imports fields will all be zeroed out.
// If NeedDeps isn't also set we want to keep the stubs.
if ld.Mode&NeedImports != 0 && ld.Mode&NeedDeps != 0 {
lpkg.Imports = make(map[string]*Package, len(stubs))
for importPath, ipkg := range stubs {
var importErr error
imp := ld.pkgs[ipkg.ID]
if imp == nil {
// (includes package "C" when DisableCgo)
importErr = fmt.Errorf("missing package: %q", ipkg.ID)
} else if imp.color == grey {
importErr = fmt.Errorf("import cycle: %s", stack)
}
if importErr != nil {
if lpkg.importErrors == nil {
lpkg.importErrors = make(map[string]error)
}
lpkg.importErrors[importPath] = importErr
continue
}
lpkg.importErrors[importPath] = importErr
continue
}
if visit(imp) {
lpkg.needsrc = true
if visit(imp) {
lpkg.needsrc = true
}
lpkg.Imports[importPath] = imp.Package
}
lpkg.Imports[importPath] = imp.Package
}
if lpkg.needsrc {
srcPkgs = append(srcPkgs, lpkg)

View File

@@ -13,7 +13,6 @@ import (
"go/parser"
"go/token"
"io/ioutil"
"log"
"os"
"os/exec"
"path"
@@ -68,6 +67,19 @@ func importGroup(env *ProcessEnv, importPath string) int {
return 0
}
type importFixType int
const (
addImport importFixType = iota
deleteImport
setImportName
)
type importFix struct {
info importInfo
fixType importFixType
}
// An importInfo represents a single import statement.
type importInfo struct {
importPath string // import path, e.g. "crypto/rand".
@@ -246,6 +258,12 @@ type pass struct {
// loadPackageNames saves the package names for everything referenced by imports.
func (p *pass) loadPackageNames(imports []*importInfo) error {
if p.env.Debug {
p.env.Logf("loading package names for %v packages", len(imports))
defer func() {
p.env.Logf("done loading package names for %v packages", len(imports))
}()
}
var unknown []string
for _, imp := range imports {
if _, ok := p.knownPackages[imp.importPath]; ok {
@@ -285,7 +303,7 @@ func (p *pass) importIdentifier(imp *importInfo) string {
// load reads in everything necessary to run a pass, and reports whether the
// file already has all the imports it needs. It fills in p.missingRefs with the
// file's missing symbols, if any, or removes unused imports if not.
func (p *pass) load() bool {
func (p *pass) load() ([]*importFix, bool) {
p.knownPackages = map[string]*packageInfo{}
p.missingRefs = references{}
p.existingImports = map[string]*importInfo{}
@@ -313,9 +331,9 @@ func (p *pass) load() bool {
err := p.loadPackageNames(append(imports, p.candidates...))
if err != nil {
if p.env.Debug {
log.Printf("loading package names: %v", err)
p.env.Logf("loading package names: %v", err)
}
return false
return nil, false
}
}
for _, imp := range imports {
@@ -334,16 +352,16 @@ func (p *pass) load() bool {
}
}
if len(p.missingRefs) != 0 {
return false
return nil, false
}
return p.fix()
}
// fix attempts to satisfy missing imports using p.candidates. If it finds
// everything, or if p.lastTry is true, it adds the imports it found,
// removes anything unused, and returns true.
func (p *pass) fix() bool {
// everything, or if p.lastTry is true, it updates fixes to add the imports it found,
// delete anything unused, and update import names, and returns true.
func (p *pass) fix() ([]*importFix, bool) {
// Find missing imports.
var selected []*importInfo
for left, rights := range p.missingRefs {
@@ -353,10 +371,11 @@ func (p *pass) fix() bool {
}
if !p.lastTry && len(selected) != len(p.missingRefs) {
return false
return nil, false
}
// Found everything, or giving up. Add the new imports and remove any unused.
var fixes []*importFix
for _, imp := range p.existingImports {
// We deliberately ignore globals here, because we can't be sure
// they're in the same package. People do things like put multiple
@@ -364,27 +383,77 @@ func (p *pass) fix() bool {
// remove imports if they happen to have the same name as a var in
// a different package.
if _, ok := p.allRefs[p.importIdentifier(imp)]; !ok {
astutil.DeleteNamedImport(p.fset, p.f, imp.name, imp.importPath)
fixes = append(fixes, &importFix{
info: *imp,
fixType: deleteImport,
})
continue
}
// An existing import may need to update its import name to be correct.
if name := p.importSpecName(imp); name != imp.name {
fixes = append(fixes, &importFix{
info: importInfo{
name: name,
importPath: imp.importPath,
},
fixType: setImportName,
})
}
}
for _, imp := range selected {
astutil.AddNamedImport(p.fset, p.f, imp.name, imp.importPath)
fixes = append(fixes, &importFix{
info: importInfo{
name: p.importSpecName(imp),
importPath: imp.importPath,
},
fixType: addImport,
})
}
if p.loadRealPackageNames {
for _, imp := range p.f.Imports {
if imp.Name != nil {
continue
}
path := strings.Trim(imp.Path.Value, `""`)
ident := p.importIdentifier(&importInfo{importPath: path})
if ident != importPathToAssumedName(path) {
imp.Name = &ast.Ident{Name: ident, NamePos: imp.Pos()}
return fixes, true
}
// importSpecName gets the import name of imp in the import spec.
//
// When the import identifier matches the assumed import name, the import name does
// not appear in the import spec.
func (p *pass) importSpecName(imp *importInfo) string {
// If we did not load the real package names, or the name is already set,
// we just return the existing name.
if !p.loadRealPackageNames || imp.name != "" {
return imp.name
}
ident := p.importIdentifier(imp)
if ident == importPathToAssumedName(imp.importPath) {
return "" // ident not needed since the assumed and real names are the same.
}
return ident
}
// apply will perform the fixes on f in order.
func apply(fset *token.FileSet, f *ast.File, fixes []*importFix) bool {
for _, fix := range fixes {
switch fix.fixType {
case deleteImport:
astutil.DeleteNamedImport(fset, f, fix.info.name, fix.info.importPath)
case addImport:
astutil.AddNamedImport(fset, f, fix.info.name, fix.info.importPath)
case setImportName:
// Find the matching import path and change the name.
for _, spec := range f.Imports {
path := strings.Trim(spec.Path.Value, `""`)
if path == fix.info.importPath {
spec.Name = &ast.Ident{
Name: fix.info.name,
NamePos: spec.Pos(),
}
}
}
}
}
return true
}
@@ -437,13 +506,24 @@ func (p *pass) addCandidate(imp *importInfo, pkg *packageInfo) {
var fixImports = fixImportsDefault
func fixImportsDefault(fset *token.FileSet, f *ast.File, filename string, env *ProcessEnv) error {
abs, err := filepath.Abs(filename)
fixes, err := getFixes(fset, f, filename, env)
if err != nil {
return err
}
apply(fset, f, fixes)
return err
}
// getFixes gets the getFixes that need to be made to f in order to fix the imports.
// It does not modify the ast.
func getFixes(fset *token.FileSet, f *ast.File, filename string, env *ProcessEnv) ([]*importFix, error) {
abs, err := filepath.Abs(filename)
if err != nil {
return nil, err
}
srcDir := filepath.Dir(abs)
if env.Debug {
log.Printf("fixImports(filename=%q), abs=%q, srcDir=%q ...", filename, abs, srcDir)
env.Logf("fixImports(filename=%q), abs=%q, srcDir=%q ...", filename, abs, srcDir)
}
// First pass: looking only at f, and using the naive algorithm to
@@ -451,8 +531,8 @@ func fixImportsDefault(fset *token.FileSet, f *ast.File, filename string, env *P
// complete. We can't add any imports yet, because we don't know
// if missing references are actually package vars.
p := &pass{fset: fset, f: f, srcDir: srcDir}
if p.load() {
return nil
if fixes, done := p.load(); done {
return fixes, nil
}
otherFiles := parseOtherFiles(fset, srcDir, filename)
@@ -460,15 +540,15 @@ func fixImportsDefault(fset *token.FileSet, f *ast.File, filename string, env *P
// Second pass: add information from other files in the same package,
// like their package vars and imports.
p.otherFiles = otherFiles
if p.load() {
return nil
if fixes, done := p.load(); done {
return fixes, nil
}
// Now we can try adding imports from the stdlib.
p.assumeSiblingImportsValid()
addStdlibCandidates(p, p.missingRefs)
if p.fix() {
return nil
if fixes, done := p.fix(); done {
return fixes, nil
}
// Third pass: get real package names where we had previously used
@@ -477,25 +557,25 @@ func fixImportsDefault(fset *token.FileSet, f *ast.File, filename string, env *P
p = &pass{fset: fset, f: f, srcDir: srcDir, env: env}
p.loadRealPackageNames = true
p.otherFiles = otherFiles
if p.load() {
return nil
if fixes, done := p.load(); done {
return fixes, nil
}
addStdlibCandidates(p, p.missingRefs)
p.assumeSiblingImportsValid()
if p.fix() {
return nil
if fixes, done := p.fix(); done {
return fixes, nil
}
// Go look for candidates in $GOPATH, etc. We don't necessarily load
// the real exports of sibling imports, so keep assuming their contents.
if err := addExternalCandidates(p, p.missingRefs, filename); err != nil {
return err
return nil, err
}
p.lastTry = true
p.fix()
return nil
fixes, _ := p.fix()
return fixes, nil
}
// ProcessEnv contains environment variables and settings that affect the use of
@@ -512,6 +592,9 @@ type ProcessEnv struct {
// If true, use go/packages regardless of the environment.
ForceGoPackages bool
// Logf is the default logger for the ProcessEnv.
Logf func(format string, args ...interface{})
resolver resolver
}
@@ -577,7 +660,7 @@ func (e *ProcessEnv) invokeGo(args ...string) (*bytes.Buffer, error) {
cmd.Dir = e.WorkingDir
if e.Debug {
defer func(start time.Time) { log.Printf("%s for %v", time.Since(start), cmdDebugStr(cmd)) }(time.Now())
defer func(start time.Time) { e.Logf("%s for %v", time.Since(start), cmdDebugStr(cmd)) }(time.Now())
}
if err := cmd.Run(); err != nil {
return nil, fmt.Errorf("running go: %v (stderr:\n%s)", err, stderr)
@@ -943,7 +1026,7 @@ func VendorlessPath(ipath string) string {
// It returns nil on error or if the package name in dir does not match expectPackage.
func loadExports(ctx context.Context, env *ProcessEnv, expectPackage string, pkg *pkg) (map[string]bool, error) {
if env.Debug {
log.Printf("loading exports in dir %s (seeking package %s)", pkg.dir, expectPackage)
env.Logf("loading exports in dir %s (seeking package %s)", pkg.dir, expectPackage)
}
if pkg.goPackage != nil {
exports := map[string]bool{}
@@ -1021,7 +1104,7 @@ func loadExports(ctx context.Context, env *ProcessEnv, expectPackage string, pkg
exportList = append(exportList, k)
}
sort.Strings(exportList)
log.Printf("loaded exports in dir %v (package %v): %v", pkg.dir, expectPackage, strings.Join(exportList, ", "))
env.Logf("loaded exports in dir %v (package %v): %v", pkg.dir, expectPackage, strings.Join(exportList, ", "))
}
return exports, nil
}
@@ -1058,7 +1141,7 @@ func findImport(ctx context.Context, pass *pass, dirScan []*pkg, pkgName string,
sort.Sort(byDistanceOrImportPathShortLength(candidates))
if pass.env.Debug {
for i, c := range candidates {
log.Printf("%s candidate %d/%d: %v in %v", pkgName, i+1, len(candidates), c.pkg.importPathShort, c.pkg.dir)
pass.env.Logf("%s candidate %d/%d: %v in %v", pkgName, i+1, len(candidates), c.pkg.importPathShort, c.pkg.dir)
}
}
@@ -1098,7 +1181,7 @@ func findImport(ctx context.Context, pass *pass, dirScan []*pkg, pkgName string,
exports, err := loadExports(ctx, pass.env, pkgName, c.pkg)
if err != nil {
if pass.env.Debug {
log.Printf("loading exports in dir %s (seeking package %s): %v", c.pkg.dir, pkgName, err)
pass.env.Logf("loading exports in dir %s (seeking package %s): %v", c.pkg.dir, pkgName, err)
}
resc <- nil
return

View File

@@ -19,6 +19,7 @@ import (
"go/token"
"io"
"io/ioutil"
"log"
"regexp"
"strconv"
"strings"
@@ -50,6 +51,11 @@ func Process(filename string, src []byte, opt *Options) ([]byte, error) {
src = b
}
// Set the logger if the user has not provided it.
if opt.Env.Logf == nil {
opt.Env.Logf = log.Printf
}
fileSet := token.NewFileSet()
file, adjust, err := parse(fileSet, filename, src, opt)
if err != nil {

View File

@@ -4,7 +4,6 @@ import (
"bytes"
"encoding/json"
"io/ioutil"
"log"
"os"
"path"
"path/filepath"
@@ -63,7 +62,7 @@ func (r *moduleResolver) init() error {
}
if mod.Dir == "" {
if r.env.Debug {
log.Printf("module %v has not been downloaded and will be ignored", mod.Path)
r.env.Logf("module %v has not been downloaded and will be ignored", mod.Path)
}
// Can't do anything with a module that's not downloaded.
continue
@@ -254,7 +253,7 @@ func (r *moduleResolver) scan(_ references) ([]*pkg, error) {
modPath, err := module.DecodePath(filepath.ToSlash(matches[1]))
if err != nil {
if r.env.Debug {
log.Printf("decoding module cache path %q: %v", subdir, err)
r.env.Logf("decoding module cache path %q: %v", subdir, err)
}
return
}