[K9CODESEC-3103] Add static Go reachability analysis

LICENSE-3rdparty.csv

@@ -4,6 +4,7 @@ core,github.com/BurntSushi/toml,MIT,Copyright (c) 2013 TOML authors
 core,github.com/BurntSushi/toml/internal,MIT,Copyright (c) 2013 TOML authors
 core,github.com/CycloneDX/cyclonedx-go,Apache-2.0,Copyright & License | Copyright (c) OWASP Foundation | Copyright (c) OWASP Foundation. All Rights Reserved | Copyright OWASP Foundation
 core,github.com/DataDog/jsonapi,Apache-2.0,"Copyright [2022-Present] Datadog, Inc"
+core,github.com/Masterminds/semver/v3,MIT,"Copyright (C) 2014-2019, Matt Butcher and Matt Farina"
 core,github.com/Microsoft/go-winio,MIT,Copyright (c) 2015 Microsoft
 core,github.com/Microsoft/go-winio/internal/fs,MIT,Copyright (c) 2015 Microsoft
 core,github.com/Microsoft/go-winio/internal/socket,MIT,Copyright (c) 2015 Microsoft
@@ -137,7 +138,6 @@ core,github.com/kevinburke/ssh_config,MIT,"Copyright (c) 2013 - 2017 Thomas Pell
 core,github.com/kr/pretty,MIT,Copyright 2012 Keith Rarick
 core,github.com/kr/text,MIT,Copyright 2012 Keith Rarick
 core,github.com/maruel/natural,Apache-2.0,Copyright 2018 Marc-Antoine Ruel
-core,github.com/Masterminds/semver/v3,MIT,"Copyright (C) 2014-2019, Matt Butcher and Matt Farina"
 core,github.com/mattn/go-pointer,MIT,Copyright (c) 2019 Yasuhiro Matsumoto
 core,github.com/mattn/go-runewidth,MIT,Copyright (c) 2016 Yasuhiro Matsumoto
 core,github.com/package-url/packageurl-go,MIT,Copyright (c) the purl authors
@@ -155,6 +155,7 @@ core,github.com/tidwall/match,MIT,Copyright (c) 2016 Josh Baker
 core,github.com/tidwall/pretty,MIT,Copyright (c) 2017 Josh Baker
 core,github.com/tidwall/sjson,MIT,Copyright (c) 2016 Josh Baker
 core,github.com/tree-sitter/go-tree-sitter,MIT,Copyright (c) 2024 Amaan Qureshi <[email protected]>
+core,github.com/tree-sitter/tree-sitter-go/bindings/go,MIT,Copyright (c) 2014 Max Brunsfeld
 core,github.com/tree-sitter/tree-sitter-java/bindings/go,MIT,Copyright (c) 2017 Ayman Nadeem
 core,github.com/tree-sitter/tree-sitter-ruby/bindings/go,MIT,Copyright (c) 2016 Rob Rix
 core,github.com/urfave/cli/v2,MIT,Copyright (c) 2022 urfave/cli maintainers

go.mod

@@ -17,6 +17,7 @@ require (
 	github.com/stretchr/testify v1.10.0
 	github.com/tidwall/jsonc v0.3.3
 	github.com/tree-sitter/go-tree-sitter v0.25.0
+	github.com/tree-sitter/tree-sitter-go v0.25.0
 	github.com/tree-sitter/tree-sitter-java v0.23.5
 	github.com/tree-sitter/tree-sitter-ruby v0.23.1
 	github.com/urfave/cli/v2 v2.27.6

go.sum

@@ -127,8 +127,8 @@ github.com/tree-sitter/tree-sitter-cpp v0.23.4 h1:LaWZsiqQKvR65yHgKmnaqA+uz6tlDJ
 github.com/tree-sitter/tree-sitter-cpp v0.23.4/go.mod h1:doqNW64BriC7WBCQ1klf0KmJpdEvfxyXtoEybnBo6v8=
 github.com/tree-sitter/tree-sitter-embedded-template v0.23.2 h1:nFkkH6Sbe56EXLmZBqHHcamTpmz3TId97I16EnGy4rg=
 github.com/tree-sitter/tree-sitter-embedded-template v0.23.2/go.mod h1:HNPOhN0qF3hWluYLdxWs5WbzP/iE4aaRVPMsdxuzIaQ=
-github.com/tree-sitter/tree-sitter-go v0.23.4 h1:yt5KMGnTHS+86pJmLIAZMWxukr8W7Ae1STPvQUuNROA=
-github.com/tree-sitter/tree-sitter-go v0.23.4/go.mod h1:Jrx8QqYN0v7npv1fJRH1AznddllYiCMUChtVjxPK040=
+github.com/tree-sitter/tree-sitter-go v0.25.0 h1:cEB0Q3LHgZtS+ECHx9wcP7AwzoOddJFQCVmytX42cVU=
+github.com/tree-sitter/tree-sitter-go v0.25.0/go.mod h1:Jrx8QqYN0v7npv1fJRH1AznddllYiCMUChtVjxPK040=
 github.com/tree-sitter/tree-sitter-html v0.23.2 h1:1UYDV+Yd05GGRhVnTcbP58GkKLSHHZwVaN+lBZV11Lc=
 github.com/tree-sitter/tree-sitter-html v0.23.2/go.mod h1:gpUv/dG3Xl/eebqgeYeFMt+JLOY9cgFinb/Nw08a9og=
 github.com/tree-sitter/tree-sitter-java v0.23.5 h1:J9YeMGMwXYlKSP3K4Us8CitC6hjtMjqpeOf2GGo6tig=

pkg/reachability/codefile/common.go

@@ -0,0 +1,93 @@
+package codefile
+
+import (
+	"context"
+	"os"
+
+	"github.com/DataDog/datadog-sbom-generator/internal/utility/converter"
+	"github.com/DataDog/datadog-sbom-generator/internal/utility/fileposition"
+	"github.com/DataDog/datadog-sbom-generator/pkg/models"
+
+	treesitter "github.com/tree-sitter/go-tree-sitter"
+)
+
+// readFileContent is a thin wrapper over os.ReadFile that reads the content of a file
+// and returns it as a byte slice.
+// TODO(daniel.strong): find a better place for this function
+func readFileContent(filePath string) ([]byte, error) {
+	data, err := os.ReadFile(filePath)
+	if err != nil {
+		return nil, err
+	}
+
+	return data, nil
+}
+
+// parseFile parses fileContent with tsParser, canceling the parse as soon as ctx is done.
+func parseFile(ctx context.Context, tsParser *treesitter.Parser, fileContent []byte) *treesitter.Tree {
+	readCallback := func(offset int, position treesitter.Point) []byte {
+		if ctx.Err() != nil {
+			return []byte{}
+		}
+		if offset >= len(fileContent) {
+			return []byte{}
+		}
+
+		return fileContent[offset:]
+	}
+
+	return tsParser.ParseWithOptions(readCallback, nil, &treesitter.ParseOptions{
+		// ProgressCallback returns true to cancel parsing
+		// We use ctx.Err() != nil to cancel the parse if the context is canceled
+		// See: https://github.com/tree-sitter/go-tree-sitter/blob/adc13ffd8b2c0b01b878fda9f7c422ce0df5fad3/parser.go#L319
+		ProgressCallback: func(_ treesitter.ParseState) bool {
+			return ctx.Err() != nil
+		},
+	})
+}
+
+// buildPackageLocation converts a tree-sitter start/end position pair into a models.PackageLocation
+// for a match found in the file at path (relative to dir).
+func buildPackageLocation(dir string, path string, start treesitter.Point, end treesitter.Point) (models.PackageLocation, error) {
+	packageLocation := models.PackageLocation{
+		Filename: fileposition.ToRelativePath(dir, path),
+	}
+
+	var err error
+	packageLocation.LineStart, err = converter.SafeUIntToInt(start.Row + 1)
+	if err != nil {
+		return models.PackageLocation{}, err
+	}
+	packageLocation.LineEnd, err = converter.SafeUIntToInt(end.Row + 1)
+	if err != nil {
+		return models.PackageLocation{}, err
+	}
+	packageLocation.ColumnStart, err = converter.SafeUIntToInt(start.Column + 1)
+	if err != nil {
+		return models.PackageLocation{}, err
+	}
+	packageLocation.ColumnEnd, err = converter.SafeUIntToInt(end.Column + 1)
+	if err != nil {
+		return models.PackageLocation{}, err
+	}
+
+	return packageLocation, nil
+}
+
+// recordMatch appends a reachable symbol match to detectionResults, initializing the
+// per-purl and per-advisory maps if this is the first match for either.
+func recordMatch(detectionResults models.DetectionResults, purl string, advisoryID string, symbol string, location models.PackageLocation) {
+	if _, ok := detectionResults[purl]; !ok {
+		detectionResults[purl] = make(map[string]models.ReachableSymbolLocations)
+	}
+	if _, ok := detectionResults[purl][advisoryID]; !ok {
+		detectionResults[purl][advisoryID] = make(models.ReachableSymbolLocations, 0)
+	}
+
+	detectionResults[purl][advisoryID] = append(
+		detectionResults[purl][advisoryID],
+		models.ReachableSymbolLocation{
+			Symbol:          symbol,
+			PackageLocation: location,
+		})
+}

pkg/reachability/codefile/detector.go

@@ -0,0 +1,19 @@
+package codefile
+
+import (
+	"context"
+
+	"github.com/DataDog/datadog-sbom-generator/pkg/models"
+)
+
+// Detector parses source files for one language and matches vulnerable symbols against how
+// those symbols are actually imported and called in that source.
+type Detector interface {
+	Detect(ctx context.Context, dir string, path string, detectionResults models.DetectionResults, advisoriesToCheck []models.AdvisoryToCheck) error
+	Close()
+}
+
+var (
+	_ Detector = (*ReachabilityJava)(nil)
+	_ Detector = (*ReachabilityGo)(nil)
+)

pkg/reachability/codefile/golang.go

@@ -0,0 +1,235 @@
+package codefile
+
+import (
+	"context"
+	"fmt"
+	"slices"
+	"strings"
+
+	"github.com/DataDog/datadog-sbom-generator/internal/cachedregexp"
+	"github.com/DataDog/datadog-sbom-generator/pkg/models"
+	"github.com/DataDog/datadog-sbom-generator/pkg/reporter"
+
+	treesitter "github.com/tree-sitter/go-tree-sitter"
+	tree_sitter_go "github.com/tree-sitter/tree-sitter-go/bindings/go"
+)
+
+// symbolTypeFunction is the only Go symbol type currently understood.
+const symbolTypeFunction = "function"
+
+const tsQueryForGoImports = `
+(import_spec
+	name: (package_identifier)? @alias
+	name: (dot)? @blankOrDotImport
+	name: (blank_identifier)? @blankOrDotImport
+	path: (interpreted_string_literal (interpreted_string_literal_content) @path))
+`
+
+const tsQueryForGoCall = `
+(call_expression
+	function: (selector_expression
+		operand: (identifier) @pkg
+		field: (field_identifier) @fn) @selector)
+`
+
+var majorVersionSuffixPattern = cachedregexp.MustCompile(`^v([2-9]|[1-9][0-9]+)$`)
+var dottedMajorVersionSuffixPattern = cachedregexp.MustCompile(`\.v[0-9]+$`)
+
+type ReachabilityGo struct {
+	tsParser    *treesitter.Parser
+	importQuery *treesitter.Query
+	callQuery   *treesitter.Query
+
+	aliasCaptureIdx      uint
+	blankOrDotCaptureIdx uint
+	pathCaptureIdx       uint
+	pkgCaptureIdx        uint
+	fnCaptureIdx         uint
+	selectorCaptureIdx   uint
+
+	reporter reporter.Reporter
+}
+
+// NewGoReachableDetector creates a new ReachabilityGo instance that once instantiated can be
+// used to parse Go files. You should call Close() on the instance once you're finished parsing.
+func NewGoReachableDetector(r reporter.Reporter) (*ReachabilityGo, error) {
+	tsLanguage := treesitter.NewLanguage(tree_sitter_go.Language())
+
+	tsParser := treesitter.NewParser()
+	if err := tsParser.SetLanguage(tsLanguage); err != nil {
+		return nil, fmt.Errorf("failed to set tree-sitter Go language on parser: %w", err)
+	}
+
+	importQuery, err := treesitter.NewQuery(tsLanguage, tsQueryForGoImports)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create tree-sitter query for Go imports: %w", err)
+	}
+
+	callQuery, err := treesitter.NewQuery(tsLanguage, tsQueryForGoCall)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create tree-sitter query for Go calls: %w", err)
+	}
+
+	aliasCaptureIdx, _ := importQuery.CaptureIndexForName("alias")
+	blankOrDotCaptureIdx, _ := importQuery.CaptureIndexForName("blankOrDotImport")
+	pathCaptureIdx, _ := importQuery.CaptureIndexForName("path")
+	pkgCaptureIdx, _ := callQuery.CaptureIndexForName("pkg")
+	fnCaptureIdx, _ := callQuery.CaptureIndexForName("fn")
+	selectorCaptureIdx, _ := callQuery.CaptureIndexForName("selector")
+
+	return &ReachabilityGo{
+		tsParser:             tsParser,
+		importQuery:          importQuery,
+		callQuery:            callQuery,
+		aliasCaptureIdx:      aliasCaptureIdx,
+		blankOrDotCaptureIdx: blankOrDotCaptureIdx,
+		pathCaptureIdx:       pathCaptureIdx,
+		pkgCaptureIdx:        pkgCaptureIdx,
+		fnCaptureIdx:         fnCaptureIdx,
+		selectorCaptureIdx:   selectorCaptureIdx,
+		reporter:             reporter.Effective(r),
+	}, nil
+}
+
+// Close closes all hanging tree-sitter related resources.
+// This should only be called once you're finished parsing all Go files.
+func (r *ReachabilityGo) Close() {
+	r.tsParser.Close()
+	r.importQuery.Close()
+	r.callQuery.Close()
+}
+
+// resolveImportAliases walks all import specs in the parsed tree and returns a map of module
+// import path -> local identifiers used to reference that module in this file. Unaliased
+// imports are assigned a heuristic identifier: the last path segment, with a trailing
+// major-version suffix (e.g. "/v2") stripped, per Go module convention. Dot imports and blank
+// imports are skipped entirely: they don't bind a package selector (e.g. "pkg.Func"), so
+// defaulting them to an identifier would risk matching an unrelated import that happens to
+// resolve to the same default alias.
+func (r *ReachabilityGo) resolveImportAliases(tree *treesitter.Tree, fileContent []byte, queryCursor *treesitter.QueryCursor) map[string][]string {
+	moduleToAliases := make(map[string][]string)
+
+	matches := queryCursor.Matches(r.importQuery, tree.RootNode(), fileContent)
+	for match := matches.Next(); match != nil; match = matches.Next() {
+		var alias, modulePath string
+		var isBlankOrDotImport bool
+
+		for _, capture := range match.Captures {
+			switch capture.Index {
+			case uint32(r.aliasCaptureIdx): //nolint:gosec
+				alias = capture.Node.Utf8Text(fileContent)
+			case uint32(r.blankOrDotCaptureIdx): //nolint:gosec
+				isBlankOrDotImport = true
+			case uint32(r.pathCaptureIdx): //nolint:gosec
+				modulePath = capture.Node.Utf8Text(fileContent)
+			}
+		}
+
+		if modulePath == "" || isBlankOrDotImport {
+			continue
+		}
+
+		if alias == "" {
+			alias = defaultIdentifierForModulePath(modulePath)
+		}
+
+		moduleToAliases[modulePath] = append(moduleToAliases[modulePath], alias)
+	}
+
+	return moduleToAliases
+}
+
+// defaultIdentifierForModulePath derives the package identifier Go code would use for an
+// unaliased import, using the last path segment and stripping a trailing major-version suffix.
+// Two version conventions are handled: the path-segment style (e.g. "github.com/foo/bar/v2" ->
+// "bar") and the dotted gopkg.in style (e.g. "gopkg.in/yaml.v3" -> "yaml"). The path-segment style
+// only strips "v2" and above: Go's semantic-import-versioning convention never produces a "v0" or
+// "v1" module-major-version suffix, so a trailing "v0"/"v1" segment (e.g. "k8s.io/api/core/v1") is
+// always a real package name, not a version marker. Leading "go-" and trailing "-go"
+// repository-naming conventions are also stripped (e.g. "github.com/redis/go-redis/v9" -> "redis",
+// "github.com/CycloneDX/cyclonedx-go" -> "cyclonedx"), since Go identifiers can't contain hyphens,
+// so a hyphenated segment is never the real package name.
+func defaultIdentifierForModulePath(modulePath string) string {
+	segments := strings.Split(modulePath, "/")
+	identifier := segments[len(segments)-1]
+
+	switch {
+	case len(segments) > 1 && majorVersionSuffixPattern.MatchString(identifier):
+		identifier = segments[len(segments)-2]
+	case dottedMajorVersionSuffixPattern.MatchString(identifier):
+		identifier = identifier[:strings.LastIndex(identifier, ".")]
+	}
+
+	if rest, ok := strings.CutPrefix(identifier, "go-"); ok && rest != "" {
+		identifier = rest
+	} else if rest, ok := strings.CutSuffix(identifier, "-go"); ok && rest != "" {
+		identifier = rest
+	}
+
+	return identifier
+}
+
+func (r *ReachabilityGo) Detect(ctx context.Context, dir string, path string, detectionResults models.DetectionResults, advisoriesToCheck []models.AdvisoryToCheck) error {
+	fileContent, err := readFileContent(path)
+	if err != nil {
+		return err
+	}
+
+	tree := parseFile(ctx, r.tsParser, fileContent)
+	defer tree.Close()
+
+	if len(advisoriesToCheck) == 0 {
+		return nil
+	}
+
+	importCursor := treesitter.NewQueryCursor()
+	defer importCursor.Close()
+	moduleToAliases := r.resolveImportAliases(tree, fileContent, importCursor)
+
+	callCursor := treesitter.NewQueryCursor()
+	defer callCursor.Close()
+
+	for _, advisoryToCheck := range advisoriesToCheck {
+		for _, s := range advisoryToCheck.Symbols {
+			if s.Type != symbolTypeFunction {
+				r.reporter.Warnf("No Go detection support for symbol type %s", s.Type)
+				continue
+			}
+
+			aliases, moduleImported := moduleToAliases[s.Value]
+			if !moduleImported {
+				continue
+			}
+
+			matches := callCursor.Matches(r.callQuery, tree.RootNode(), fileContent)
+			for match := matches.Next(); match != nil; match = matches.Next() {
+				var pkgText, fnText string
+				var selectorNode treesitter.Node
+
+				for _, capture := range match.Captures {
+					switch capture.Index {
+					case uint32(r.pkgCaptureIdx): //nolint:gosec
+						pkgText = capture.Node.Utf8Text(fileContent)
+					case uint32(r.fnCaptureIdx): //nolint:gosec
+						fnText = capture.Node.Utf8Text(fileContent)
+					case uint32(r.selectorCaptureIdx): //nolint:gosec
+						selectorNode = capture.Node
+					}
+				}
+
+				if fnText != s.Name || !slices.Contains(aliases, pkgText) {
+					continue
+				}
+
+				packageLocation, err := buildPackageLocation(dir, path, selectorNode.StartPosition(), selectorNode.EndPosition())
+				if err != nil {
+					return err
+				}
+
+				recordMatch(detectionResults, advisoryToCheck.Purl, advisoryToCheck.AdvisoryID, selectorNode.Utf8Text(fileContent), packageLocation)
+			}
+		}
+	}
+
+	return nil
+}