Skip to content
Open
Show file tree
Hide file tree
Changes from 4 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 11 additions & 5 deletions main.go
Original file line number Diff line number Diff line change
Expand Up @@ -202,6 +202,7 @@ var (
dockerExcludePaths = dockerScan.Flag("exclude-paths", "Comma separated list of paths to exclude from scan").String()
dockerScanNamespace = dockerScan.Flag("namespace", "Docker namespace (organization or user). For non-Docker Hub registries, include the registry address as well (e.g., ghcr.io/namespace or quay.io/namespace).").String()
dockerScanRegistryToken = dockerScan.Flag("registry-token", "Optional Docker registry access token. Provide this if you want to include private images within the specified namespace.").String()
dockerScanRegistry = dockerScan.Flag("registry", "Scan all images in a registry host. Supports OCI Distribution Spec compliant registries (Harbor, Nexus, Artifactory, etc.). Use --registry-token for authentication.").String()

travisCiScan = cli.Command("travisci", "Scan TravisCI")
travisCiScanToken = travisCiScan.Flag("token", "TravisCI token. Can also be provided with environment variable").Envar("TRAVISCI_TOKEN").Required().String()
Expand Down Expand Up @@ -1014,21 +1015,26 @@ func runSingleScan(ctx context.Context, cmd string, cfg engine.Config) (metrics,
return scanMetrics, fmt.Errorf("invalid config: you cannot specify both images and namespace at the same time")
}

if *dockerScanImages == nil && *dockerScanNamespace == "" {
return scanMetrics, fmt.Errorf("invalid config: both images and namespace cannot be empty; one is required")
if *dockerScanImages == nil && *dockerScanNamespace == "" && *dockerScanRegistry == "" {
return scanMetrics, fmt.Errorf("invalid config: one of --image, --namespace, or --registry is required")
}

if *dockerScanRegistryToken != "" && *dockerScanNamespace == "" {
return scanMetrics, fmt.Errorf("invalid config: registry token can only be used with registry namespace")
if *dockerScanRegistry != "" && (*dockerScanImages != nil || *dockerScanNamespace != "") {
return scanMetrics, fmt.Errorf("invalid config: --registry cannot be combined with --image or --namespace")
}

if *dockerScanRegistryToken != "" && *dockerScanNamespace == "" && *dockerScanRegistry == "" {
return scanMetrics, fmt.Errorf("invalid config: --registry-token requires --namespace or --registry")
}

cfg := sources.DockerConfig{
BearerToken: *dockerScanToken,
Images: *dockerScanImages,
UseDockerKeychain: *dockerScanToken == "",
UseDockerKeychain: *dockerScanToken == "" && *dockerScanRegistry == "",
ExcludePaths: strings.Split(*dockerExcludePaths, ","),
Namespace: *dockerScanNamespace,
RegistryToken: *dockerScanRegistryToken,
Registry: *dockerScanRegistry,
}
if ref, err := eng.ScanDocker(ctx, cfg); err != nil {
return scanMetrics, fmt.Errorf("failed to scan Docker: %v", err)
Expand Down
1 change: 1 addition & 0 deletions pkg/engine/docker.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ func (e *Engine) ScanDocker(ctx context.Context, c sources.DockerConfig) (source
ExcludePaths: c.ExcludePaths,
Namespace: c.Namespace,
RegistryToken: c.RegistryToken,
Registry: c.Registry,
}

switch {
Expand Down
8 changes: 8 additions & 0 deletions pkg/pb/sourcespb/sources.pb.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

15 changes: 15 additions & 0 deletions pkg/sources/docker/docker.go
Original file line number Diff line number Diff line change
Expand Up @@ -140,6 +140,21 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ .
s.conn.Images = append(s.conn.Images, namespaceImages...)
}

// if a registry host is set, enumerate all images from that registry via /v2/_catalog.
if registryHost := s.conn.GetRegistry(); registryHost != "" {
start := time.Now()
registry := MakeRegistryFromHost(registryHost)
if token := s.conn.GetRegistryToken(); token != "" {
registry.WithRegistryToken(token)
}
registryImages, err := registry.ListImages(ctx, "")
if err != nil {
return fmt.Errorf("failed to list registry %s images: %w", registryHost, err)
}
dockerListImagesAPIDuration.WithLabelValues(s.name).Observe(time.Since(start).Seconds())
s.conn.Images = append(s.conn.Images, registryImages...)
}

for _, image := range s.conn.GetImages() {
if common.IsDone(ctx) {
return nil
Expand Down
122 changes: 122 additions & 0 deletions pkg/sources/docker/registries.go
Original file line number Diff line number Diff line change
Expand Up @@ -413,3 +413,125 @@ func discardBody(resp *http.Response) {
_ = resp.Body.Close()
}
}

// === Generic OCI Registry ===

// GenericOCIRegistry implements the Registry interface for any OCI Distribution Spec
// compliant registry (Harbor, Nexus, Artifactory, etc.) using the /v2/_catalog endpoint.
type GenericOCIRegistry struct {
Host string
Token string
Client *http.Client
scheme string // defaults to "https"; overridable for testing
}

// catalogResp models the JSON response from the /v2/_catalog endpoint.
type catalogResp struct {
Repositories []string `json:"repositories"`
}

func (g *GenericOCIRegistry) Name() string {
return g.Host
}

func (g *GenericOCIRegistry) WithRegistryToken(token string) {
g.Token = token
}

func (g *GenericOCIRegistry) WithClient(client *http.Client) {
g.Client = client
}

// ListImages enumerates all repositories from an OCI Distribution Spec compliant registry
// using the /v2/_catalog endpoint. The namespace parameter is unused.
// Pagination is handled via the Link response header.
func (g *GenericOCIRegistry) ListImages(ctx context.Context, _ string) ([]string, error) {
scheme := g.scheme
if scheme == "" {
scheme = "https"
}

baseURL := &url.URL{
Scheme: scheme,
Host: g.Host,
Path: "v2/_catalog",
}

query := baseURL.Query()
query.Set("n", fmt.Sprint(maxRegistryPageSize))
baseURL.RawQuery = query.Encode()

allImages := []string{}
nextURL := baseURL.String()

for nextURL != "" {
if err := registryRateLimiter.Wait(ctx); err != nil {
return nil, err
}

req, err := http.NewRequestWithContext(ctx, http.MethodGet, nextURL, http.NoBody)
if err != nil {
return nil, err
}

if g.Token != "" {
req.Header.Set("Authorization", "Bearer "+g.Token)
}

client := g.Client
if client == nil {
client = defaultHTTPClient
}
resp, err := client.Do(req)
if err != nil {
return nil, err
}

body, err := io.ReadAll(resp.Body)
discardBody(resp)
if err != nil {
return nil, err
}

if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("failed to list registry images: unexpected status code: %d", resp.StatusCode)
}

var page catalogResp
if err := json.Unmarshal(body, &page); err != nil {
return nil, err
}

for _, repo := range page.Repositories {
allImages = append(allImages, fmt.Sprintf("%s/%s", g.Host, repo))
}

linkHeader := resp.Header.Get("Link")
if linkHeader != "" {
nextURL = resolveNextURL(baseURL, linkHeader)
} else {
nextURL = ""
}
}

return allImages, nil
}

func resolveNextURL(baseURL *url.URL, linkHeader string) string {
nextLink := parseNextLinkURL(linkHeader)
if nextLink == "" {
return ""
}

parsedNext, err := url.Parse(nextLink)
if err != nil {
return ""
}

return baseURL.ResolveReference(parsedNext).String()
}
Comment thread
cursor[bot] marked this conversation as resolved.

// MakeRegistryFromHost returns a GenericOCIRegistry for the given registry host.
func MakeRegistryFromHost(host string) Registry {
return &GenericOCIRegistry{Host: host}
}
118 changes: 118 additions & 0 deletions pkg/sources/docker/registries_test.go
Original file line number Diff line number Diff line change
@@ -1,8 +1,10 @@
package docker

import (
"encoding/json"
"fmt"
"net/http"
"net/http/httptest"
"slices"
"testing"

Expand Down Expand Up @@ -100,3 +102,119 @@ func TestGHCRListImages_RateLimitError(t *testing.T) {
assert.Error(t, err)
assert.Nil(t, ghcrImages)
}

func TestGenericOCIRegistryListImages(t *testing.T) {
t.Parallel()

srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
assert.Equal(t, "/v2/_catalog", r.URL.Path)
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(catalogResp{Repositories: []string{"myapp", "mydb"}})
}))
defer srv.Close()

reg := &GenericOCIRegistry{Host: srv.Listener.Addr().String()}
reg.WithClient(srv.Client())

// Override scheme to http for the test server.
reg.scheme = "http"

images, err := reg.ListImages(context.Background(), "")
assert.NoError(t, err)

expected := []string{
srv.Listener.Addr().String() + "/myapp",
srv.Listener.Addr().String() + "/mydb",
}
slices.Sort(images)
slices.Sort(expected)
assert.Equal(t, expected, images)
}

func TestGenericOCIRegistryListImages_Pagination(t *testing.T) {
t.Parallel()

page := 0
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
if page == 0 {
w.Header().Set("Link", `</v2/_catalog?n=2&last=repo2>; rel="next"`)
_ = json.NewEncoder(w).Encode(catalogResp{Repositories: []string{"repo1", "repo2"}})
page++
} else {
_ = json.NewEncoder(w).Encode(catalogResp{Repositories: []string{"repo3"}})
}
}))
defer srv.Close()

reg := &GenericOCIRegistry{Host: srv.Listener.Addr().String(), scheme: "http"}
reg.WithClient(srv.Client())

images, err := reg.ListImages(context.Background(), "")
assert.NoError(t, err)
assert.Len(t, images, 3)
}

func TestGenericOCIRegistryListImages_PaginationAbsoluteURL(t *testing.T) {
t.Parallel()

page := 0
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
if page == 0 {
w.Header().Set("Link", fmt.Sprintf(`<%s/v2/_catalog?n=2&last=repo2>; rel="next"`, "http://"+r.Host))
_ = json.NewEncoder(w).Encode(catalogResp{Repositories: []string{"repo1", "repo2"}})
page++
} else {
_ = json.NewEncoder(w).Encode(catalogResp{Repositories: []string{"repo3"}})
}
}))
defer srv.Close()

reg := &GenericOCIRegistry{Host: srv.Listener.Addr().String(), scheme: "http"}
reg.WithClient(srv.Client())

images, err := reg.ListImages(context.Background(), "")
assert.NoError(t, err)
assert.Len(t, images, 3)
}

func TestGenericOCIRegistryListImages_AuthHeader(t *testing.T) {
t.Parallel()

srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
assert.Equal(t, "Bearer mytoken", r.Header.Get("Authorization"))
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(catalogResp{Repositories: []string{"secured-app"}})
}))
defer srv.Close()

reg := &GenericOCIRegistry{Host: srv.Listener.Addr().String(), scheme: "http"}
reg.WithClient(srv.Client())
reg.WithRegistryToken("mytoken")

images, err := reg.ListImages(context.Background(), "")
assert.NoError(t, err)
assert.Equal(t, []string{srv.Listener.Addr().String() + "/secured-app"}, images)
}

func TestGenericOCIRegistryListImages_ErrorStatus(t *testing.T) {
t.Parallel()

reg := &GenericOCIRegistry{Host: "127.0.0.1:9"}
reg.WithClient(common.ConstantResponseHttpClient(http.StatusUnauthorized, "{}"))
reg.scheme = "http"

images, err := reg.ListImages(context.Background(), "")
assert.Error(t, err)
assert.Nil(t, images)
}

func TestMakeRegistryFromHost(t *testing.T) {
t.Parallel()

reg := MakeRegistryFromHost("registry.example.com")
assert.Equal(t, "registry.example.com", reg.Name())
_, ok := reg.(*GenericOCIRegistry)
assert.True(t, ok)
}
2 changes: 2 additions & 0 deletions pkg/sources/sources.go
Original file line number Diff line number Diff line change
Expand Up @@ -233,6 +233,8 @@ type DockerConfig struct {
Namespace string
// RegistryToken is an optional authentication token used to access private images within the namespace.
RegistryToken string
// Registry is the full registry host to enumerate all images from (e.g., registry.example.com).
Registry string
}

// GCSConfig defines the optional configuration for a GCS source.
Expand Down
1 change: 1 addition & 0 deletions proto/sources.proto
Original file line number Diff line number Diff line change
Expand Up @@ -168,6 +168,7 @@ message Docker {
repeated string exclude_paths = 6;
string namespace = 7;
string registry_token = 8;
string registry = 9;
}

message ECR {
Expand Down