Skip to content

Prevent source credentials leaking to output #615

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
alecthomas wants to merge 3 commits into from
Closed

Prevent source credentials leaking to output #615

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
c58a158
fix: prevent source credentials leaking to output
alecthomas Aug 14, 2026
78927f5
refactor: preserve source redaction boundaries
alecthomas Aug 14, 2026
6095d5c
refactor: preserve the existing Source API name
alecthomas Aug 14, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
83 changes: 59 additions & 24 deletions cache/cache.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import (
"time"

"github.com/cashapp/hermit/errors"
"github.com/cashapp/hermit/sources"
"github.com/cashapp/hermit/ui"
"github.com/cashapp/hermit/util"
)
Expand All @@ -25,8 +26,12 @@ type Cache struct {

// BasePath returns the subfolder in the cache path for the given file
func BasePath(checksum, uri string) string {
hash := util.Hash(uri, checksum)
return filepath.Join(hash[:2], hash+"-"+filepath.Base(uri))
return basePath(checksum, sources.NewSourceURI(uri))
}

func basePath(checksum string, uri sources.SourceURI) string {
hash := util.Hash(uri.Get(), checksum)
return filepath.Join(hash[:2], hash+"-"+filepath.Base(uri.Get()))
}

// Open or create a Cache at the given directory, using the given http client.
Expand Down Expand Up @@ -65,13 +70,13 @@ func (c *Cache) Root() string {

// Mkdir makes a directory for the given URI.
func (c *Cache) Mkdir(uri string) (string, error) {
path := c.Path("", uri)
path := c.path("", sources.NewSourceURI(uri))
return path, os.MkdirAll(path, os.ModePerm) //nolint:gosec
}

// Create a new, empty, cache entry.
func (c *Cache) Create(checksum, uri string) (*os.File, error) {
path := c.Path(checksum, uri)
path := c.path(checksum, sources.NewSourceURI(uri))
dir := filepath.Dir(path)
err := os.MkdirAll(dir, os.ModePerm) //nolint:gosec
if err != nil {
Expand All @@ -82,7 +87,7 @@ func (c *Cache) Create(checksum, uri string) (*os.File, error) {

// OpenLocal opens a local cached copy of "uri", or errors.
func (c *Cache) OpenLocal(checksum, uri string) (*os.File, error) {
source, err := c.GetSource(c.httpClient, uri)
source, err := c.GetSource(c.httpClient, sources.NewSourceURI(uri))
if err != nil {
return nil, errors.WithStack(err)
}
Expand All @@ -93,18 +98,19 @@ func (c *Cache) OpenLocal(checksum, uri string) (*os.File, error) {
//
// If checksum is present it must be the SHA256 hash of the downloaded artifact.
func (c *Cache) Open(b *ui.Task, checksum, uri string, mirrors ...string) (*os.File, error) {
cachePath := c.Path(checksum, uri)
source := sources.NewSourceURI(uri)
cachePath := c.path(checksum, source)
_, err := os.Stat(cachePath)
if err == nil {
b.Tracef("returning cached path %s for %s", cachePath, uri)
b.Tracef("returning cached path %s for %s", cachePath, source)
// TODO: Checksum it again?
return os.Open(cachePath)
} else if !os.IsNotExist(err) {
return nil, errors.WithStack(err)
}

// No local cached copy, download it.
path, _, _, err := c.Download(b, checksum, uri, mirrors...)
path, _, _, err := c.download(b, checksum, source, wrapSources(mirrors))
if err != nil {
return nil, err
}
Expand All @@ -115,13 +121,17 @@ func (c *Cache) Open(b *ui.Task, checksum, uri string, mirrors ...string) (*os.F
//
// If checksum is present it must be the SHA256 hash of the downloaded artifact.
func (c *Cache) Download(b *ui.Task, checksum, uri string, mirrors ...string) (path string, etag string, actualChecksum string, err error) {
uris := append([]string{uri}, mirrors...)
return c.download(b, checksum, sources.NewSourceURI(uri), wrapSources(mirrors))
}

func (c *Cache) download(b *ui.Task, checksum string, uri sources.SourceURI, mirrors []sources.SourceURI) (path string, etag string, actualChecksum string, err error) {
uris := append([]sources.SourceURI{uri}, mirrors...)
var lastError error
attempts := 3
for attempt := 1; attempt <= attempts; attempt++ {
for _, uri := range uris {
defer ui.LogElapsed(b, "Download %s", uri)()
source, err := c.GetSource(c.httpClient, uri)
for _, sourceURI := range uris {
defer ui.LogElapsed(b, "Download %s", sourceURI)()
source, err := c.GetSource(c.httpClient, sourceURI)
if err != nil {
return "", "", "", errors.WithStack(err)
}
Expand All @@ -130,12 +140,12 @@ func (c *Cache) Download(b *ui.Task, checksum, uri string, mirrors ...string) (p
return path, etag, actualChecksum, nil
}
lastError = err
b.Debugf("%s: %s", uri, err)
b.Debugf("%s: %s", sourceURI, err)
}
if lastError == nil {
return "", "", "", errors.Errorf("failed to download from any of %s", strings.Join(uris, ", "))
return "", "", "", errors.Errorf("failed to download from any of %s", joinSources(uris))
}
msg := fmt.Sprintf("Failed to download any of %s on attempt %d/%d: %s", strings.Join(uris, ", "), attempt, attempts, lastError)
msg := fmt.Sprintf("Failed to download any of %s on attempt %d/%d: %s", joinSources(uris), attempt, attempts, lastError)
if attempt < attempts {
msg = "Retrying. " + msg
}
Expand All @@ -153,14 +163,18 @@ func (c *Cache) Download(b *ui.Task, checksum, uri string, mirrors ...string) (p
// ETag fetches the etag from given URI if available.
// Otherwise an empty string is returned
func (c *Cache) ETag(b *ui.Task, uri string, mirrors ...string) (etag string, err error) {
for _, uri := range append([]string{uri}, mirrors...) {
source, err := c.GetSource(c.fastFailHTTPClient, uri)
return c.etag(b, sources.NewSourceURI(uri), wrapSources(mirrors))
}

func (c *Cache) etag(b *ui.Task, uri sources.SourceURI, mirrors []sources.SourceURI) (etag string, err error) {
for _, sourceURI := range append([]sources.SourceURI{uri}, mirrors...) {
source, err := c.GetSource(c.fastFailHTTPClient, sourceURI)
if err != nil {
return "", errors.WithStack(err)
}
result, err := source.ETag(b)
if err != nil {
b.Debugf("%s failed: %s", uri, err)
b.Debugf("%s failed: %s", sourceURI, err)
continue
}
return result, nil
Expand All @@ -170,14 +184,15 @@ func (c *Cache) ETag(b *ui.Task, uri string, mirrors ...string) (etag string, er

// IsCached returns true if the URI is cached.
func (c *Cache) IsCached(checksum, uri string) bool {
_, err := os.Stat(c.Path(checksum, uri))
_, err := os.Stat(c.path(checksum, sources.NewSourceURI(uri)))
return err == nil
}

// Evict a file from the cache.
func (c *Cache) Evict(b *ui.Task, checksum, uri string) error {
b.SubTask("remove").Debugf("rm -rf %s", c.Path(checksum, uri))
err := os.RemoveAll(c.Path(checksum, uri))
path := c.path(checksum, sources.NewSourceURI(uri))
b.SubTask("remove").Debugf("rm -rf %s", path)
err := os.RemoveAll(path)
if err != nil && !os.IsNotExist(err) {
return errors.WithStack(err)
}
Expand All @@ -191,19 +206,39 @@ func (c *Cache) Clean() error {

// Path to cached object.
func (c *Cache) Path(checksum, uri string) string {
base := BasePath(checksum, uri)
return c.path(checksum, sources.NewSourceURI(uri))
}

func (c *Cache) path(checksum string, uri sources.SourceURI) string {
base := basePath(checksum, uri)
return filepath.Join(c.root, base)
}

func wrapSources(raw []string) []sources.SourceURI {
wrapped := make([]sources.SourceURI, len(raw))
for i, uri := range raw {
wrapped[i] = sources.NewSourceURI(uri)
}
return wrapped
}

func joinSources(uris []sources.SourceURI) string {
redacted := make([]string, len(uris))
for i, uri := range uris {
redacted[i] = uri.String()
}
return strings.Join(redacted, ", ")
}

// UnavailableError returns 101 for the exit code.
type UnavailableError struct {
URI string
URI sources.SourceURI
Err error
}

// Error returns the error string for the unavailable error
func (e *UnavailableError) Error() string {
msg := e.URI + " is unavailable"
msg := e.URI.String() + " is unavailable"
if e.Err != nil {
return fmt.Sprintf("%s: %v", msg, e.Err)
}
Expand Down
8 changes: 5 additions & 3 deletions cache/cachew.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,13 +3,15 @@ package cache
import (
"net/http"
"net/url"

"github.com/cashapp/hermit/sources"
)

// CachewSourceSelector wraps another PackageSourceSelector to redirect HTTP/HTTPS
// downloads through a Cachew proxy server https://github.com/block/cachew.
func CachewSourceSelector(getSource PackageSourceSelector, cachewURL string) PackageSourceSelector {
return func(client *http.Client, uri string) (PackageSource, error) {
u, err := url.Parse(uri)
return func(client *http.Client, uri sources.SourceURI) (PackageSource, error) {
u, err := url.Parse(uri.Get())
if err != nil {
return getSource(client, uri)
}
Expand All @@ -25,6 +27,6 @@ func CachewSourceSelector(getSource PackageSourceSelector, cachewURL string) Pac
rewrittenURI += "?" + u.RawQuery
}

return HTTPSource(client, rewrittenURI), nil
return HTTPSource(client, sources.NewSourceURI(rewrittenURI)), nil
}
}
12 changes: 7 additions & 5 deletions cache/cachew_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,11 +3,13 @@ package cache
import (
"net/http"
"testing"

"github.com/cashapp/hermit/sources"
)

func TestCachewSourceSelector(t *testing.T) {
// Mock base selector that always returns a file source
baseSelector := func(client *http.Client, uri string) (PackageSource, error) {
baseSelector := func(client *http.Client, uri sources.SourceURI) (PackageSource, error) {
return &fileSource{path: "/tmp/test"}, nil
}

Expand Down Expand Up @@ -54,7 +56,7 @@ func TestCachewSourceSelector(t *testing.T) {

for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
source, err := selector(nil, tt.inputURI)
source, err := selector(nil, sources.NewSourceURI(tt.inputURI))
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
Expand All @@ -64,7 +66,7 @@ func TestCachewSourceSelector(t *testing.T) {
if !ok {
t.Fatalf("expected httpSource, got %T", source)
}
if httpSrc.url != tt.expectedURI {
if httpSrc.url.Get() != tt.expectedURI {
t.Errorf("expected URL %s, got %s", tt.expectedURI, httpSrc.url)
}
} else {
Expand All @@ -82,14 +84,14 @@ func TestCachewSourceSelector(t *testing.T) {
}

func TestCachewSourceSelectorInvalidURL(t *testing.T) {
baseSelector := func(client *http.Client, uri string) (PackageSource, error) {
baseSelector := func(client *http.Client, uri sources.SourceURI) (PackageSource, error) {
return &fileSource{path: "/tmp/fallback"}, nil
}

selector := CachewSourceSelector(baseSelector, "https://cachew.example.com")

// Test invalid URL - should fall back to base selector
source, err := selector(nil, "://invalid-url")
source, err := selector(nil, sources.NewSourceURI("://invalid-url"))
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
Expand Down
32 changes: 17 additions & 15 deletions cache/git.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,21 +6,23 @@ import (
"strings"

"github.com/cashapp/hermit/errors"
"github.com/cashapp/hermit/internal/redact"
"github.com/cashapp/hermit/sources"
"github.com/cashapp/hermit/ui"
"github.com/cashapp/hermit/util"
)

type gitSource struct {
URL string
URL sources.SourceURI
}

func (s *gitSource) OpenLocal(c *Cache, checksum string) (*os.File, error) {
f, err := os.Open(c.Path(checksum, s.URL))
f, err := os.Open(c.path(checksum, s.URL))
return f, errors.WithStack(err)
}

func (s *gitSource) Download(b *ui.Task, cache *Cache, checksum string) (string, string, string, error) {
base := BasePath(checksum, s.URL)
base := basePath(checksum, s.URL)
checkoutDir := filepath.Join(cache.root, base)
repo, tag, err := parseGitURL(s.URL)
if err != nil {
Expand All @@ -30,8 +32,8 @@ func (s *gitSource) Download(b *ui.Task, cache *Cache, checksum string) (string,
if tag != "" {
args = append(args, "--branch="+tag)
}
args = append(args, "--", repo, checkoutDir)
err = util.RunSystemInDir(b, cache.root, args...)
args = append(args, "--")
err = util.RunSystemInDirWithSource(b, cache.root, args, repo, checkoutDir)
if err != nil {
return "", "", "", errors.WithStack(err)
}
Expand All @@ -53,14 +55,14 @@ func (s *gitSource) ETag(b *ui.Task) (etag string, err error) {
if tag == "" {
tag = "HEAD"
}
bts, err := util.CaptureSystem(b, util.GitArgs("ls-remote", "--", repo, tag)...)
bts, err := util.CaptureSystemWithSource(b, util.GitArgs("ls-remote", "--"), repo, tag)
if err != nil {
return "", errors.Wrap(err, s.URL)
return "", errors.Wrap(err, s.URL.String())
}
str := string(bts)
parts := strings.Split(str, "\t")
if len(parts) != 2 {
return "", errors.Errorf("invalid HEAD: %s", str)
return "", errors.Errorf("invalid HEAD: %s", redact.Credentials(str))
}

return parts[0], nil
Expand All @@ -74,24 +76,24 @@ func (s *gitSource) Validate() error {
if tag == "" {
tag = "HEAD"
}
args := util.GitArgs("ls-remote", "--", repo, tag)
cmd, err := util.SystemCommand(args...)
args := util.GitArgs("ls-remote", "--")
cmd, err := util.SystemCommandWithSource(args, repo, tag)
if err != nil {
return errors.WithStack(err)
}
out, err := cmd.CombinedOutput()
if err != nil {
return errors.Wrapf(err, "error getting remote HEAD: %s", string(out))
return errors.Wrapf(err, "error getting remote HEAD: %s", redact.Credentials(string(out)))
}
return nil
}

func parseGitURL(source string) (repo, tag string, err error) {
parts := strings.SplitN(source, "#", 2)
repo = parts[0]
func parseGitURL(source sources.SourceURI) (repo sources.SourceURI, tag string, err error) {
parts := strings.SplitN(source.Get(), "#", 2)
repo = sources.NewSourceURI(parts[0])

if err := util.ValidateGitURL(repo); err != nil {
return "", "", errors.WithStack(err)
return sources.SourceURI{}, "", errors.WithStack(err)
}

if len(parts) > 1 {
Expand Down
Loading