diff --git a/go/cli/mcap/utils/readers/s3.go b/go/cli/mcap/utils/readers/s3.go index 0bf585ba0..159774027 100644 --- a/go/cli/mcap/utils/readers/s3.go +++ b/go/cli/mcap/utils/readers/s3.go @@ -4,6 +4,8 @@ import ( "context" "fmt" "io" + "os" + "strings" "github.com/aws/aws-sdk-go-v2/aws" "github.com/aws/aws-sdk-go-v2/config" @@ -15,8 +17,15 @@ func init() { RegisterReader("s3", newS3Reader) } +func forcePathStyle() bool { + value := strings.TrimSpace(strings.ToLower(os.Getenv("MCAP_S3_FORCE_PATH_STYLE"))) + return value == "1" || value == "true" || value == "yes" +} + // Factory for S3 readers (called by registry). func newS3Reader(ctx context.Context, bucket, path string) (func() error, io.ReadSeekCloser, error) { + pathStyle := forcePathStyle() + // Try anonymous first cfg, err := config.LoadDefaultConfig(ctx, config.WithCredentialsProvider(aws.AnonymousCredentials{}), @@ -25,7 +34,12 @@ func newS3Reader(ctx context.Context, bucket, path string) (func() error, io.Rea return func() error { return nil }, nil, fmt.Errorf("failed to load AWS config: %w", err) } - client := s3.NewFromConfig(cfg) + client := s3.NewFromConfig(cfg, func(o *s3.Options) { + if pathStyle { + o.UsePathStyle = true + } + }) + rs, err := NewS3ReadSeekCloser(ctx, client, bucket, path) if err == nil { return rs.Close, rs, nil @@ -37,7 +51,12 @@ func newS3Reader(ctx context.Context, bucket, path string) (func() error, io.Rea return func() error { return nil }, nil, fmt.Errorf("failed to load authenticated AWS config: %w", err) } - client = s3.NewFromConfig(cfg) + client = s3.NewFromConfig(cfg, func(o *s3.Options) { + if pathStyle { + o.UsePathStyle = true + } + }) + rs, err = NewS3ReadSeekCloser(ctx, client, bucket, path) if err != nil { return func() error { return nil }, nil, fmt.Errorf("failed to create S3 reader: %w", err) diff --git a/website/docs/guides/cli.md b/website/docs/guides/cli.md index a1cac1f6f..9bffb0ae5 100644 --- a/website/docs/guides/cli.md +++ b/website/docs/guides/cli.md @@ -170,6 +170,23 @@ When reading from S3 you must specify the region of the bucket: AWS_REGION=eu-north-1 mcap info s3://my-public-bucket/demo.mcap ``` +Some S3-compatible storage systems require path-style addressing instead of the default virtual-host style. To enable this behavior, set the following environment variable: + +```bash +MCAP_S3_FORCE_PATH_STYLE=true +``` + +Example: + +```bash +MCAP_S3_FORCE_PATH_STYLE=true \ +AWS_REGION=eu-north-1 \ +AWS_ENDPOINT_URL_S3=https://s3.example.com:9000 \ +AWS_ACCESS_KEY_ID=... \ +AWS_SECRET_ACCESS_KEY=... \ +mcap info s3://my-bucket/demo.mcap +``` + Remote reads will use the index at the end of the file to minimize latency and data transfer. ### File Diagnostics