diff --git a/docs/examples/ecs/ec2/terraform/infrastructure/casting.yaml.lock b/docs/examples/ecs/ec2/terraform/infrastructure/casting.yaml.lock new file mode 100644 index 00000000..b0938a7e --- /dev/null +++ b/docs/examples/ecs/ec2/terraform/infrastructure/casting.yaml.lock @@ -0,0 +1,77 @@ +apiVersion: v1alpha1 +kind: Infrastructure +metadata: + name: foundry +spec: + deployment: + flavor: terraform + mode: ec2 + platform: ecs + patches: + - operations: + - op: replace + path: /variable/aws_region/default + value: us-east-1 + target: infrastructure/variables.tf.json + type: jsonpatch + resource: + spec: + cluster: {} + config: + data: + resource.yaml: | + networking: + networkCIDR: 10.0.0.0/16 + subnets: + private-a: + type: private + zone: us-east-1a + cidr: 10.0.0.0/19 + public-a: + type: public + zone: us-east-1a + cidr: 10.0.96.0/22 + instanceGroups: + persistent: + minSize: 3 + maxSize: 3 + machineType: t3.medium + dataVolume: + size: 5 + ephemeral: + machineType: t3.small + status: + config: + data: + resource.yaml: | + instanceGroups: + ephemeral: + machineType: t3.small + maxSize: 1 + minSize: 1 + rootVolume: + size: 30 + type: gp3 + storage: ephemeral + persistent: + dataVolume: + size: 5 + type: gp3 + machineType: t3.medium + maxSize: 3 + minSize: 3 + rootVolume: + size: 30 + type: gp3 + storage: persistent + networking: + networkCIDR: 10.0.0.0/16 + subnets: + private-a: + cidr: 10.0.0.0/19 + type: private + zone: us-east-1a + public-a: + cidr: 10.0.96.0/22 + type: public + zone: us-east-1a diff --git a/docs/examples/ecs/ec2/terraform/infrastructure/infra.yaml b/docs/examples/ecs/ec2/terraform/infrastructure/infra.yaml new file mode 100644 index 00000000..81f6baff --- /dev/null +++ b/docs/examples/ecs/ec2/terraform/infrastructure/infra.yaml @@ -0,0 +1,41 @@ +apiVersion: v1alpha1 +kind: Infrastructure +metadata: + name: foundry +spec: + deployment: + platform: ecs + mode: ec2 + flavor: terraform + resource: + spec: + config: + data: + resource.yaml: | + networking: + networkCIDR: 10.0.0.0/16 + subnets: + private-a: + type: private + zone: us-east-1a + cidr: 10.0.0.0/19 + public-a: + type: public + zone: us-east-1a + cidr: 10.0.96.0/22 + instanceGroups: + persistent: + minSize: 3 + maxSize: 3 + machineType: t3.medium + dataVolume: + size: 5 + ephemeral: + machineType: t3.small + patches: + - target: "infrastructure/variables.tf.json" + type: jsonpatch + operations: + - op: replace + path: /variable/aws_region/default + value: us-east-1 diff --git a/internal/casting/infrastructure/ecsec2terraformcasting/casting.go b/internal/casting/infrastructure/ecsec2terraformcasting/casting.go new file mode 100644 index 00000000..b7c2cdca --- /dev/null +++ b/internal/casting/infrastructure/ecsec2terraformcasting/casting.go @@ -0,0 +1,140 @@ +package ecsec2terraformcasting + +import ( + "bytes" + "context" + "log/slog" + "path/filepath" + + "github.com/signoz/foundry/api/v1alpha1/infrastructure" + "github.com/signoz/foundry/internal/contract" + ecscontract "github.com/signoz/foundry/internal/contract/aws/ecs" + "github.com/signoz/foundry/internal/domain" + foundryerrors "github.com/signoz/foundry/internal/errors" + infrastructuremolding "github.com/signoz/foundry/internal/molding/infrastructure" + "github.com/signoz/foundry/internal/molding/infrastructure/resourcemolding" + "github.com/signoz/foundry/internal/pourer" + "github.com/signoz/foundry/internal/tooler" + "github.com/signoz/foundry/internal/tooler/terraformtooler" +) + +type cloudInitData struct { + Cluster string + Selector map[string]string + DataVolume bool +} + +type ecsEc2TerraformCasting struct { + logger *slog.Logger +} + +func New(logger *slog.Logger) *ecsEc2TerraformCasting { + return &ecsEc2TerraformCasting{logger: logger} +} + +func (c *ecsEc2TerraformCasting) Enricher(ctx context.Context, config *infrastructure.Casting) (infrastructuremolding.MoldingEnricher, error) { + return newEcsEc2TerraformMoldingEnricher(), nil +} + +func (c *ecsEc2TerraformCasting) Forge(ctx context.Context, config infrastructure.Casting, p *pourer.Pourer) error { + data, err := newResources(config) + if err != nil { + return err + } + + items := []struct { + template *domain.Template + path string + }{ + {versionsTFTemplate, "versions.tf.json"}, + {backendTFTemplate, "backend.tf.json"}, + {providersTFTemplate, "providers.tf.json"}, + {mainTFTemplate, "main.tf.json"}, + {variablesTFTemplate, "variables.tf.json"}, + {outputsTFTemplate, "outputs.tf.json"}, + } + + for _, item := range items { + buf := bytes.NewBuffer(nil) + if err := item.template.Execute(buf, data); err != nil { + return foundryerrors.Wrapf(err, foundryerrors.TypeInternal, "failed to execute %s template", item.path) + } + + p.AddJSON(buf.Bytes(), item.path) + } + + // Blobs stay byte-exact, preserving the #cloud-config header. + boots := map[string]cloudInitData{} + + for key, group := range data.Pinned { + boots[key] = cloudInitData{ + Cluster: data.Cluster.Name, + Selector: group.Selector, + DataVolume: group.Storage.RequiresDataVolume(), + } + } + + for key, group := range data.Pools { + boots[key] = cloudInitData{ + Cluster: data.Cluster.Name, + Selector: group.Selector, + DataVolume: group.Storage.RequiresDataVolume(), + } + } + + for key, boot := range boots { + buf := bytes.NewBuffer(nil) + if err := cloudInitTemplate.Execute(buf, boot); err != nil { + return foundryerrors.Wrapf(err, foundryerrors.TypeInternal, "failed to execute cloud-init template") + } + + p.AddBlob(buf.Bytes(), "cloud-init", key+".yaml") + } + + return nil +} + +func (c *ecsEc2TerraformCasting) Cast(ctx context.Context, config infrastructure.Casting, outputPath string, p *pourer.Pourer, toolers []tooler.Tooler) error { + terraform, err := terraformtooler.Lookup(toolers) + if err != nil { + return err + } + + return terraform.Apply(ctx, terraformtooler.Release{ + Release: domain.Release{Name: config.Metadata.Name, Owner: config.Labels()}, + Root: filepath.Join(outputPath, p.Dir()), + }) +} + +// Melt destroys the substrate, and the volumes it holds go with it. +func (c *ecsEc2TerraformCasting) Melt(ctx context.Context, config infrastructure.Casting, outputPath string, p *pourer.Pourer, toolers []tooler.Tooler) error { + terraform, err := terraformtooler.Lookup(toolers) + if err != nil { + return err + } + + return terraform.Destroy(ctx, terraformtooler.Release{ + Release: domain.Release{Name: config.Metadata.Name, Owner: config.Labels()}, + Root: filepath.Join(outputPath, p.Dir()), + }) +} + +func newResources(config infrastructure.Casting) (*ecscontract.Resources, error) { + doc := config.Spec.Resource.Status.Config.Data[resourcemolding.ResourceConfigName] + + if doc == "" { + return nil, foundryerrors.Newf(foundryerrors.TypeInternal, "resource config %q is missing from the resource status", resourcemolding.ResourceConfigName) + } + + declaration := &infrastructure.ResourceConfig{} + if err := domain.UnmarshalYAML([]byte(doc), declaration); err != nil { + return nil, foundryerrors.Wrapf(err, foundryerrors.TypeInternal, "failed to unmarshal resource config") + } + + substrate, err := contract.NewSubstrate(config.Metadata.Name) + if err != nil { + return nil, foundryerrors.Wrapf(err, foundryerrors.TypeInvalidInput, "failed to resolve the substrate being provisioned") + } + + return ecscontract.Derive(substrate, declaration, config.Labels()) +} diff --git a/internal/casting/infrastructure/ecsec2terraformcasting/embed.go b/internal/casting/infrastructure/ecsec2terraformcasting/embed.go new file mode 100644 index 00000000..c332db4d --- /dev/null +++ b/internal/casting/infrastructure/ecsec2terraformcasting/embed.go @@ -0,0 +1,20 @@ +package ecsec2terraformcasting + +import ( + "embed" + + "github.com/signoz/foundry/internal/domain" +) + +//go:embed templates/*.gotmpl +var templates embed.FS + +var ( + versionsTFTemplate *domain.Template = domain.MustNewTemplateFromFS(templates, "templates/versions.tf.json.gotmpl", domain.FormatJSON) + backendTFTemplate *domain.Template = domain.MustNewTemplateFromFS(templates, "templates/backend.tf.json.gotmpl", domain.FormatJSON) + providersTFTemplate *domain.Template = domain.MustNewTemplateFromFS(templates, "templates/providers.tf.json.gotmpl", domain.FormatJSON) + mainTFTemplate *domain.Template = domain.MustNewTemplateFromFS(templates, "templates/main.tf.json.gotmpl", domain.FormatJSON) + variablesTFTemplate *domain.Template = domain.MustNewTemplateFromFS(templates, "templates/variables.tf.json.gotmpl", domain.FormatJSON) + outputsTFTemplate *domain.Template = domain.MustNewTemplateFromFS(templates, "templates/outputs.tf.json.gotmpl", domain.FormatJSON) + cloudInitTemplate *domain.Template = domain.MustNewTemplateFromFS(templates, "templates/cloudinit.yaml.gotmpl", domain.FormatText) +) diff --git a/internal/casting/infrastructure/ecsec2terraformcasting/embed_test.go b/internal/casting/infrastructure/ecsec2terraformcasting/embed_test.go new file mode 100644 index 00000000..795746bf --- /dev/null +++ b/internal/casting/infrastructure/ecsec2terraformcasting/embed_test.go @@ -0,0 +1,130 @@ +package ecsec2terraformcasting + +import ( + "context" + "encoding/json" + "log/slog" + "maps" + "slices" + "testing" + + "github.com/signoz/foundry/api/v1alpha1" + "github.com/signoz/foundry/api/v1alpha1/infrastructure" + "github.com/signoz/foundry/internal/domain" + "github.com/signoz/foundry/internal/molding/infrastructure/resourcemolding" + "github.com/stretchr/testify/assert" +) + +// A zone is per-account, so the fixture must state one. +const subnets = `networking: + subnets: + private-a: {type: private, zone: us-east-1a, cidr: 10.0.0.0/19} + public-a: {type: public, zone: us-east-1a, cidr: 10.0.96.0/22} +` + +func moldedCasting(t *testing.T) *infrastructure.Casting { + t.Helper() + + config := infrastructure.Default() + config.Spec.Resource.Spec.Config.Set(resourcemolding.ResourceConfigName, []byte(subnets)) + + logger := slog.New(slog.DiscardHandler) + assert.NoError(t, newEcsEc2TerraformMoldingEnricher().EnrichStatus(context.Background(), v1alpha1.MoldingKindResource, config)) + assert.NoError(t, resourcemolding.New(logger).MoldV1Alpha1(context.Background(), config)) + + return config +} + +func TestTemplates_RenderValidJSON(t *testing.T) { + config := moldedCasting(t) + + data, err := newResources(*config) + assert.NoError(t, err) + + tests := []struct { + name string + template *domain.Template + }{ + {name: "ProvidersTemplate_RendersValidJSON", template: providersTFTemplate}, + {name: "MainTemplate_RendersValidJSON", template: mainTFTemplate}, + {name: "VariablesTemplate_RendersValidJSON", template: variablesTFTemplate}, + {name: "OutputsTemplate_RendersValidJSON", template: outputsTFTemplate}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + material, err := tt.template.Render(data, "out.tf.json") + assert.NoError(t, err) + assert.NotEmpty(t, material.FmtContents()) + }) + } +} + +func TestMainTemplate_PinsPersistentAndPoolsEphemeral(t *testing.T) { + config := moldedCasting(t) + + data, err := newResources(*config) + assert.NoError(t, err) + + material, err := mainTFTemplate.Render(data, "out.tf.json") + assert.NoError(t, err) + + contents := string(material.FmtContents()) + assert.Contains(t, contents, `"persistent-0"`) + assert.Contains(t, contents, `"persistent-2"`) + assert.Contains(t, contents, `"aws_ebs_volume"`) + assert.Contains(t, contents, `"aws_volume_attachment"`) + assert.Contains(t, contents, `"aws_autoscaling_group"`) + assert.Contains(t, contents, "cloud-init/persistent.yaml") + assert.Contains(t, contents, "cloud-init/ephemeral.yaml") + assert.NotContains(t, contents, "aws_launch_template.persistent") + assert.NotContains(t, contents, "aws_instance.ephemeral") +} + +// The addresses below are the patch surface: renaming one breaks every stored +// spec.patches entry and, for resource labels, live state addresses. +func TestMainTemplate_FreezesThePatchSurface(t *testing.T) { + config := moldedCasting(t) + + data, err := newResources(*config) + assert.NoError(t, err) + + material, err := mainTFTemplate.Render(data, "out.tf.json") + assert.NoError(t, err) + + main := map[string]any{} + assert.NoError(t, json.Unmarshal(material.FmtContents(), &main)) + + resources, _ := main["resource"].(map[string]any) + + expected := map[string][]string{ + "aws_vpc": {"main"}, + "aws_subnet": {"private-a", "public-a"}, + "aws_internet_gateway": {"main"}, + "aws_eip": {"private-a"}, + "aws_nat_gateway": {"private-a"}, + "aws_route_table": {"private-a", "public-a"}, + "aws_route": {"private-a", "public-a"}, + "aws_route_table_association": {"private-a", "public-a"}, + "terraform_data": {"persistent"}, + "aws_instance": {"persistent-0", "persistent-1", "persistent-2"}, + "aws_ebs_volume": {"persistent-0", "persistent-1", "persistent-2"}, + "aws_volume_attachment": {"persistent-0", "persistent-1", "persistent-2"}, + "aws_launch_template": {"ephemeral"}, + "aws_autoscaling_group": {"ephemeral"}, + "aws_security_group": {"tasks"}, + "aws_vpc_security_group_ingress_rule": {"intra_cluster"}, + "aws_vpc_security_group_egress_rule": {"all_outbound"}, + "aws_iam_role": {"node"}, + "aws_iam_instance_profile": {"node"}, + "aws_iam_role_policy_attachment": {"node"}, + "aws_ecs_cluster": {"main"}, + } + + assert.ElementsMatch(t, slices.Sorted(maps.Keys(expected)), slices.Sorted(maps.Keys(resources))) + + for resourceType, labels := range expected { + instances, _ := resources[resourceType].(map[string]any) + assert.ElementsMatch(t, labels, slices.Sorted(maps.Keys(instances)), "labels of %s", resourceType) + } +} diff --git a/internal/casting/infrastructure/ecsec2terraformcasting/enricher.go b/internal/casting/infrastructure/ecsec2terraformcasting/enricher.go new file mode 100644 index 00000000..800decba --- /dev/null +++ b/internal/casting/infrastructure/ecsec2terraformcasting/enricher.go @@ -0,0 +1,56 @@ +package ecsec2terraformcasting + +import ( + "context" + + "github.com/signoz/foundry/api/v1alpha1" + "github.com/signoz/foundry/api/v1alpha1/infrastructure" + "github.com/signoz/foundry/internal/domain" + foundryerrors "github.com/signoz/foundry/internal/errors" + infrastructuremolding "github.com/signoz/foundry/internal/molding/infrastructure" + "github.com/signoz/foundry/internal/molding/infrastructure/resourcemolding" +) + +// Neither machine type is burstable: a store that throttles under sustained +// ingest reads as an outage. +const ( + machineTypePersistent = "m5.large" + machineTypeEphemeral = "c5.large" + volumeType = "gp3" +) + +var _ infrastructuremolding.MoldingEnricher = (*ecsEc2TerraformMoldingEnricher)(nil) + +type ecsEc2TerraformMoldingEnricher struct{} + +func newEcsEc2TerraformMoldingEnricher() *ecsEc2TerraformMoldingEnricher { + return &ecsEc2TerraformMoldingEnricher{} +} + +// EnrichStatus omits subnets: an availability zone is per-account. +func (e *ecsEc2TerraformMoldingEnricher) EnrichStatus(ctx context.Context, kind v1alpha1.MoldingKind, config *infrastructure.Casting) error { + if kind != v1alpha1.MoldingKindResource { + return nil + } + + groups := map[string]infrastructure.ResourceConfigInstanceGroup{ + resourcemolding.GroupPersistent: { + MachineType: machineTypePersistent, + RootVolume: infrastructure.ResourceConfigVolume{Type: volumeType}, + DataVolume: &infrastructure.ResourceConfigVolume{Type: volumeType}, + }, + resourcemolding.GroupEphemeral: { + MachineType: machineTypeEphemeral, + RootVolume: infrastructure.ResourceConfigVolume{Type: volumeType}, + }, + } + + contribution, err := domain.MarshalYAML(&infrastructure.ResourceConfig{InstanceGroups: groups}) + if err != nil { + return foundryerrors.Wrapf(err, foundryerrors.TypeInternal, "failed to marshal resource config contribution") + } + + config.Spec.Resource.Status.Config.Set(resourcemolding.ResourceConfigName, contribution) + + return nil +} diff --git a/internal/casting/infrastructure/ecsec2terraformcasting/templates/backend.tf.json.gotmpl b/internal/casting/infrastructure/ecsec2terraformcasting/templates/backend.tf.json.gotmpl new file mode 100644 index 00000000..fff9005b --- /dev/null +++ b/internal/casting/infrastructure/ecsec2terraformcasting/templates/backend.tf.json.gotmpl @@ -0,0 +1,9 @@ +{ + "terraform": { + "backend": { + "local": { + "path": "terraform.tfstate" + } + } + } +} diff --git a/internal/casting/infrastructure/ecsec2terraformcasting/templates/cloudinit.yaml.gotmpl b/internal/casting/infrastructure/ecsec2terraformcasting/templates/cloudinit.yaml.gotmpl new file mode 100644 index 00000000..f5bc7d37 --- /dev/null +++ b/internal/casting/infrastructure/ecsec2terraformcasting/templates/cloudinit.yaml.gotmpl @@ -0,0 +1,33 @@ +#cloud-config +write_files: + - path: /etc/ecs/ecs.config + content: | + ECS_CLUSTER={{ .Cluster }} + {{- /* A container instance advertises the group's own selector. A + consuming casting's placement constraint filters on these. */}} + ECS_INSTANCE_ATTRIBUTES={{ toJson .Selector }} +{{- if .DataVolume }} + # The agent must never register a node whose data volume is not mounted: + # tasks would bind-mount onto the root disk and state would silently land + # on a disk that dies with the instance. + - path: /etc/systemd/system/ecs.service.d/10-foundry-data.conf + content: | + [Unit] + RequiresMountsFor=/var/lib/foundry +# Terraform attaches the data volume after boot begins, and cloud-init's +# fs_setup/mounts run once, first boot only, silently skipping devices that do +# not exist yet (canonical/cloud-init#3386). bootcmd runs every boot, before +# anything downstream of cloud-init (the agent included), so it waits for the +# attachment, formats only a device with no filesystem signature, and mounts. +bootcmd: + - 'for i in $(seq 1 120); do test -b /dev/xvdf && break; sleep 5; done' + - 'test -b /dev/xvdf || { echo "foundry: data volume /dev/xvdf never attached" >&2; exit 1; }' + - 'blkid /dev/xvdf >/dev/null || mkfs.ext4 -L foundry-data /dev/xvdf' + - 'mkdir -p /var/lib/foundry' + - 'mountpoint -q /var/lib/foundry || mount /dev/xvdf /var/lib/foundry' +# The fstab entry mounts at local-fs on every later boot and is what gives the +# RequiresMountsFor gate a real mount unit to require. The device exists by the +# time this module runs, because bootcmd waited. +mounts: + - [/dev/xvdf, /var/lib/foundry, ext4, "defaults,nofail,x-systemd.device-timeout=10min", "0", "2"] +{{- end }} diff --git a/internal/casting/infrastructure/ecsec2terraformcasting/templates/main.tf.json.gotmpl b/internal/casting/infrastructure/ecsec2terraformcasting/templates/main.tf.json.gotmpl new file mode 100644 index 00000000..a4bff3bd --- /dev/null +++ b/internal/casting/infrastructure/ecsec2terraformcasting/templates/main.tf.json.gotmpl @@ -0,0 +1,292 @@ +{{- /* Names and tags are interpolated from the derived data, never assembled + here. A literal spelled twice is a filter that matches nothing. */}} +{{- $subnets := dict -}} +{{- range $key, $subnet := $.Network.Subnets }}{{ if not $subnet.ID }}{{ $_ := set $subnets $key $subnet }}{{ end }}{{ end -}} +{{- $gateways := dict -}} +{{- range $key, $gateway := $.Network.NATGateways }}{{ if $gateway.Name }}{{ $_ := set $gateways $key $gateway }}{{ end }}{{ end -}} +{ + "locals": { + "vpc_id": "{{ if $.Network.VPC.ID }}{{ $.Network.VPC.ID }}{{ else }}${aws_vpc.main.id}{{ end }}", + "subnet_ids": { + {{- $first := true }}{{ range $key, $subnet := $.Network.Subnets }}{{ if not $first }},{{ end }}{{ $first = false }} + "{{ $key }}": "{{ if $subnet.ID }}{{ $subnet.ID }}{{ else }}${aws_subnet.{{ $key }}.id}{{ end }}" + {{- end }} + } + }, + "data": { + "aws_ssm_parameter": { + "ecs_ami": { + "name": "/aws/service/ecs/optimized-ami/amazon-linux-2023/recommended/image_id" + } + } + }, + "resource": { + {{- if not $.Network.VPC.ID }} + "aws_vpc": { + "main": { + "cidr_block": "${var.network_cidr}", + "enable_dns_hostnames": true, + "enable_dns_support": true, + "tags": {{ toJson $.Network.VPC.Tags }} + } + }, + {{- end }} + {{- if $subnets }} + "aws_subnet": { + {{- $first := true }}{{ range $key, $subnet := $subnets }}{{ if not $first }},{{ end }}{{ $first = false }} + "{{ $key }}": { + "vpc_id": "${local.vpc_id}", + "cidr_block": "${var.subnet_{{ identifier $key }}_cidr}", + "availability_zone": "${var.subnet_{{ identifier $key }}_zone}", + {{- if $subnet.Public }} + "map_public_ip_on_launch": true, + {{- end }} + "tags": {{ toJson $subnet.Tags }} + } + {{- end }} + }, + {{- end }} + {{- if $.Network.InternetGateway.Name }} + "aws_internet_gateway": { + "main": { + "vpc_id": "${local.vpc_id}", + "tags": {{ toJson $.Network.InternetGateway.Tags }} + } + }, + {{- end }} + {{- if $gateways }} + "aws_eip": { + {{- $first := true }}{{ range $key, $gateway := $gateways }}{{ if not $first }},{{ end }}{{ $first = false }} + "{{ $key }}": { + "domain": "vpc", + "tags": {{ toJson $gateway.Address.Tags }} + } + {{- end }} + }, + "aws_nat_gateway": { + {{- $first := true }}{{ range $key, $gateway := $gateways }}{{ if not $first }},{{ end }}{{ $first = false }} + "{{ $key }}": { + "allocation_id": "${aws_eip.{{ $key }}.id}", + "subnet_id": "${local.subnet_ids[\"{{ $gateway.Subnet }}\"]}", + "tags": {{ toJson $gateway.Tags }}, + "depends_on": ["aws_internet_gateway.main"] + } + {{- end }} + }, + {{- end }} + {{- if $.Network.RouteTables }} + {{- /* A table per subnet. A private subnet's default route is its own + zone's gateway. */}} + "aws_route_table": { + {{- $first := true }}{{ range $key, $table := $.Network.RouteTables }}{{ if not $first }},{{ end }}{{ $first = false }} + "{{ $key }}": { + "vpc_id": "${local.vpc_id}", + "tags": {{ toJson $table.Tags }} + } + {{- end }} + }, + "aws_route": { + {{- $first := true }}{{ range $key, $table := $.Network.RouteTables }} + {{- $subnet := index $.Network.Subnets $key }} + {{- if $subnet.Public }}{{ if not $first }},{{ end }}{{ $first = false }} + "{{ $key }}": { + "route_table_id": "${aws_route_table.{{ $key }}.id}", + "destination_cidr_block": "0.0.0.0/0", + "gateway_id": "${aws_internet_gateway.main.id}" + } + {{- else }}{{ $gateway := index $.Network.NATGateways $key }}{{ if or $gateway.Name $gateway.ID }}{{ if not $first }},{{ end }}{{ $first = false }} + "{{ $key }}": { + "route_table_id": "${aws_route_table.{{ $key }}.id}", + "destination_cidr_block": "0.0.0.0/0", + "nat_gateway_id": "{{ if $gateway.ID }}{{ $gateway.ID }}{{ else }}${aws_nat_gateway.{{ $key }}.id}{{ end }}" + } + {{- end }}{{ end }}{{ end }} + }, + "aws_route_table_association": { + {{- $first := true }}{{ range $key, $table := $.Network.RouteTables }}{{ if not $first }},{{ end }}{{ $first = false }} + "{{ $key }}": { + "subnet_id": "${local.subnet_ids[\"{{ $key }}\"]}", + "route_table_id": "${aws_route_table.{{ $key }}.id}" + } + {{- end }} + }, + {{- end }} + {{- if $.Pinned }} + "terraform_data": { + {{- $first := true }}{{ range $key, $group := $.Pinned }}{{ if not $first }},{{ end }}{{ $first = false }} + "{{ $key }}": { + "triggers_replace": "${var.group_{{ identifier $key }}_machine_type}" + } + {{- end }} + }, + "aws_instance": { + {{- $first := true }}{{ range $key, $group := $.Pinned }}{{ range $node := $group.Nodes }}{{ if not $first }},{{ end }}{{ $first = false }} + "{{ $key }}-{{ $node.Ordinal }}": { + "ami": "${data.aws_ssm_parameter.ecs_ami.value}", + "instance_type": "${var.group_{{ identifier $key }}_machine_type}", + "subnet_id": "${local.subnet_ids[\"{{ $node.Subnet }}\"]}", + "vpc_security_group_ids": ["${aws_security_group.tasks.id}"], + "iam_instance_profile": "${aws_iam_instance_profile.node.name}", + "user_data_base64": "${filebase64(\"${path.module}/cloud-init/{{ $key }}.yaml\")}", + {{- /* Boot config runs on first boot only. A change means a new node. */}} + "user_data_replace_on_change": true, + "root_block_device": [ + { + "volume_size": "${var.group_{{ identifier $key }}_root_volume_size}", + "volume_type": "${var.group_{{ identifier $key }}_root_volume_type}", + "delete_on_termination": true + } + ], + "tags": {{ toJson $node.Tags }}, + {{- /* ECS refuses to re-register an instance whose type changed, and + the agent then exits terminally. A type change must REPLACE the + node. The AMI is pinned so SSM rotation does not replace the + fleet as a side effect. */}} + "lifecycle": { + "replace_triggered_by": ["terraform_data.{{ $key }}"], + "ignore_changes": ["ami"] + }, + "depends_on": ["aws_ecs_cluster.main"] + } + {{- end }}{{ end }} + }, + "aws_ebs_volume": { + {{- $first := true }}{{ range $key, $group := $.Pinned }}{{ range $node := $group.Nodes }}{{ if not $first }},{{ end }}{{ $first = false }} + "{{ $key }}-{{ $node.Ordinal }}": { + "availability_zone": "${var.subnet_{{ identifier $node.Subnet }}_zone}", + "size": "${var.group_{{ identifier $key }}_data_volume_size}", + "type": "${var.group_{{ identifier $key }}_data_volume_type}", + "tags": {{ toJson $node.Volume.Tags }} + } + {{- end }}{{ end }} + }, + "aws_volume_attachment": { + {{- $first := true }}{{ range $key, $group := $.Pinned }}{{ range $node := $group.Nodes }}{{ if not $first }},{{ end }}{{ $first = false }} + "{{ $key }}-{{ $node.Ordinal }}": { + "device_name": "/dev/xvdf", + "volume_id": "${aws_ebs_volume.{{ $key }}-{{ $node.Ordinal }}.id}", + "instance_id": "${aws_instance.{{ $key }}-{{ $node.Ordinal }}.id}" + } + {{- end }}{{ end }} + }, + {{- end }} + {{- if $.Pools }} + "aws_launch_template": { + {{- $first := true }}{{ range $key, $group := $.Pools }}{{ if not $first }},{{ end }}{{ $first = false }} + "{{ $key }}": { + "name": "{{ $group.LaunchTemplate.Name }}", + "image_id": "${data.aws_ssm_parameter.ecs_ami.value}", + "instance_type": "${var.group_{{ identifier $key }}_machine_type}", + "iam_instance_profile": { + "arn": "${aws_iam_instance_profile.node.arn}" + }, + "vpc_security_group_ids": ["${aws_security_group.tasks.id}"], + "user_data": "${filebase64(\"${path.module}/cloud-init/{{ $key }}.yaml\")}", + "block_device_mappings": [ + { + "device_name": "/dev/xvda", + "ebs": [ + { + "volume_size": "${var.group_{{ identifier $key }}_root_volume_size}", + "volume_type": "${var.group_{{ identifier $key }}_root_volume_type}", + "delete_on_termination": true + } + ] + } + ], + "tag_specifications": [ + { + "resource_type": "instance", + "tags": {{ toJson $group.LaunchTemplate.Tags }} + } + ], + "tags": {{ toJson $group.LaunchTemplate.Tags }} + } + {{- end }} + }, + "aws_autoscaling_group": { + {{- $first := true }}{{ range $key, $group := $.Pools }}{{ if not $first }},{{ end }}{{ $first = false }} + "{{ $key }}": { + "name": "{{ $group.AutoscalingGroup.Name }}", + "desired_capacity": "${var.group_{{ identifier $key }}_min_size}", + "min_size": "${var.group_{{ identifier $key }}_min_size}", + "max_size": "${var.group_{{ identifier $key }}_max_size}", + "vpc_zone_identifier": [{{ range $i, $subnet := $group.Subnets }}{{ if $i }}, {{ end }}"${local.subnet_ids[\"{{ $subnet }}\"]}"{{ end }}], + "launch_template": [ + { + "id": "${aws_launch_template.{{ $key }}.id}", + "version": "$Latest" + } + ], + "tag": [ + {{- $first := true }}{{ range $tag, $value := $group.AutoscalingGroup.Tags }}{{ if not $first }},{{ end }}{{ $first = false }} + {"key": "{{ $tag }}", "value": "{{ $value }}", "propagate_at_launch": true} + {{- end }} + ], + "depends_on": ["aws_ecs_cluster.main"] + } + {{- end }} + }, + {{- end }} + "aws_security_group": { + "tasks": { + "name": "{{ $.SecurityGroup.Name }}", + "description": "SigNoz ECS tasks and container instances", + "vpc_id": "${local.vpc_id}", + "tags": {{ toJson $.SecurityGroup.Tags }} + } + }, + "aws_vpc_security_group_ingress_rule": { + "intra_cluster": { + "security_group_id": "${aws_security_group.tasks.id}", + "description": "intra-cluster traffic", + "ip_protocol": "-1", + "referenced_security_group_id": "${aws_security_group.tasks.id}", + "tags": {{ toJson (index $.SecurityGroupRules "intra-cluster").Tags }} + } + }, + {{- /* Egress stays open. Image pulls and OS packages have no stable CIDR. + Narrow it with VPC endpoints. */}} + "aws_vpc_security_group_egress_rule": { + "all_outbound": { + "security_group_id": "${aws_security_group.tasks.id}", + "description": "all outbound", + "ip_protocol": "-1", + "cidr_ipv4": "0.0.0.0/0", + "tags": {{ toJson (index $.SecurityGroupRules "all-outbound").Tags }} + } + }, + {{- /* The node's own credential only. Without the managed policy the ECS + agent cannot register the instance. Task and execution roles belong + to the casting that runs the tasks. */}} + "aws_iam_role": { + "node": { + "name": "{{ $.Roles.node.Name }}", + "assume_role_policy": "${jsonencode({\"Version\" = \"2012-10-17\", \"Statement\" = [{\"Action\" = \"sts:AssumeRole\", \"Effect\" = \"Allow\", \"Principal\" = {\"Service\" = \"ec2.amazonaws.com\"}}]})}", + {{- if $.Declaration.IAM.PermissionsBoundary }} + "permissions_boundary": "{{ $.Declaration.IAM.PermissionsBoundary }}", + {{- end }} + "tags": {{ toJson $.Roles.node.Tags }} + } + }, + "aws_iam_instance_profile": { + "node": { + "name": "{{ $.InstanceProfile.Name }}", + "role": "${aws_iam_role.node.name}", + "tags": {{ toJson $.InstanceProfile.Tags }} + } + }, + "aws_iam_role_policy_attachment": { + "node": { + "policy_arn": "arn:aws:iam::aws:policy/service-role/AmazonEC2ContainerServiceforEC2Role", + "role": "${aws_iam_role.node.name}" + } + }, + "aws_ecs_cluster": { + "main": { + "name": "{{ $.Cluster.Name }}", + "tags": {{ toJson $.Cluster.Tags }} + } + } + } +} diff --git a/internal/casting/infrastructure/ecsec2terraformcasting/templates/outputs.tf.json.gotmpl b/internal/casting/infrastructure/ecsec2terraformcasting/templates/outputs.tf.json.gotmpl new file mode 100644 index 00000000..243bc45d --- /dev/null +++ b/internal/casting/infrastructure/ecsec2terraformcasting/templates/outputs.tf.json.gotmpl @@ -0,0 +1,51 @@ +{{- $private := list -}} +{{- $public := list -}} +{{- range $key, $subnet := $.Network.Subnets }}{{ if $subnet.Public }}{{ $public = append $public $key }}{{ else }}{{ $private = append $private $key }}{{ end }}{{ end -}} +{ + "output": { + "cluster_name": { + "description": "Name of the ECS cluster", + "value": "${aws_ecs_cluster.main.name}" + }, + "cluster_arn": { + "description": "ARN of the ECS cluster", + "value": "${aws_ecs_cluster.main.arn}" + }, + "vpc_id": { + "description": "ID of the VPC", + "value": "${local.vpc_id}" + }, + "private_subnet_ids": { + "description": "IDs of the private subnets, which is where workloads are placed", + "value": [{{ range $i, $key := $private }}{{ if $i }}, {{ end }}"${local.subnet_ids[\"{{ $key }}\"]}"{{ end }}] + }, + "public_subnet_ids": { + "description": "IDs of the public subnets", + "value": [{{ range $i, $key := $public }}{{ if $i }}, {{ end }}"${local.subnet_ids[\"{{ $key }}\"]}"{{ end }}] + }, + "security_group_ids": { + "description": "IDs of the tasks security group", + "value": ["${aws_security_group.tasks.id}"] + }, + "node_role_arn": { + "description": "ARN of the ECS container-instance role", + "value": "${aws_iam_role.node.arn}" + } + {{- range $key, $group := $.Pinned }}, + "instance_group_{{ identifier $key }}_instance_ids": { + "description": "IDs of the pinned {{ $key }} instances, by ordinal", + "value": [{{ range $i, $node := $group.Nodes }}{{ if $i }}, {{ end }}"${aws_instance.{{ $key }}-{{ $node.Ordinal }}.id}"{{ end }}] + }, + "instance_group_{{ identifier $key }}_volume_ids": { + "description": "IDs of the {{ $key }} data volumes, by ordinal", + "value": [{{ range $i, $node := $group.Nodes }}{{ if $i }}, {{ end }}"${aws_ebs_volume.{{ $key }}-{{ $node.Ordinal }}.id}"{{ end }}] + } + {{- end }} + {{- range $key, $group := $.Pools }}, + "instance_group_{{ identifier $key }}_asg_name": { + "description": "Name of the {{ $key }} autoscaling group", + "value": "${aws_autoscaling_group.{{ $key }}.name}" + } + {{- end }} + } +} diff --git a/internal/casting/infrastructure/ecsec2terraformcasting/templates/providers.tf.json.gotmpl b/internal/casting/infrastructure/ecsec2terraformcasting/templates/providers.tf.json.gotmpl new file mode 100644 index 00000000..3d92dd10 --- /dev/null +++ b/internal/casting/infrastructure/ecsec2terraformcasting/templates/providers.tf.json.gotmpl @@ -0,0 +1,12 @@ +{{- /* A volume's claim is stamped by whoever claims it, long after this + applies. Reconciling it reverts a live claim on every apply. */ -}} +{ + "provider": { + "aws": { + "region": "${var.aws_region}", + "ignore_tags": { + "keys": {{ toJson $.IgnoredTags }} + } + } + } +} diff --git a/internal/casting/infrastructure/ecsec2terraformcasting/templates/variables.tf.json.gotmpl b/internal/casting/infrastructure/ecsec2terraformcasting/templates/variables.tf.json.gotmpl new file mode 100644 index 00000000..b6249df7 --- /dev/null +++ b/internal/casting/infrastructure/ecsec2terraformcasting/templates/variables.tf.json.gotmpl @@ -0,0 +1,156 @@ +{{- /* Every declared knob arrives as a variable defaulted to what was + declared. Names and tags are not knobs; a consumer matches on them. */ -}} +{ + "variable": { + "aws_region": { + "description": "AWS region to deploy resources; it must be the region the declared zones belong to", + "type": "string", + "default": "us-east-1", + "nullable": false, + "validation": { + "condition": "${can(regex(\"^[a-z]{2}(-gov)?-[a-z]+-[0-9]$\", var.aws_region))}", + "error_message": "aws_region must be a region identifier such as us-east-1." + } + } + {{- if not $.Network.VPC.ID }}, + "network_cidr": { + "description": "CIDR block for the network", + "type": "string", + "default": "{{ $.Declaration.Networking.NetworkCIDR }}", + "nullable": false, + "validation": { + "condition": "${can(cidrhost(var.network_cidr, 0))}", + "error_message": "network_cidr must be a valid IPv4 CIDR." + } + } + {{- end }} + {{- range $key, $subnet := $.Network.Subnets }}, + "subnet_{{ identifier $key }}_zone": { + "description": "Availability zone the {{ $key }} subnet lives in; a volume can only attach to a machine in its own zone", + "type": "string", + "default": "{{ (index $.Declaration.Networking.Subnets $key).Zone }}", + "nullable": false + } + {{- if not $subnet.ID }}, + "subnet_{{ identifier $key }}_cidr": { + "description": "CIDR block for the {{ $key }} subnet, carved out of the network", + "type": "string", + "default": "{{ (index $.Declaration.Networking.Subnets $key).CIDR }}", + "nullable": false, + "validation": { + "condition": "${can(cidrhost(var.subnet_{{ identifier $key }}_cidr, 0))}", + "error_message": "subnet_{{ identifier $key }}_cidr must be a valid IPv4 CIDR." + } + } + {{- end }} + {{- end }} + {{- range $key, $group := $.Pinned }}{{ $declared := $group.Declared }}, + "group_{{ identifier $key }}_machine_type": { + "description": "Provider machine type for each node in the {{ $key }} group", + "type": "string", + "default": "{{ $declared.MachineType }}", + "nullable": false + }, + {{- /* The root disk carries the AMI. Its floor is the snapshot size. */}} + "group_{{ identifier $key }}_root_volume_size": { + "description": "Root volume size (GB) for each {{ $key }} node; at least the ECS-optimized AMI snapshot size (30)", + "type": "number", + "default": {{ derefInt $declared.RootVolume.Size }}, + "nullable": false, + "validation": { + "condition": "${var.group_{{ identifier $key }}_root_volume_size >= 30}", + "error_message": "group_{{ identifier $key }}_root_volume_size must be at least 30 GB, the ECS-optimized AMI snapshot size." + } + }, + "group_{{ identifier $key }}_root_volume_type": { + "description": "Root volume type for each {{ $key }} node", + "type": "string", + "default": "{{ $declared.RootVolume.Type }}", + "nullable": false + } + {{- if $declared.DataVolume }}, + "group_{{ identifier $key }}_data_volume_size": { + "description": "Data volume size (GB) attached to each {{ $key }} node; it outlives the node", + "type": "number", + "default": {{ derefInt $declared.DataVolume.Size }}, + "nullable": false, + "validation": { + "condition": "${var.group_{{ identifier $key }}_data_volume_size >= 1}", + "error_message": "group_{{ identifier $key }}_data_volume_size must be at least 1 GB." + } + }, + "group_{{ identifier $key }}_data_volume_type": { + "description": "Data volume type for each {{ $key }} node", + "type": "string", + "default": "{{ $declared.DataVolume.Type }}", + "nullable": false + } + {{- end }} + {{- /* A pinned group has no size variable. Each node is its own resource, + and the count changes the plan's shape. */}} + {{- end }} + {{- range $key, $group := $.Pools }}{{ $declared := $group.Declared }}, + "group_{{ identifier $key }}_machine_type": { + "description": "Provider machine type for each node in the {{ $key }} group", + "type": "string", + "default": "{{ $declared.MachineType }}", + "nullable": false + }, + {{- /* The root disk carries the AMI. Its floor is the snapshot size. */}} + "group_{{ identifier $key }}_root_volume_size": { + "description": "Root volume size (GB) for each {{ $key }} node; at least the ECS-optimized AMI snapshot size (30)", + "type": "number", + "default": {{ derefInt $declared.RootVolume.Size }}, + "nullable": false, + "validation": { + "condition": "${var.group_{{ identifier $key }}_root_volume_size >= 30}", + "error_message": "group_{{ identifier $key }}_root_volume_size must be at least 30 GB, the ECS-optimized AMI snapshot size." + } + }, + "group_{{ identifier $key }}_root_volume_type": { + "description": "Root volume type for each {{ $key }} node", + "type": "string", + "default": "{{ $declared.RootVolume.Type }}", + "nullable": false + } + {{- if $declared.DataVolume }}, + "group_{{ identifier $key }}_data_volume_size": { + "description": "Data volume size (GB) attached to each {{ $key }} node; it outlives the node", + "type": "number", + "default": {{ derefInt $declared.DataVolume.Size }}, + "nullable": false, + "validation": { + "condition": "${var.group_{{ identifier $key }}_data_volume_size >= 1}", + "error_message": "group_{{ identifier $key }}_data_volume_size must be at least 1 GB." + } + }, + "group_{{ identifier $key }}_data_volume_type": { + "description": "Data volume type for each {{ $key }} node", + "type": "string", + "default": "{{ $declared.DataVolume.Type }}", + "nullable": false + } + {{- end }}, + "group_{{ identifier $key }}_min_size": { + "description": "Smallest the {{ $key }} group may be", + "type": "number", + "default": {{ derefInt $declared.MinSize }}, + "nullable": false, + "validation": { + "condition": "${var.group_{{ identifier $key }}_min_size >= 0}", + "error_message": "group_{{ identifier $key }}_min_size cannot be negative." + } + }, + "group_{{ identifier $key }}_max_size": { + "description": "Largest the {{ $key }} group may grow to", + "type": "number", + "default": {{ derefInt $declared.MaxSize }}, + "nullable": false, + "validation": { + "condition": "${var.group_{{ identifier $key }}_max_size >= var.group_{{ identifier $key }}_min_size}", + "error_message": "group_{{ identifier $key }}_max_size cannot be below group_{{ identifier $key }}_min_size." + } + } + {{- end }} + } +} diff --git a/internal/casting/infrastructure/ecsec2terraformcasting/templates/versions.tf.json.gotmpl b/internal/casting/infrastructure/ecsec2terraformcasting/templates/versions.tf.json.gotmpl new file mode 100644 index 00000000..7814645c --- /dev/null +++ b/internal/casting/infrastructure/ecsec2terraformcasting/templates/versions.tf.json.gotmpl @@ -0,0 +1,11 @@ +{ + "terraform": { + "required_version": ">= 1.4.0", + "required_providers": { + "aws": { + "source": "hashicorp/aws", + "version": "5.100.0" + } + } + } +} diff --git a/internal/casting/infrastructure/registry.go b/internal/casting/infrastructure/registry.go index 23da6254..34db5517 100644 --- a/internal/casting/infrastructure/registry.go +++ b/internal/casting/infrastructure/registry.go @@ -4,8 +4,10 @@ import ( "log/slog" "github.com/signoz/foundry/api/v1alpha1" + "github.com/signoz/foundry/internal/casting/infrastructure/ecsec2terraformcasting" foundryerrors "github.com/signoz/foundry/internal/errors" "github.com/signoz/foundry/internal/tooler" + "github.com/signoz/foundry/internal/tooler/terraformtooler" ) type CastingItem struct { @@ -20,7 +22,16 @@ type Registry struct { func NewRegistry(logger *slog.Logger) *Registry { return &Registry{ - castings: map[v1alpha1.TypeDeployment]CastingItem{}, + castings: map[v1alpha1.TypeDeployment]CastingItem{ + { + Platform: v1alpha1.PlatformECS, + Mode: v1alpha1.ModeEC2, + Flavor: v1alpha1.FlavorTerraform, + }: { + Casting: ecsec2terraformcasting.New(logger), + Toolers: []tooler.Tooler{terraformtooler.New(logger)}, + }, + }, } } diff --git a/internal/domain/template.go b/internal/domain/template.go index fdf1c72d..c8eb7b8d 100644 --- a/internal/domain/template.go +++ b/internal/domain/template.go @@ -5,6 +5,7 @@ import ( "embed" "io" "path/filepath" + "strings" "text/template" "github.com/Masterminds/sprig/v3" @@ -110,6 +111,8 @@ func (t *Template) Format() Format { // - toYaml / fromYaml: round-trip a value through YAML inside templates. // - flattenKeys: flatten a nested map into "/"-joined leaf keys (used to // project hierarchical config into flat env-var-style maps). +// - identifier: respell a key's hyphens as underscores for a generated +// identifier. Keys cannot contain underscores, so no two keys collide. func templateFuncMap() template.FuncMap { fm := template.FuncMap(sprig.FuncMap()) fm["derefInt"] = func(p *int) int { @@ -147,6 +150,9 @@ func templateFuncMap() template.FuncMap { flattenMapKeys("", m, result) return result } + fm["identifier"] = func(s string) string { + return strings.ReplaceAll(s, "-", "_") + } return fm } diff --git a/internal/domain/template_test.go b/internal/domain/template_test.go index 8bddc229..4f1992ae 100644 --- a/internal/domain/template_test.go +++ b/internal/domain/template_test.go @@ -86,3 +86,37 @@ func TestTemplateRender(t *testing.T) { }) } } + +func TestTemplateIdentifierFunc(t *testing.T) { + tests := []struct { + name string + key string + expectedFmt []byte + }{ + { + name: "Hyphenated_Underscored", + key: "private-a", + expectedFmt: []byte("subnet_private_a_cidr"), + }, + { + name: "NoHyphens_Unchanged", + key: "ephemeral", + expectedFmt: []byte("subnet_ephemeral_cidr"), + }, + { + name: "ManyHyphens_AllUnderscored", + key: "a-b-c", + expectedFmt: []byte("subnet_a_b_c_cidr"), + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + tmpl := MustNewTemplate(tt.name, []byte("subnet_{{ identifier .Key }}_cidr"), FormatText) + + material, err := tmpl.Render(map[string]string{"Key": tt.key}, "out.txt") + assert.NoError(t, err) + assert.Equal(t, tt.expectedFmt, material.FmtContents()) + }) + } +}