From 0d70a15cbc28a86dec93abba937145fb1c96c44a Mon Sep 17 00:00:00 2001 From: Nagesh Bansal Date: Fri, 11 Sep 2026 01:13:42 +0530 Subject: [PATCH 1/4] feat(casting/ecs): run stateful components as multi-node clusters Every stateful component renders one ECS service per node from the casting's cluster fields: the store as shards x (replicas + 1), the keeper and ZooKeeper as replicas, PostgreSQL and SigNoz as replicas. Each node has its own task definition, Cloud Map name, host path and AppConfig profile, and the loops wrap the base's single-node shape without renaming anything, so a default casting forges the same pour. ZooKeeper nodes learn their peers through ZOO_SERVERS; ClickHouse and keeper nodes read the per-node config the molding already writes; the enricher publishes every node's address, so SigNoz receives every PostgreSQL DSN. Nothing spreads nodes across instances yet: placement belongs to the substrate and arrives with it. --- docs/examples/ecs/ec2/terraform/README.md | 45 +++-- .../casting/ecsterraformcasting/embed_test.go | 158 ++++++++++++++++++ .../templates/metastore.tf.json.gotmpl | 23 ++- .../templates/outputs.tf.json.gotmpl | 9 +- .../templates/signoz.tf.json.gotmpl | 28 +++- .../templates/telemetrykeeper.tf.json.gotmpl | 53 +++++- .../templates/telemetrystore.tf.json.gotmpl | 83 +++++---- 7 files changed, 337 insertions(+), 62 deletions(-) diff --git a/docs/examples/ecs/ec2/terraform/README.md b/docs/examples/ecs/ec2/terraform/README.md index 36c8571e..a48063c3 100644 --- a/docs/examples/ecs/ec2/terraform/README.md +++ b/docs/examples/ecs/ec2/terraform/README.md @@ -18,12 +18,10 @@ Components: - OTel Collector (ingester) - Schema migrator, run once on Fargate -The example beside this file is [`byo/`](byo/): the cluster is yours, and every object it is made of is stated on the casting. - > [!IMPORTANT] > Mount a durable volume at `/var/lib/foundry` on the instances that run stateful services, otherwise data lives on the instance's root disk. This deployment sets no placement constraints; pin stateful services to their instances yourself. > -> One node per component: cluster shard and replica counts are not honoured yet. +> Cluster shard and replica counts are honoured: every node is its own service. ## Prerequisites @@ -64,6 +62,31 @@ Component configuration is delivered through AWS AppConfig and reloaded in place Stateful data lives on the instance running the task, under `/var/lib/foundry////`, for example `/var/lib/foundry/signoz/telemetrystore/clickhouse/0-0`. +Shard and replica counts become nodes, and every node is its own ECS service: + +```yaml +spec: + telemetrystore: + spec: + cluster: + shards: 2 + replicas: 1 # copies beside each shard, so four nodes here + telemetrykeeper: + spec: + cluster: + replicas: 3 # keep it odd, for quorum + signoz: + spec: + cluster: + replicas: 2 # under sqlite each node holds its own database + metastore: + spec: + cluster: + replicas: 2 +``` + +Nothing spreads the nodes across instances, so pin them yourself if that matters. + ## Deploy ```bash @@ -109,9 +132,9 @@ One root module, one file per component, and no child modules. It creates: | Resource | Count | | --- | --- | | `aws_service_discovery_private_dns_namespace` | 1, named `-installation.local` | -| `aws_service_discovery_service` | one per component | -| `aws_ecs_service` | one per component | -| `aws_ecs_task_definition` | one per component, plus the migrator | +| `aws_service_discovery_service` | one per node | +| `aws_ecs_service` | one per node | +| `aws_ecs_task_definition` | one per node, plus the migrator | | `aws_appconfig_*` | one set per config file | | `aws_iam_role` | 2, unless stated on the casting | @@ -145,11 +168,11 @@ Components resolve each other inside `-installation.local`: | Component | DNS name | Ports | | --- | --- | --- | -| ClickHouse Keeper | `telemetrykeeper-clickhousekeeper-0` | 9181 client, 9234 raft | -| ZooKeeper | `telemetrykeeper-zookeeper-0` | 2181 client, 2888 raft, 3888 election, 9141 metrics | -| ClickHouse | `telemetrystore-clickhouse-0-0` | 9000 native, 8123 HTTP, 9009 interserver, 9363 metrics | -| PostgreSQL | `metastore-postgres-0` | 5432 | -| SigNoz | `signoz-0` | 8080 API, 4320 OpAMP | +| ClickHouse Keeper | `telemetrykeeper-clickhousekeeper-` | 9181 client, 9234 raft | +| ZooKeeper | `telemetrykeeper-zookeeper-` | 2181 client, 2888 raft, 3888 election, 9141 metrics | +| ClickHouse | `telemetrystore-clickhouse--` | 9000 native, 8123 HTTP, 9009 interserver, 9363 metrics | +| PostgreSQL | `metastore-postgres-` | 5432 | +| SigNoz | `signoz-` | 8080 API, 4320 OpAMP | | Ingester | `ingester` | 4317 gRPC, 4318 HTTP | | MCP | `mcp` | 8000 | diff --git a/internal/casting/ecsterraformcasting/embed_test.go b/internal/casting/ecsterraformcasting/embed_test.go index 603f7d58..dc46973c 100644 --- a/internal/casting/ecsterraformcasting/embed_test.go +++ b/internal/casting/ecsterraformcasting/embed_test.go @@ -737,3 +737,161 @@ func TestMetaStoreIsANodeGivenTimeToStart(t *testing.T) { }) } } + +// A clustered fixture is one service per node, each named by the ordinal the +// molding gives it. +func TestStatefulComponentsForgeAServicePerNode(t *testing.T) { + shards, perShard, keepers, nodes := 2, 1, 3, 2 + + store := statedCasting(&installation.Casting{}) + store.Spec.TelemetryStore.Spec.Cluster.Shards = &shards + store.Spec.TelemetryStore.Spec.Cluster.Replicas = &perShard + + keeper := statedCasting(&installation.Casting{}) + keeper.Spec.TelemetryKeeper.Kind = installation.TelemetryKeeperKindClickhouseKeeper + keeper.Spec.TelemetryKeeper.Spec.Cluster.Replicas = &keepers + + metaStore := statedCasting(&installation.Casting{}) + metaStore.Spec.MetaStore.Spec.Cluster.Replicas = &nodes + + signoz := statedCasting(&installation.Casting{}) + signoz.Spec.Signoz.Spec.Cluster.Replicas = &nodes + + tests := []struct { + name string + template *domain.Template + casting *installation.Casting + expectedServices []string + }{ + { + name: "TelemetryStore_ServicePerNode", template: telemetryStoreTF, casting: store, + expectedServices: []string{ + "signoz-telemetrystore-clickhouse-0-0", + "signoz-telemetrystore-clickhouse-0-1", + "signoz-telemetrystore-clickhouse-1-0", + "signoz-telemetrystore-clickhouse-1-1", + }, + }, + { + name: "TelemetryKeeper_ServicePerNode", template: telemetryKeeperTF, casting: keeper, + expectedServices: []string{ + "signoz-telemetrykeeper-clickhousekeeper-0", + "signoz-telemetrykeeper-clickhousekeeper-1", + "signoz-telemetrykeeper-clickhousekeeper-2", + }, + }, + { + name: "MetaStore_ServicePerNode", template: metaStoreTF, casting: metaStore, + expectedServices: []string{"signoz-metastore-postgres-0", "signoz-metastore-postgres-1"}, + }, + { + name: "Signoz_ServicePerNode", template: signozTF, casting: signoz, + expectedServices: []string{"signoz-signoz-0", "signoz-signoz-1"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + buf := bytes.NewBuffer(nil) + require.NoError(t, tt.template.Execute(buf, templateDataFor(t, tt.casting))) + + material, err := domain.NewJSONMaterial(buf.Bytes(), "component.tf.json") + require.NoError(t, err) + + services, err := material.GetStringSlice("resource.aws_ecs_service.@values.#.name") + require.NoError(t, err) + assert.ElementsMatch(t, tt.expectedServices, services) + }) + } +} + +// ZooKeeper numbers its own nodes from one, and an ensemble names itself +// 0.0.0.0 so the node does not dial its own service record. +func TestZookeeperEnsembleKnowsItsPeers(t *testing.T) { + tests := []struct { + name string + replicas int + node int + expectedID string + expectedServers string + }{ + {name: "SingleNode_NoEnsemble", replicas: 1, node: 0, expectedID: "1"}, + { + name: "FirstOfThree_Ensemble", replicas: 3, node: 0, expectedID: "1", + expectedServers: "0.0.0.0:2888:3888,telemetrykeeper-zookeeper-1.signoz-installation.local:2888:3888,telemetrykeeper-zookeeper-2.signoz-installation.local:2888:3888", + }, + { + name: "LastOfThree_Ensemble", replicas: 3, node: 2, expectedID: "3", + expectedServers: "telemetrykeeper-zookeeper-0.signoz-installation.local:2888:3888,telemetrykeeper-zookeeper-1.signoz-installation.local:2888:3888,0.0.0.0:2888:3888", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + casting := statedCasting(&installation.Casting{}) + casting.Spec.TelemetryKeeper.Kind = installation.TelemetryKeeperKindZookeeper + casting.Spec.TelemetryKeeper.Spec.Cluster.Replicas = &tt.replicas + + buf := bytes.NewBuffer(nil) + require.NoError(t, telemetryKeeperTF.Execute(buf, templateDataFor(t, casting))) + + material, err := domain.NewJSONMaterial(buf.Bytes(), "telemetrykeeper.tf.json") + require.NoError(t, err) + + container := fmt.Sprintf(`locals.containers_telemetrykeeper_zookeeper_%d.#(name=="signoz-telemetrykeeper-zookeeper-%d")`, tt.node, tt.node) + + id, err := material.GetBytes(container + `.environment.#(name=="ZOO_SERVER_ID").value`) + require.NoError(t, err) + assert.Equal(t, tt.expectedID, string(id)) + + servers, err := material.GetBytes(container + `.environment.#(name=="ZOO_SERVERS").value`) + if tt.expectedServers == "" { + assert.Error(t, err, "a single node stands alone") + + return + } + + require.NoError(t, err) + assert.Equal(t, tt.expectedServers, string(servers)) + }) + } +} + +// A patch written against the singular output keeps working, so both shapes +// stand. +func TestOutputsEnumerateEveryNode(t *testing.T) { + shards, perShard, keepers := 2, 1, 3 + + casting := statedCasting(&installation.Casting{}) + casting.Spec.TelemetryStore.Spec.Cluster.Shards = &shards + casting.Spec.TelemetryStore.Spec.Cluster.Replicas = &perShard + casting.Spec.TelemetryKeeper.Spec.Cluster.Replicas = &keepers + + buf := bytes.NewBuffer(nil) + require.NoError(t, outputsTF.Execute(buf, templateDataFor(t, casting))) + + material, err := domain.NewJSONMaterial(buf.Bytes(), "outputs.tf.json") + require.NoError(t, err) + + tests := []struct { + name string + path string + expectedCount int + }{ + {name: "TelemetryStore_PerNode", path: "output.telemetrystore_service_names.value", expectedCount: 4}, + {name: "TelemetryKeeper_PerNode", path: "output.telemetrykeeper_service_names.value", expectedCount: 3}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + names, err := material.GetStringSlice(tt.path) + require.NoError(t, err, "reading %s", tt.path) + assert.Len(t, names, tt.expectedCount) + }) + } + + for _, path := range []string{"output.metastore_service_name.value", "output.signoz_service_name.value", "output.signoz_service_arn.value"} { + _, err := material.GetBytes(path) + assert.NoError(t, err, "reading %s", path) + } +} diff --git a/internal/casting/ecsterraformcasting/templates/metastore.tf.json.gotmpl b/internal/casting/ecsterraformcasting/templates/metastore.tf.json.gotmpl index 156bf7eb..c64da882 100644 --- a/internal/casting/ecsterraformcasting/templates/metastore.tf.json.gotmpl +++ b/internal/casting/ecsterraformcasting/templates/metastore.tf.json.gotmpl @@ -1,11 +1,12 @@ {{- $name := $.Metadata.Name -}} {{- $kind := $.Spec.MetaStore.Kind.String -}} -{{- $i := 0 -}} -{{- $label := printf "metastore_%s_%d" $kind $i -}} -{{- $dns := printf "metastore-%s-%d" $kind $i -}} -{{- $family := printf "%s-%s" $name $dns -}} +{{- $replicas := derefIntDefault $.Spec.MetaStore.Spec.Cluster.Replicas 1 -}} +{{- if lt $replicas 1 }}{{- $replicas = 1 -}}{{- end -}} { "locals": { + {{- range $i := until $replicas }}{{ if $i }},{{ end }} + {{- $label := printf "metastore_%s_%d" $kind $i }} + {{- $family := printf "%s-metastore-%s-%d" $name $kind $i }} "containers_{{ $label }}": [ { "name": "{{ $family }}", @@ -44,9 +45,13 @@ } } ] + {{- end }} }, "resource": { "aws_ecs_task_definition": { + {{- range $i := until $replicas }}{{ if $i }},{{ end }} + {{- $label := printf "metastore_%s_%d" $kind $i }} + {{- $family := printf "%s-metastore-%s-%d" $name $kind $i }} "{{ $label }}": { "family": "{{ $family }}", "tags": {{ toJson $.Labels }}, @@ -62,10 +67,13 @@ } ] } + {{- end }} }, "aws_service_discovery_service": { + {{- range $i := until $replicas }}{{ if $i }},{{ end }} + {{- $label := printf "metastore_%s_%d" $kind $i }} "{{ $label }}": { - "name": "{{ $dns }}", + "name": "metastore-{{ $kind }}-{{ $i }}", "tags": {{ toJson $.Labels }}, "dns_config": { "namespace_id": "${aws_service_discovery_private_dns_namespace.main.id}", @@ -78,8 +86,12 @@ "routing_policy": "MULTIVALUE" } } + {{- end }} }, "aws_ecs_service": { + {{- range $i := until $replicas }}{{ if $i }},{{ end }} + {{- $label := printf "metastore_%s_%d" $kind $i }} + {{- $family := printf "%s-metastore-%s-%d" $name $kind $i }} "{{ $label }}": { "name": "{{ $family }}", "cluster": "${local.cluster_arn}", @@ -101,6 +113,7 @@ "registry_arn": "${aws_service_discovery_service.{{ $label }}.arn}" } } + {{- end }} } } } diff --git a/internal/casting/ecsterraformcasting/templates/outputs.tf.json.gotmpl b/internal/casting/ecsterraformcasting/templates/outputs.tf.json.gotmpl index 946e069d..ebc720bd 100644 --- a/internal/casting/ecsterraformcasting/templates/outputs.tf.json.gotmpl +++ b/internal/casting/ecsterraformcasting/templates/outputs.tf.json.gotmpl @@ -1,3 +1,8 @@ +{{- $storeShards := derefIntDefault $.Spec.TelemetryStore.Spec.Cluster.Shards 1 -}} +{{- if lt $storeShards 1 }}{{- $storeShards = 1 -}}{{- end -}} +{{- $storePerShard := int (add (derefIntDefault $.Spec.TelemetryStore.Spec.Cluster.Replicas 0) 1) -}} +{{- $keeperReplicas := derefIntDefault $.Spec.TelemetryKeeper.Spec.Cluster.Replicas 1 -}} +{{- if lt $keeperReplicas 1 }}{{- $keeperReplicas = 1 -}}{{- end -}} { "output": { "cluster_arn": { @@ -43,13 +48,13 @@ {{- if derefBool $.Spec.TelemetryStore.Spec.Enabled }}, "telemetrystore_service_names": { "description": "TelemetryStore ECS service names (one per node)", - "value": ["${aws_ecs_service.telemetrystore_{{ $.Spec.TelemetryStore.Kind.String }}_0_0.name}"] + "value": [{{ $first := true }}{{ range $s := until $storeShards }}{{ range $r := until $storePerShard }}{{ if not $first }}, {{ end }}{{ $first = false }}"${aws_ecs_service.telemetrystore_{{ $.Spec.TelemetryStore.Kind.String }}_{{ $s }}_{{ $r }}.name}"{{ end }}{{ end }}] } {{- end }} {{- if derefBool $.Spec.TelemetryKeeper.Spec.Enabled }}, "telemetrykeeper_service_names": { "description": "TelemetryKeeper ECS service names (one per node)", - "value": ["${aws_ecs_service.telemetrykeeper_{{ $.Spec.TelemetryKeeper.Kind.String }}_0.name}"] + "value": [{{ range $i := until $keeperReplicas }}{{ if $i }}, {{ end }}"${aws_ecs_service.telemetrykeeper_{{ $.Spec.TelemetryKeeper.Kind.String }}_{{ $i }}.name}"{{ end }}] } {{- end }} {{- if and (derefBool $.Spec.MetaStore.Spec.Enabled) (eq $.Spec.MetaStore.Kind.String "postgres") }}, diff --git a/internal/casting/ecsterraformcasting/templates/signoz.tf.json.gotmpl b/internal/casting/ecsterraformcasting/templates/signoz.tf.json.gotmpl index a85b06db..3b3afe64 100644 --- a/internal/casting/ecsterraformcasting/templates/signoz.tf.json.gotmpl +++ b/internal/casting/ecsterraformcasting/templates/signoz.tf.json.gotmpl @@ -1,13 +1,13 @@ {{- $name := $.Metadata.Name -}} {{- $sqlite := eq $.Spec.MetaStore.Kind.String "sqlite" -}} -{{- $i := 0 -}} -{{- $label := printf "signoz_%d" $i -}} -{{- $dns := printf "signoz-%d" $i -}} -{{- $family := printf "%s-signoz-%d" $name $i -}} -{{- $local := printf "containers_%s" $label -}} +{{- $replicas := derefIntDefault $.Spec.Signoz.Spec.Cluster.Replicas 1 -}} +{{- if lt $replicas 1 }}{{- $replicas = 1 -}}{{- end -}} { "locals": { - "{{ $local }}": [ + {{- range $i := until $replicas }}{{ if $i }},{{ end }} + {{- $label := printf "signoz_%d" $i }} + {{- $family := printf "%s-signoz-%d" $name $i }} + "containers_{{ $label }}": [ { "name": "{{ $family }}", "image": "{{ $.Spec.Signoz.Spec.Image }}", @@ -50,9 +50,13 @@ } } ] + {{- end }} }, "resource": { "aws_ecs_task_definition": { + {{- range $i := until $replicas }}{{ if $i }},{{ end }} + {{- $label := printf "signoz_%d" $i }} + {{- $family := printf "%s-signoz-%d" $name $i }} "{{ $label }}": { "family": "{{ $family }}", "tags": {{ toJson $.Labels }}, @@ -60,7 +64,7 @@ "requires_compatibilities": ["EC2"], "task_role_arn": "${local.task_role_arn}", "execution_role_arn": "${local.execution_role_arn}", - "container_definitions": "${jsonencode(local.{{ $local }})}" + "container_definitions": "${jsonencode(local.containers_{{ $label }})}" {{- if $sqlite }}, "volume": [ { @@ -70,10 +74,13 @@ ] {{- end }} } + {{- end }} }, "aws_service_discovery_service": { + {{- range $i := until $replicas }}{{ if $i }},{{ end }} + {{- $label := printf "signoz_%d" $i }} "{{ $label }}": { - "name": "{{ $dns }}", + "name": "signoz-{{ $i }}", "tags": {{ toJson $.Labels }}, "dns_config": { "namespace_id": "${aws_service_discovery_private_dns_namespace.main.id}", @@ -86,8 +93,12 @@ "routing_policy": "MULTIVALUE" } } + {{- end }} }, "aws_ecs_service": { + {{- range $i := until $replicas }}{{ if $i }},{{ end }} + {{- $label := printf "signoz_%d" $i }} + {{- $family := printf "%s-signoz-%d" $name $i }} "{{ $label }}": { "name": "{{ $family }}", "cluster": "${local.cluster_arn}", @@ -109,6 +120,7 @@ "registry_arn": "${aws_service_discovery_service.{{ $label }}.arn}" } } + {{- end }} } } } diff --git a/internal/casting/ecsterraformcasting/templates/telemetrykeeper.tf.json.gotmpl b/internal/casting/ecsterraformcasting/templates/telemetrykeeper.tf.json.gotmpl index e943fb2c..22281d29 100644 --- a/internal/casting/ecsterraformcasting/templates/telemetrykeeper.tf.json.gotmpl +++ b/internal/casting/ecsterraformcasting/templates/telemetrykeeper.tf.json.gotmpl @@ -1,11 +1,14 @@ {{- $name := $.Metadata.Name -}} {{- $kind := $.Spec.TelemetryKeeper.Kind.String -}} +{{- $namespace := printf "%s-installation.local" $name -}} +{{- $replicas := derefIntDefault $.Spec.TelemetryKeeper.Spec.Cluster.Replicas 1 -}} +{{- if lt $replicas 1 }}{{- $replicas = 1 -}}{{- end -}} {{- /* The molding names its config file by the same ordinal. */ -}} -{{- $i := 0 -}} -{{- $label := printf "telemetrykeeper_%s_%d" $kind $i -}} {{- if eq $kind "zookeeper" -}} { "locals": { + {{- range $i := until $replicas }}{{ if $i }},{{ end }} + {{- $label := printf "telemetrykeeper_%s_%d" $kind $i }} "containers_{{ $label }}": [ { "name": "{{ $name }}-telemetrykeeper-{{ $kind }}-{{ $i }}", @@ -22,6 +25,17 @@ "user": "0", {{- /* One-based: ZooKeeper's own myid starts at 1. */}} {{- $env := list (dict "name" "ZOO_SERVER_ID" "value" (printf "%d" (add $i 1))) }} + {{- if gt $replicas 1 }} + {{- $servers := list }} + {{- range $j := until $replicas }} + {{- if eq $j $i }} + {{- $servers = append $servers "0.0.0.0:2888:3888" }} + {{- else }} + {{- $servers = append $servers (printf "telemetrykeeper-%s-%d.%s:2888:3888" $kind $j $namespace) }} + {{- end }} + {{- end }} + {{- $env = append $env (dict "name" "ZOO_SERVERS" "value" (join "," $servers)) }} + {{- end }} {{- range $key, $value := $.Spec.TelemetryKeeper.Spec.Env }} {{- $env = append $env (dict "name" $key "value" $value) }} {{- end }} @@ -49,9 +63,12 @@ } } ] + {{- end }} }, "resource": { "aws_ecs_task_definition": { + {{- range $i := until $replicas }}{{ if $i }},{{ end }} + {{- $label := printf "telemetrykeeper_%s_%d" $kind $i }} "{{ $label }}": { "family": "{{ $name }}-telemetrykeeper-{{ $kind }}-{{ $i }}", "tags": {{ toJson $.Labels }}, @@ -67,8 +84,11 @@ } ] } + {{- end }} }, "aws_service_discovery_service": { + {{- range $i := until $replicas }}{{ if $i }},{{ end }} + {{- $label := printf "telemetrykeeper_%s_%d" $kind $i }} "{{ $label }}": { "name": "telemetrykeeper-{{ $kind }}-{{ $i }}", "tags": {{ toJson $.Labels }}, @@ -83,8 +103,11 @@ "routing_policy": "MULTIVALUE" } } + {{- end }} }, "aws_ecs_service": { + {{- range $i := until $replicas }}{{ if $i }},{{ end }} + {{- $label := printf "telemetrykeeper_%s_%d" $kind $i }} "{{ $label }}": { "name": "{{ $name }}-telemetrykeeper-{{ $kind }}-{{ $i }}", "cluster": "${local.cluster_arn}", @@ -106,13 +129,16 @@ "registry_arn": "${aws_service_discovery_service.{{ $label }}.arn}" } } + {{- end }} } } } {{- else -}} -{{- $profile := printf "telemetrykeeper-%s-%d" $kind $i -}} { "locals": { + {{- range $i := until $replicas }}{{ if $i }},{{ end }} + {{- $label := printf "telemetrykeeper_%s_%d" $kind $i }} + {{- $profile := printf "telemetrykeeper-%s-%d" $kind $i }} "containers_{{ $label }}": [ {{- /* A config change reloads keeper rather than replacing the node that holds the raft log. */}} @@ -198,28 +224,37 @@ } } ] + {{- end }} }, "resource": { {{- /* Hosted versions are immutable, so editing the pour's YAML mints a new version and a new deployment. */}} "aws_appconfig_configuration_profile": { + {{- range $i := until $replicas }}{{ if $i }},{{ end }} + {{- $label := printf "telemetrykeeper_%s_%d" $kind $i }} "{{ $label }}": { "application_id": "${aws_appconfig_application.main.id}", - "name": "{{ $profile }}", + "name": "telemetrykeeper-{{ $kind }}-{{ $i }}", "location_uri": "hosted", "type": "AWS.Freeform", "tags": {{ toJson $.Labels }} } + {{- end }} }, "aws_appconfig_hosted_configuration_version": { + {{- range $i := until $replicas }}{{ if $i }},{{ end }} + {{- $label := printf "telemetrykeeper_%s_%d" $kind $i }} "{{ $label }}": { "application_id": "${aws_appconfig_application.main.id}", "configuration_profile_id": "${aws_appconfig_configuration_profile.{{ $label }}.configuration_profile_id}", "content_type": "application/x-yaml", "content": "${file(\"${path.module}/telemetrykeeper/{{ $kind }}/keeper-{{ $i }}.yaml\")}" } + {{- end }} }, "aws_appconfig_deployment": { + {{- range $i := until $replicas }}{{ if $i }},{{ end }} + {{- $label := printf "telemetrykeeper_%s_%d" $kind $i }} "{{ $label }}": { "application_id": "${aws_appconfig_application.main.id}", "environment_id": "${aws_appconfig_environment.main.environment_id}", @@ -228,8 +263,11 @@ "deployment_strategy_id": "${aws_appconfig_deployment_strategy.main.id}", "tags": {{ toJson $.Labels }} } + {{- end }} }, "aws_ecs_task_definition": { + {{- range $i := until $replicas }}{{ if $i }},{{ end }} + {{- $label := printf "telemetrykeeper_%s_%d" $kind $i }} "{{ $label }}": { "family": "{{ $name }}-telemetrykeeper-{{ $kind }}-{{ $i }}", "tags": {{ toJson $.Labels }}, @@ -253,8 +291,11 @@ ], "depends_on": ["aws_appconfig_deployment.{{ $label }}"] } + {{- end }} }, "aws_service_discovery_service": { + {{- range $i := until $replicas }}{{ if $i }},{{ end }} + {{- $label := printf "telemetrykeeper_%s_%d" $kind $i }} "{{ $label }}": { "name": "telemetrykeeper-{{ $kind }}-{{ $i }}", "tags": {{ toJson $.Labels }}, @@ -269,8 +310,11 @@ "routing_policy": "MULTIVALUE" } } + {{- end }} }, "aws_ecs_service": { + {{- range $i := until $replicas }}{{ if $i }},{{ end }} + {{- $label := printf "telemetrykeeper_%s_%d" $kind $i }} "{{ $label }}": { "name": "{{ $name }}-telemetrykeeper-{{ $kind }}-{{ $i }}", "cluster": "${local.cluster_arn}", @@ -292,6 +336,7 @@ "registry_arn": "${aws_service_discovery_service.{{ $label }}.arn}" } } + {{- end }} } } } diff --git a/internal/casting/ecsterraformcasting/templates/telemetrystore.tf.json.gotmpl b/internal/casting/ecsterraformcasting/templates/telemetrystore.tf.json.gotmpl index 61a7f5d8..78349244 100644 --- a/internal/casting/ecsterraformcasting/templates/telemetrystore.tf.json.gotmpl +++ b/internal/casting/ecsterraformcasting/templates/telemetrystore.tf.json.gotmpl @@ -1,15 +1,21 @@ {{- $name := $.Metadata.Name -}} {{- $kind := $.Spec.TelemetryStore.Kind.String -}} +{{- $shards := derefIntDefault $.Spec.TelemetryStore.Spec.Cluster.Shards 1 -}} +{{- if lt $shards 1 }}{{- $shards = 1 -}}{{- end -}} +{{- $perShard := int (add (derefIntDefault $.Spec.TelemetryStore.Spec.Cluster.Replicas 0) 1) -}} {{- /* The molding names its config file by the same id. */ -}} -{{- $id := "0-0" -}} -{{- $nodeKey := "0_0" -}} -{{- $label := printf "telemetrystore_%s_%s" $kind $nodeKey -}} +{{- $nodes := list -}} +{{- range $s := until $shards -}} +{{- range $r := until $perShard -}} +{{- $nodes = append $nodes (dict "id" (printf "%d-%d" $s $r) "label" (printf "telemetrystore_%s_%d_%d" $kind $s $r) "profile" (printf "telemetrystore-%s-%d-%d" $kind $s $r)) -}} +{{- end -}} +{{- end -}} {{- $fnLabel := printf "telemetrystore_%s_functions" $kind -}} -{{- $cfgProfile := printf "telemetrystore-%s-%s" $kind $id -}} {{- $fnProfile := printf "telemetrystore-%s-functions" $kind -}} { "locals": { - "containers_{{ $label }}": [ + {{- range $i, $node := $nodes }}{{ if $i }},{{ end }} + "containers_{{ $node.label }}": [ { "name": "{{ $name }}-telemetrystore-user-scripts", "image": "{{ $.Spec.TelemetryStore.Spec.Image }}", @@ -52,9 +58,9 @@ {"containerName": "{{ $name }}-telemetrystore-user-scripts", "condition": "SUCCESS"} ], "environment": [ - {"name": "PREFETCH_LIST", "value": "{{ $name }}-installation-appconfig:default:{{ $cfgProfile }},{{ $name }}-installation-appconfig:default:{{ $fnProfile }}"}, + {"name": "PREFETCH_LIST", "value": "{{ $name }}-installation-appconfig:default:{{ $node.profile }},{{ $name }}-installation-appconfig:default:{{ $fnProfile }}"}, {"name": "POLL_INTERVAL", "value": "45s"}, - {"name": "MANIFEST", "value": "{\"{{ $name }}-installation-appconfig:default:{{ $cfgProfile }}\":{\"writeTo\":{\"path\":\"/etc/clickhouse-server/config.d/config-{{ $id }}.yaml\"}},\"{{ $name }}-installation-appconfig:default:{{ $fnProfile }}\":{\"writeTo\":{\"path\":\"/etc/clickhouse-server/config.d/functions.yaml\"}}}"} + {"name": "MANIFEST", "value": "{\"{{ $name }}-installation-appconfig:default:{{ $node.profile }}\":{\"writeTo\":{\"path\":\"/etc/clickhouse-server/config.d/config-{{ $node.id }}.yaml\"}},\"{{ $name }}-installation-appconfig:default:{{ $fnProfile }}\":{\"writeTo\":{\"path\":\"/etc/clickhouse-server/config.d/functions.yaml\"}}}"} ], "mountPoints": [ { @@ -63,7 +69,7 @@ } ], "healthCheck": { - "command": ["CMD-SHELL", "test -s /etc/clickhouse-server/config.d/config-{{ $id }}.yaml && test -s /etc/clickhouse-server/config.d/functions.yaml"], + "command": ["CMD-SHELL", "test -s /etc/clickhouse-server/config.d/config-{{ $node.id }}.yaml && test -s /etc/clickhouse-server/config.d/functions.yaml"], "interval": 5, "timeout": 3, "retries": 10, @@ -72,7 +78,7 @@ "memoryReservation": 102 }, { - "name": "{{ $name }}-telemetrystore-{{ $kind }}-{{ $id }}", + "name": "{{ $name }}-telemetrystore-{{ $kind }}-{{ $node.id }}", "image": "{{ $.Spec.TelemetryStore.Spec.Image }}", "logConfiguration": { "logDriver": "json-file", @@ -84,7 +90,7 @@ }, "essential": true, {{- $env := list (dict "name" "CLICKHOUSE_SKIP_USER_SETUP" "value" "1") }} - {{- $env = append $env (dict "name" "CLICKHOUSE_CONFIG" "value" (printf "/etc/clickhouse-server/config.d/config-%s.yaml" $id)) }} + {{- $env = append $env (dict "name" "CLICKHOUSE_CONFIG" "value" (printf "/etc/clickhouse-server/config.d/config-%s.yaml" $node.id)) }} {{- /* The agent writes its files as root, so the server stays root through the image's own switch. */}} {{- $env = append $env (dict "name" "CLICKHOUSE_RUN_AS_ROOT" "value" "1") }} @@ -128,16 +134,19 @@ } } ] + {{- end }} }, "resource": { "aws_appconfig_configuration_profile": { - "{{ $label }}": { + {{- range $i, $node := $nodes }}{{ if $i }},{{ end }} + "{{ $node.label }}": { "application_id": "${aws_appconfig_application.main.id}", - "name": "{{ $cfgProfile }}", + "name": "{{ $node.profile }}", "location_uri": "hosted", "type": "AWS.Freeform", "tags": {{ toJson $.Labels }} - }, + } + {{- end }}, "{{ $fnLabel }}": { "application_id": "${aws_appconfig_application.main.id}", "name": "{{ $fnProfile }}", @@ -147,12 +156,14 @@ } }, "aws_appconfig_hosted_configuration_version": { - "{{ $label }}": { + {{- range $i, $node := $nodes }}{{ if $i }},{{ end }} + "{{ $node.label }}": { "application_id": "${aws_appconfig_application.main.id}", - "configuration_profile_id": "${aws_appconfig_configuration_profile.{{ $label }}.configuration_profile_id}", + "configuration_profile_id": "${aws_appconfig_configuration_profile.{{ $node.label }}.configuration_profile_id}", "content_type": "application/x-yaml", - "content": "${file(\"${path.module}/telemetrystore/{{ $kind }}/config-{{ $id }}.yaml\")}" - }, + "content": "${file(\"${path.module}/telemetrystore/{{ $kind }}/config-{{ $node.id }}.yaml\")}" + } + {{- end }}, "{{ $fnLabel }}": { "application_id": "${aws_appconfig_application.main.id}", "configuration_profile_id": "${aws_appconfig_configuration_profile.{{ $fnLabel }}.configuration_profile_id}", @@ -161,14 +172,16 @@ } }, "aws_appconfig_deployment": { - "{{ $label }}": { + {{- range $i, $node := $nodes }}{{ if $i }},{{ end }} + "{{ $node.label }}": { "application_id": "${aws_appconfig_application.main.id}", "environment_id": "${aws_appconfig_environment.main.environment_id}", - "configuration_profile_id": "${aws_appconfig_configuration_profile.{{ $label }}.configuration_profile_id}", - "configuration_version": "${aws_appconfig_hosted_configuration_version.{{ $label }}.version_number}", + "configuration_profile_id": "${aws_appconfig_configuration_profile.{{ $node.label }}.configuration_profile_id}", + "configuration_version": "${aws_appconfig_hosted_configuration_version.{{ $node.label }}.version_number}", "deployment_strategy_id": "${aws_appconfig_deployment_strategy.main.id}", "tags": {{ toJson $.Labels }} - }, + } + {{- end }}, "{{ $fnLabel }}": { "application_id": "${aws_appconfig_application.main.id}", "environment_id": "${aws_appconfig_environment.main.environment_id}", @@ -179,14 +192,15 @@ } }, "aws_ecs_task_definition": { - "{{ $label }}": { - "family": "{{ $name }}-telemetrystore-{{ $kind }}-{{ $id }}", + {{- range $i, $node := $nodes }}{{ if $i }},{{ end }} + "{{ $node.label }}": { + "family": "{{ $name }}-telemetrystore-{{ $kind }}-{{ $node.id }}", "tags": {{ toJson $.Labels }}, "network_mode": "awsvpc", "requires_compatibilities": ["EC2"], "task_role_arn": "${local.task_role_arn}", "execution_role_arn": "${local.execution_role_arn}", - "container_definitions": "${jsonencode(local.containers_{{ $label }})}", + "container_definitions": "${jsonencode(local.containers_{{ $node.label }})}", "volume": [ { "name": "telemetrystore-user-scripts", @@ -204,15 +218,17 @@ }, { "name": "telemetrystore-data", - "host_path": "/var/lib/foundry/{{ $name }}/telemetrystore/{{ $kind }}/{{ $id }}" + "host_path": "/var/lib/foundry/{{ $name }}/telemetrystore/{{ $kind }}/{{ $node.id }}" } ], - "depends_on": ["aws_appconfig_deployment.{{ $label }}", "aws_appconfig_deployment.{{ $fnLabel }}"] + "depends_on": ["aws_appconfig_deployment.{{ $node.label }}", "aws_appconfig_deployment.{{ $fnLabel }}"] } + {{- end }} }, "aws_service_discovery_service": { - "{{ $label }}": { - "name": "telemetrystore-{{ $kind }}-{{ $id }}", + {{- range $i, $node := $nodes }}{{ if $i }},{{ end }} + "{{ $node.label }}": { + "name": "telemetrystore-{{ $kind }}-{{ $node.id }}", "tags": {{ toJson $.Labels }}, "dns_config": { "namespace_id": "${aws_service_discovery_private_dns_namespace.main.id}", @@ -225,12 +241,14 @@ "routing_policy": "MULTIVALUE" } } + {{- end }} }, "aws_ecs_service": { - "{{ $label }}": { - "name": "{{ $name }}-telemetrystore-{{ $kind }}-{{ $id }}", + {{- range $i, $node := $nodes }}{{ if $i }},{{ end }} + "{{ $node.label }}": { + "name": "{{ $name }}-telemetrystore-{{ $kind }}-{{ $node.id }}", "cluster": "${local.cluster_arn}", - "task_definition": "${aws_ecs_task_definition.{{ $label }}.arn}", + "task_definition": "${aws_ecs_task_definition.{{ $node.label }}.arn}", "desired_count": 1, {{- /* Two tasks cannot co-schedule over one data directory. */}} "deployment_minimum_healthy_percent": 0, @@ -246,9 +264,10 @@ "security_groups": "${local.security_group_ids}" }, "service_registries": { - "registry_arn": "${aws_service_discovery_service.{{ $label }}.arn}" + "registry_arn": "${aws_service_discovery_service.{{ $node.label }}.arn}" } } + {{- end }} } } } From 180fe77752a5128da4969950af9566a74fe9eb31 Mon Sep 17 00:00:00 2001 From: Nagesh Bansal Date: Fri, 11 Sep 2026 16:13:18 +0530 Subject: [PATCH 2/4] test(casting/ecs): keep the embed tests to what can break The tests assert behaviour: render validity, template data resolution and its refusal, component selection, the variables and tfvars contract, the subnet check, and node counts and wiring. They no longer pin literals the forged pour records. --- .../casting/ecsterraformcasting/embed_test.go | 752 ++++-------------- 1 file changed, 134 insertions(+), 618 deletions(-) diff --git a/internal/casting/ecsterraformcasting/embed_test.go b/internal/casting/ecsterraformcasting/embed_test.go index dc46973c..cf145b9e 100644 --- a/internal/casting/ecsterraformcasting/embed_test.go +++ b/internal/casting/ecsterraformcasting/embed_test.go @@ -9,6 +9,7 @@ import ( "strings" "testing" + "github.com/signoz/foundry/api/v1alpha1" "github.com/signoz/foundry/api/v1alpha1/installation" "github.com/signoz/foundry/internal/domain" "github.com/stretchr/testify/assert" @@ -37,204 +38,32 @@ func templateDataFor(t *testing.T, casting *installation.Casting) templateData { return data } -func TestNotEmptyAndValid(t *testing.T) { +func TestEveryTemplateRendersValidJSON(t *testing.T) { + data := templateDataFor(t, installation.Default(&installation.Casting{})) + templates := map[string]*domain.Template{ - "versionsTF": versionsTF, - "backendTF": backendTF, - "providersTF": providersTF, - "mainTF": mainTF, - "variablesTF": variablesTF, - "outputsTF": outputsTF, - "telemetryKeeperTF": telemetryKeeperTF, - "telemetryStoreTF": telemetryStoreTF, - "migratorTF": migratorTF, - "metaStoreTF": metaStoreTF, - "signozTF": signozTF, - "ingesterTF": ingesterTF, - "mcpTF": mcpTF, + "versions.tf.json": versionsTF, + "backend.tf.json": backendTF, + "providers.tf.json": providersTF, + "main.tf.json": mainTF, + "variables.tf.json": variablesTF, + "outputs.tf.json": outputsTF, + "terraform.tfvars.json": tfarsTF, + "telemetrykeeper.tf.json": telemetryKeeperTF, + "telemetrystore.tf.json": telemetryStoreTF, + "migrator.tf.json": migratorTF, + "metastore.tf.json": metaStoreTF, + "signoz.tf.json": signozTF, + "ingester.tf.json": ingesterTF, + "mcp.tf.json": mcpTF, } for name, tmpl := range templates { - assert.NotEmpty(t, tmpl, "%s should not be empty", name) buf := bytes.NewBuffer(nil) - err := tmpl.Execute(buf, nil) - assert.NoError(t, err, "error executing %s", name) - assert.NotEmpty(t, buf.String(), "%s output should not be empty", name) - } -} - -// A default in the variables would race the tfvars. -func TestTfvarsTemplateCarriesEveryValue(t *testing.T) { - buf := bytes.NewBuffer(nil) - require.NoError(t, tfarsTF.Execute(buf, templateDataFor(t, statedCasting(&installation.Casting{})))) - - assert.JSONEq(t, `{ - "aws_region": "us-east-1", - "cluster_arn": "arn:aws:ecs:us-east-1:123456789012:cluster/test", - "subnet_ids": ["subnet-abc123", "subnet-def456"], - "security_group_ids": ["sg-abc123"], - "vpc_id": "vpc-abc123", - "task_role_name": "signoz-installation-iam-task", - "execution_role_name": "signoz-installation-iam-exec" - }`, buf.String()) -} - -// No name is computed for a role this stack does not create. -func TestTfvarsTemplateCarriesStatedRoles(t *testing.T) { - casting := statedCasting(&installation.Casting{}) - casting.Metadata.Annotations[installation.ECSTaskRoleARN.Key] = "arn:aws:iam::123456789012:role/task" - casting.Metadata.Annotations[installation.ECSTaskExecutionRoleARN.Key] = "arn:aws:iam::123456789012:role/exec" - - buf := bytes.NewBuffer(nil) - require.NoError(t, tfarsTF.Execute(buf, templateDataFor(t, casting))) - - assert.JSONEq(t, `{ - "aws_region": "us-east-1", - "cluster_arn": "arn:aws:ecs:us-east-1:123456789012:cluster/test", - "subnet_ids": ["subnet-abc123", "subnet-def456"], - "security_group_ids": ["sg-abc123"], - "vpc_id": "vpc-abc123", - "task_role_arn": "arn:aws:iam::123456789012:role/task", - "execution_role_arn": "arn:aws:iam::123456789012:role/exec" - }`, buf.String()) -} - -func TestStatedObjectsAreResolvedThroughLocals(t *testing.T) { - data := templateDataFor(t, statedCasting(&installation.Casting{})) - - variables := bytes.NewBuffer(nil) - require.NoError(t, variablesTF.Execute(variables, data)) - - for _, expected := range []string{ - `"cluster_arn"`, - `"subnet_ids"`, - `"security_group_ids"`, - `"vpc_id"`, - `"task_role_name"`, - `"execution_role_name"`, - } { - assert.Contains(t, variables.String(), expected) - } - - assert.NotContains(t, variables.String(), `"default"`) - - main := bytes.NewBuffer(nil) - require.NoError(t, mainTF.Execute(main, data)) - - out := main.String() - for _, expected := range []string{ - `"cluster_arn": "${var.cluster_arn}"`, - `"subnet_ids": "${var.subnet_ids}"`, - `"security_group_ids": "${var.security_group_ids}"`, - `"vpc_id": "${var.vpc_id}"`, - `"name": "${var.task_role_name}"`, - } { - assert.Contains(t, out, expected) - } -} - -// A public subnet hands an awsvpc task on the EC2 launch type a route it -// cannot use. An empty "data" object is a root terraform refuses outright. -func TestStatedSubnetsAreCheckedAtPlan(t *testing.T) { - tests := []struct { - name string - subnetIDs string - expectedLookups []string - expectedConditions []string - expectedMessages []string - pass bool - }{ - { - name: "TwoSubnetsStated_Checked", - subnetIDs: "subnet-abc123, subnet-def456", - expectedLookups: []string{"${var.subnet_ids[0]}", "${var.subnet_ids[1]}"}, - expectedConditions: []string{"${!data.aws_subnet.tasks_0.map_public_ip_on_launch}", "${!data.aws_subnet.tasks_1.map_public_ip_on_launch}"}, - expectedMessages: []string{"subnet subnet-abc123 assigns public IPs on launch", "subnet subnet-def456 assigns public IPs on launch"}, - pass: true, - }, - {name: "NoSubnetStated_NotChecked"}, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - casting := statedCasting(&installation.Casting{}) - - if tt.subnetIDs == "" { - delete(casting.Metadata.Annotations, installation.ECSPrivateSubnetIDs.Key) - } else { - casting.Metadata.Annotations[installation.ECSPrivateSubnetIDs.Key] = tt.subnetIDs - } - - main := bytes.NewBuffer(nil) - require.NoError(t, mainTF.Execute(main, templateDataFor(t, casting))) - - material, err := domain.NewJSONMaterial(main.Bytes(), "main.tf.json") - require.NoError(t, err) - - if !tt.pass { - _, err := material.GetBytes("data") - assert.Error(t, err, "an unstated subnet list emits no lookup") - - _, err = material.GetBytes("resource.aws_service_discovery_private_dns_namespace.main.lifecycle") - assert.Error(t, err, "an unstated subnet list emits no precondition") - - return - } - - for index, expected := range tt.expectedLookups { - id, err := material.GetBytes(fmt.Sprintf("data.aws_subnet.tasks_%d.id", index)) - require.NoError(t, err) - assert.Equal(t, expected, string(id)) - } - - preconditions, err := material.GetBytes("resource.aws_service_discovery_private_dns_namespace.main.lifecycle.precondition") - require.NoError(t, err) - - rendered := string(preconditions) - assert.Equal(t, len(tt.expectedConditions), strings.Count(rendered, `"condition"`)) - - for _, expected := range tt.expectedConditions { - assert.Contains(t, rendered, expected) - } - - for _, expected := range tt.expectedMessages { - assert.Contains(t, rendered, expected) - } - }) - } -} - -func TestStatedRoleIsNotCreated(t *testing.T) { - casting := statedCasting(&installation.Casting{}) - casting.Metadata.Annotations[installation.ECSTaskRoleARN.Key] = "arn:aws:iam::123456789012:role/task" - casting.Metadata.Annotations[installation.ECSTaskExecutionRoleARN.Key] = "arn:aws:iam::123456789012:role/exec" - - data := templateDataFor(t, casting) - - main := bytes.NewBuffer(nil) - require.NoError(t, mainTF.Execute(main, data)) - - out := main.String() - assert.NotContains(t, out, "aws_iam_role") - assert.NotContains(t, out, "appconfig:StartConfigurationSession") - assert.Contains(t, out, `"task_role_arn": "${var.task_role_arn}"`) -} - -func TestReferenceIsStated(t *testing.T) { - tests := []struct { - name string - reference Reference - pass bool - }{ - {name: "Stated_Valid", reference: Reference{Stated: "arn:aws:ecs:us-east-1:1:cluster/x"}, pass: true}, - {name: "StatedIDs_Valid", reference: Reference{StatedIDs: []string{"subnet-a"}}, pass: true}, - {name: "Empty_Invalid", reference: Reference{}, pass: false}, - } + require.NoError(t, tmpl.Execute(buf, data), name) - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - assert.Equal(t, tt.pass, tt.reference.IsStated()) - }) + _, err := domain.NewJSONMaterial(buf.Bytes(), name) + require.NoError(t, err, name) } } @@ -282,458 +111,174 @@ func TestTemplateDataResolution(t *testing.T) { } } -// The platform judges its own inputs, so an unstated casting still forges. -func TestUnstatedCastingForges(t *testing.T) { - casting := installation.Default(&installation.Casting{}) - - materials, err := New(slog.New(slog.DiscardHandler)).Forge(context.Background(), *casting, "") - require.NoError(t, err) - assert.NotEmpty(t, materials) - - buf := bytes.NewBuffer(nil) - require.NoError(t, tfarsTF.Execute(buf, templateDataFor(t, casting))) - - assert.JSONEq(t, `{ - "aws_region": "", - "cluster_arn": "", - "subnet_ids": [], - "security_group_ids": [], - "vpc_id": "", - "task_role_name": "signoz-installation-iam-task", - "execution_role_name": "signoz-installation-iam-exec" - }`, buf.String()) -} - -// Terraform refuses an empty or malformed value at plan, so every variable the -// casting does not judge carries its own condition. -func TestVariablesCarryValidations(t *testing.T) { - casting := statedCasting(&installation.Casting{}) - casting.Metadata.Annotations[installation.ECSTaskRoleARN.Key] = "arn:aws:iam::123456789012:role/task" - casting.Metadata.Annotations[installation.ECSTaskExecutionRoleARN.Key] = "arn:aws:iam::123456789012:role/exec" - - buf := bytes.NewBuffer(nil) - require.NoError(t, variablesTF.Execute(buf, templateDataFor(t, casting))) +// Sqlite is a file the signoz task holds, so it is no service of its own. +func TestForgeSelectsComponents(t *testing.T) { + sqlite := statedCasting(&installation.Casting{}) + sqlite.Spec.MetaStore.Kind = installation.MetaStoreKindSQLite - material, err := domain.NewJSONMaterial(buf.Bytes(), "variables.tf.json") - require.NoError(t, err) + withMCP := statedCasting(&installation.Casting{}) + withMCP.Spec.MCP.Spec.Enabled = v1alpha1.BoolPtr(true) tests := []struct { - name string - variable string - expectedCondition string + name string + casting *installation.Casting + path string + expectedForged bool }{ - {name: "Region_Validated", variable: "aws_region", expectedCondition: `^[a-z]{2}(-gov)?-[a-z]+-[0-9]$`}, - {name: "ClusterARN_Validated", variable: "cluster_arn", expectedCondition: `^arn:aws:ecs:`}, - {name: "SubnetIDs_Validated", variable: "subnet_ids", expectedCondition: `^subnet-`}, - {name: "SecurityGroupIDs_Validated", variable: "security_group_ids", expectedCondition: `^sg-`}, - {name: "VPCID_Validated", variable: "vpc_id", expectedCondition: `^vpc-`}, - {name: "TaskRoleARN_Validated", variable: "task_role_arn", expectedCondition: `^arn:aws:iam::`}, - {name: "ExecutionRoleARN_Validated", variable: "execution_role_arn", expectedCondition: `^arn:aws:iam::`}, + {name: "Postgres_MetaStoreForged", casting: statedCasting(&installation.Casting{}), path: "metastore.tf.json", expectedForged: true}, + {name: "Sqlite_MetaStoreUnforged", casting: sqlite, path: "metastore.tf.json"}, + {name: "MCPEnabled_Forged", casting: withMCP, path: "mcp.tf.json", expectedForged: true}, + {name: "MCPDisabled_Unforged", casting: statedCasting(&installation.Casting{}), path: "mcp.tf.json"}, + {name: "NothingStated_Forged", casting: installation.Default(&installation.Casting{}), path: "main.tf.json", expectedForged: true}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - condition, err := material.GetBytes("variable." + tt.variable + ".validation.condition") + materials, err := New(slog.New(slog.DiscardHandler)).Forge(context.Background(), *tt.casting, "") require.NoError(t, err) - assert.Contains(t, string(condition), tt.expectedCondition) - }) - } -} - -// Sqlite is a file the signoz task holds, so it is no service of its own. -func TestSqliteForgesNoMetaStoreService(t *testing.T) { - casting := statedCasting(&installation.Casting{}) - casting.Spec.MetaStore.Kind = installation.MetaStoreKindSQLite - - materials, err := New(slog.New(slog.DiscardHandler)).Forge(context.Background(), *casting, "") - require.NoError(t, err) - - for _, material := range materials { - assert.NotContains(t, material.Path(), "metastore.tf.json") - } - - outputs := bytes.NewBuffer(nil) - require.NoError(t, outputsTF.Execute(outputs, templateDataFor(t, casting))) - - assert.NotContains(t, outputs.String(), "metastore_service_name") -} - -func TestPostgresForgesAMetaStoreService(t *testing.T) { - casting := statedCasting(&installation.Casting{}) - casting.Spec.MetaStore.Kind = installation.MetaStoreKindPostgres - - materials, err := New(slog.New(slog.DiscardHandler)).Forge(context.Background(), *casting, "") - require.NoError(t, err) - - forged := false - for _, material := range materials { - if strings.Contains(material.Path(), "metastore.tf.json") { - forged = true - } - } - - assert.True(t, forged) - - outputs := bytes.NewBuffer(nil) - require.NoError(t, outputsTF.Execute(outputs, templateDataFor(t, casting))) - - assert.Contains(t, outputs.String(), "metastore_service_name") -} - -// A container that logs without the labels reaches SigNoz carrying only a -// container id. Fargate refuses json-file, so the migrator carries no block. -func TestEveryEC2ContainerLogsWithLabels(t *testing.T) { - data := templateDataFor(t, statedCasting(&installation.Casting{})) - - for name, tmpl := range map[string]*domain.Template{ - "signozTF": signozTF, - "ingesterTF": ingesterTF, - "metaStoreTF": metaStoreTF, - "mcpTF": mcpTF, - "telemetryStoreTF": telemetryStoreTF, - "telemetryKeeperTF": telemetryKeeperTF, - } { - buf := bytes.NewBuffer(nil) - require.NoError(t, tmpl.Execute(buf, data), name) - - rendered := buf.String() - - assert.Equal(t, strings.Count(rendered, `"image":`), strings.Count(rendered, `"logConfiguration":`), - "%s: every container logs, or none of them is attributed", name) - assert.Contains(t, rendered, "com.amazonaws.ecs.task-definition-family,com.amazonaws.ecs.container-name,com.amazonaws.ecs.task-arn,com.amazonaws.ecs.cluster", name) - // Without rotation docker fills the instance disk. - assert.Contains(t, rendered, `"max-size": "10m"`, name) - } - - migrator := bytes.NewBuffer(nil) - require.NoError(t, migratorTF.Execute(migrator, data)) - - assert.NotContains(t, migrator.String(), "logConfiguration") -} - -// Nothing else puts a previous revision back. -func TestEveryServiceRollsBackABadRevision(t *testing.T) { - data := templateDataFor(t, statedCasting(&installation.Casting{})) - - for name, tmpl := range map[string]*domain.Template{ - "signozTF": signozTF, - "ingesterTF": ingesterTF, - "metaStoreTF": metaStoreTF, - "mcpTF": mcpTF, - "telemetryStoreTF": telemetryStoreTF, - "telemetryKeeperTF": telemetryKeeperTF, - } { - buf := bytes.NewBuffer(nil) - require.NoError(t, tmpl.Execute(buf, data), name) - - rendered := buf.String() - - assert.Equal(t, strings.Count(rendered, `"launch_type": "EC2"`), strings.Count(rendered, `"deployment_circuit_breaker"`), name) - assert.NotContains(t, rendered, "capacity_provider") - } -} - -// A CollectionAgent of the same name on the same account holds its own. -func TestAppConfigApplicationCarriesTheKind(t *testing.T) { - data := templateDataFor(t, statedCasting(&installation.Casting{})) - - main := bytes.NewBuffer(nil) - require.NoError(t, mainTF.Execute(main, data)) - - material, err := domain.NewJSONMaterial(main.Bytes(), "main.tf.json") - require.NoError(t, err) - - for path, expected := range map[string]string{ - "resource.aws_appconfig_application.main.name": "signoz-installation-appconfig", - "resource.aws_appconfig_deployment_strategy.main.name": "signoz-installation-appconfig-strategy", - - "resource.aws_service_discovery_private_dns_namespace.main.name": "signoz-installation.local", - } { - value, err := material.GetBytes(path) + forged := false + for _, material := range materials { + if strings.Contains(material.Path(), tt.path) { + forged = true + } + } - assert.NoError(t, err, "reading %s", path) - assert.Equal(t, expected, string(value), "at %s", path) + assert.Equal(t, tt.expectedForged, forged) + }) } - - // The sidecar prefetches by the application name, so a rename must reach it. - ingester := bytes.NewBuffer(nil) - require.NoError(t, ingesterTF.Execute(ingester, data)) - - assert.Contains(t, ingester.String(), "signoz-installation-appconfig:default:ingester") } -// A task bound to a disk cannot be replaced before the one holding it stops. -func TestServiceRollsBeforeStopping(t *testing.T) { +// A public subnet hands an awsvpc task on the EC2 launch type a route it +// cannot use. An empty "data" object is a root terraform refuses outright. +func TestStatedSubnetsAreCheckedAtPlan(t *testing.T) { tests := []struct { - name string - template *domain.Template - metaStoreKind installation.MetaStoreKind - service string - expectedMinimumPercent string - expectedMaximumPercent string + name string + subnetIDs string + expectedIDs []string + pass bool }{ - {name: "Ingester_Rolling", template: ingesterTF, metaStoreKind: installation.MetaStoreKindPostgres, service: "ingester", expectedMinimumPercent: "100", expectedMaximumPercent: "200"}, - {name: "MCP_Rolling", template: mcpTF, metaStoreKind: installation.MetaStoreKindPostgres, service: "mcp", expectedMinimumPercent: "100", expectedMaximumPercent: "200"}, - {name: "SignozOnPostgres_Rolling", template: signozTF, metaStoreKind: installation.MetaStoreKindPostgres, service: "signoz_0", expectedMinimumPercent: "100", expectedMaximumPercent: "200"}, - {name: "SignozOnSqlite_StopFirst", template: signozTF, metaStoreKind: installation.MetaStoreKindSQLite, service: "signoz_0", expectedMinimumPercent: "0", expectedMaximumPercent: "100"}, - {name: "TelemetryStore_StopFirst", template: telemetryStoreTF, metaStoreKind: installation.MetaStoreKindPostgres, service: "telemetrystore_clickhouse_0_0", expectedMinimumPercent: "0", expectedMaximumPercent: "100"}, + {name: "TwoSubnetsStated_Checked", subnetIDs: "subnet-abc123, subnet-def456", expectedIDs: []string{"subnet-abc123", "subnet-def456"}, pass: true}, + {name: "NoSubnetStated_NotChecked"}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { casting := statedCasting(&installation.Casting{}) - casting.Spec.MetaStore.Kind = tt.metaStoreKind - buf := bytes.NewBuffer(nil) - require.NoError(t, tt.template.Execute(buf, templateDataFor(t, casting))) + if tt.subnetIDs == "" { + delete(casting.Metadata.Annotations, installation.ECSPrivateSubnetIDs.Key) + } else { + casting.Metadata.Annotations[installation.ECSPrivateSubnetIDs.Key] = tt.subnetIDs + } - material, err := domain.NewJSONMaterial(buf.Bytes(), "service.tf.json") - require.NoError(t, err) + main := bytes.NewBuffer(nil) + require.NoError(t, mainTF.Execute(main, templateDataFor(t, casting))) - minimum, err := material.GetBytes("resource.aws_ecs_service." + tt.service + ".deployment_minimum_healthy_percent") + material, err := domain.NewJSONMaterial(main.Bytes(), "main.tf.json") require.NoError(t, err) - assert.Equal(t, tt.expectedMinimumPercent, string(minimum)) - maximum, err := material.GetBytes("resource.aws_ecs_service." + tt.service + ".deployment_maximum_percent") - require.NoError(t, err) - assert.Equal(t, tt.expectedMaximumPercent, string(maximum)) - }) - } -} + preconditions, err := material.GetBytes("resource.aws_service_discovery_private_dns_namespace.main.lifecycle.precondition") + if !tt.pass { + assert.Error(t, err, "an unstated subnet list emits no precondition") -// Replication reaches a node on the interserver port, and a node holding data -// takes longer to answer than the default start period allows. -func TestTelemetryStoreContainerIsReachableAndGivenTimeToStart(t *testing.T) { - buf := bytes.NewBuffer(nil) - require.NoError(t, telemetryStoreTF.Execute(buf, templateDataFor(t, statedCasting(&installation.Casting{})))) + _, err := material.GetBytes("data") + assert.Error(t, err, "an unstated subnet list emits no lookup") - material, err := domain.NewJSONMaterial(buf.Bytes(), "telemetrystore.tf.json") - require.NoError(t, err) + return + } - container := `locals.containers_telemetrystore_clickhouse_0_0.#(name=="signoz-telemetrystore-clickhouse-0-0")` + require.NoError(t, err) + assert.Equal(t, len(tt.expectedIDs), strings.Count(string(preconditions), `"condition"`)) - tests := []struct { - name string - path string - expectedValue string - }{ - {name: "NativePort_Mapped", path: container + `.portMappings.#(containerPort==9000).name`, expectedValue: "native"}, - {name: "HTTPPort_Mapped", path: container + `.portMappings.#(containerPort==8123).name`, expectedValue: "http"}, - {name: "PrometheusPort_Mapped", path: container + `.portMappings.#(containerPort==9363).name`, expectedValue: "prometheus"}, - {name: "InterserverPort_Mapped", path: container + `.portMappings.#(containerPort==9009).name`, expectedValue: "interserver"}, - {name: "StartPeriod_Stated", path: container + `.healthCheck.startPeriod`, expectedValue: "300"}, - } + for _, id := range tt.expectedIDs { + assert.Contains(t, string(preconditions), fmt.Sprintf("subnet %s assigns public IPs on launch", id)) + } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - value, err := material.GetBytes(tt.path) + lookups, err := material.GetBytes("data.aws_subnet") require.NoError(t, err) - assert.Equal(t, tt.expectedValue, string(value)) + assert.Equal(t, len(tt.expectedIDs), strings.Count(string(lookups), `"id"`)) }) } } -// Nothing chowns the task volume, so the agent writes as root and the server -// stays root to read what it wrote. -func TestTelemetryStoreRunsAsRoot(t *testing.T) { - buf := bytes.NewBuffer(nil) - require.NoError(t, telemetryStoreTF.Execute(buf, templateDataFor(t, statedCasting(&installation.Casting{})))) - - material, err := domain.NewJSONMaterial(buf.Bytes(), "telemetrystore.tf.json") - require.NoError(t, err) - - agent := `locals.containers_telemetrystore_clickhouse_0_0.#(name=="signoz-telemetrystore-appconfig-agent")` - server := `locals.containers_telemetrystore_clickhouse_0_0.#(name=="signoz-telemetrystore-clickhouse-0-0")` - scripts := `locals.containers_telemetrystore_clickhouse_0_0.#(name=="signoz-telemetrystore-user-scripts")` - - t.Run("AgentUser_Unstated", func(t *testing.T) { - _, err := material.GetBytes(agent + ".user") - assert.Error(t, err) - }) - - t.Run("ServerRunAsRoot_Stated", func(t *testing.T) { - value, err := material.GetBytes(server + `.environment.#(name=="CLICKHOUSE_RUN_AS_ROOT").value`) - require.NoError(t, err) - assert.Equal(t, "1", string(value)) - }) - - t.Run("UserScriptsCommand_Chownless", func(t *testing.T) { - value, err := material.GetBytes(scripts + ".command.0") - require.NoError(t, err) - assert.NotContains(t, string(value), "chown") - }) -} +// Terraform refuses an empty or malformed value at plan, so every variable the +// casting does not judge carries its own condition, and a default would race +// the tfvars. +func TestVariablesCarryValidationsAndNoDefault(t *testing.T) { + casting := statedCasting(&installation.Casting{}) + casting.Metadata.Annotations[installation.ECSTaskRoleARN.Key] = "arn:aws:iam::123456789012:role/task" + casting.Metadata.Annotations[installation.ECSTaskExecutionRoleARN.Key] = "arn:aws:iam::123456789012:role/exec" -// Nothing chowns the task volume, so the agent writes as root and the -// collector reads what it wrote. -func TestIngesterContainersRunAsRoot(t *testing.T) { buf := bytes.NewBuffer(nil) - require.NoError(t, ingesterTF.Execute(buf, templateDataFor(t, statedCasting(&installation.Casting{})))) + require.NoError(t, variablesTF.Execute(buf, templateDataFor(t, casting))) + + assert.NotContains(t, buf.String(), `"default"`) - material, err := domain.NewJSONMaterial(buf.Bytes(), "ingester.tf.json") + material, err := domain.NewJSONMaterial(buf.Bytes(), "variables.tf.json") require.NoError(t, err) tests := []struct { - name string - path string - expectedValue string + name string + variable string + expectedCondition string }{ - {name: "ContainerCount_Stated", path: "locals.containers_ingester.#", expectedValue: "2"}, - {name: "AgentName_Stated", path: "locals.containers_ingester.0.name", expectedValue: "signoz-ingester-appconfig-agent"}, - {name: "CollectorName_Stated", path: "locals.containers_ingester.1.name", expectedValue: "signoz-ingester"}, - {name: "AgentUser_Root", path: `locals.containers_ingester.#(name=="signoz-ingester-appconfig-agent").user`, expectedValue: "0"}, - {name: "CollectorUser_Root", path: `locals.containers_ingester.#(name=="signoz-ingester").user`, expectedValue: "0"}, + {name: "Region_Validated", variable: "aws_region", expectedCondition: `^[a-z]{2}(-gov)?-[a-z]+-[0-9]$`}, + {name: "ClusterARN_Validated", variable: "cluster_arn", expectedCondition: `^arn:aws:ecs:`}, + {name: "SubnetIDs_Validated", variable: "subnet_ids", expectedCondition: `^subnet-`}, + {name: "SecurityGroupIDs_Validated", variable: "security_group_ids", expectedCondition: `^sg-`}, + {name: "VPCID_Validated", variable: "vpc_id", expectedCondition: `^vpc-`}, + {name: "TaskRoleARN_Validated", variable: "task_role_arn", expectedCondition: `^arn:aws:iam::`}, + {name: "ExecutionRoleARN_Validated", variable: "execution_role_arn", expectedCondition: `^arn:aws:iam::`}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - value, err := material.GetBytes(tt.path) + condition, err := material.GetBytes("variable." + tt.variable + ".validation.condition") require.NoError(t, err) - assert.Equal(t, tt.expectedValue, string(value)) + assert.Contains(t, string(condition), tt.expectedCondition) }) } } -// The keeper writes a host path owned by root, and a node holding a raft log -// takes longer to answer than the default start period allows. -func TestTelemetryKeeperContainerRunsAsRootAndIsGivenTimeToStart(t *testing.T) { - zookeeper := `locals.containers_telemetrykeeper_zookeeper_0.#(name=="signoz-telemetrykeeper-zookeeper-0")` - clickhouseKeeper := `locals.containers_telemetrykeeper_clickhousekeeper_0.#(name=="signoz-telemetrykeeper-clickhousekeeper-0")` - - tests := []struct { - name string - kind installation.TelemetryKeeperKind - path string - expectedValue string - }{ - {name: "ZookeeperUser_Root", kind: installation.TelemetryKeeperKindZookeeper, path: zookeeper + ".user", expectedValue: "0"}, - {name: "ZookeeperPrometheusPort_Mapped", kind: installation.TelemetryKeeperKindZookeeper, path: zookeeper + `.portMappings.#(containerPort==9141).name`, expectedValue: "prometheus"}, - {name: "ZookeeperStartPeriod_Stated", kind: installation.TelemetryKeeperKindZookeeper, path: zookeeper + ".healthCheck.startPeriod", expectedValue: "300"}, - {name: "ClickhouseKeeperStartPeriod_Stated", kind: installation.TelemetryKeeperKindClickhouseKeeper, path: clickhouseKeeper + ".healthCheck.startPeriod", expectedValue: "300"}, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - casting := statedCasting(&installation.Casting{}) - casting.Spec.TelemetryKeeper.Kind = tt.kind - - buf := bytes.NewBuffer(nil) - require.NoError(t, telemetryKeeperTF.Execute(buf, templateDataFor(t, casting))) - - material, err := domain.NewJSONMaterial(buf.Bytes(), "telemetrykeeper.tf.json") - require.NoError(t, err) +// No name is computed for a role this stack does not create. +func TestTfvarsCarriesEveryStatedValue(t *testing.T) { + statedRoles := statedCasting(&installation.Casting{}) + statedRoles.Metadata.Annotations[installation.ECSTaskRoleARN.Key] = "arn:aws:iam::123456789012:role/task" + statedRoles.Metadata.Annotations[installation.ECSTaskExecutionRoleARN.Key] = "arn:aws:iam::123456789012:role/exec" - value, err := material.GetBytes(tt.path) - require.NoError(t, err) - assert.Equal(t, tt.expectedValue, string(value)) - }) - } -} + tfvars := `{ + "aws_region": "us-east-1", + "cluster_arn": "arn:aws:ecs:us-east-1:123456789012:cluster/test", + "subnet_ids": ["subnet-abc123", "subnet-def456"], + "security_group_ids": ["sg-abc123"], + "vpc_id": "vpc-abc123", + %s + }` -// SigNoz carries a node ordinal whatever the metadata store is, so only the -// sqlite disk and its rollout tell the two apart. -func TestSignozIsAlwaysANode(t *testing.T) { tests := []struct { - name string - metaStoreKind installation.MetaStoreKind - expectedHostPath string - expectedMinimumPercent string - expectedMaximumPercent string + name string + casting *installation.Casting + expectedRoles string }{ { - name: "Postgres_Diskless", - metaStoreKind: installation.MetaStoreKindPostgres, - expectedMinimumPercent: "100", - expectedMaximumPercent: "200", + name: "CreatedRoles_Named", + casting: statedCasting(&installation.Casting{}), + expectedRoles: `"task_role_name": "signoz-installation-iam-task", "execution_role_name": "signoz-installation-iam-exec"`, }, { - name: "Sqlite_HoldsADisk", - metaStoreKind: installation.MetaStoreKindSQLite, - expectedHostPath: "/var/lib/foundry/signoz/metastore/sqlite/0", - expectedMinimumPercent: "0", - expectedMaximumPercent: "100", + name: "StatedRoles_Carried", + casting: statedRoles, + expectedRoles: `"task_role_arn": "arn:aws:iam::123456789012:role/task", "execution_role_arn": "arn:aws:iam::123456789012:role/exec"`, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - casting := statedCasting(&installation.Casting{}) - casting.Spec.MetaStore.Kind = tt.metaStoreKind - buf := bytes.NewBuffer(nil) - require.NoError(t, signozTF.Execute(buf, templateDataFor(t, casting))) - - material, err := domain.NewJSONMaterial(buf.Bytes(), "signoz.tf.json") - require.NoError(t, err) - - expected := map[string]string{ - "locals.containers_signoz_0.0.name": "signoz-signoz-0", - "locals.containers_signoz_0.0.healthCheck.startPeriod": "300", - "resource.aws_ecs_task_definition.signoz_0.family": "signoz-signoz-0", - "resource.aws_service_discovery_service.signoz_0.name": "signoz-0", - "resource.aws_ecs_service.signoz_0.name": "signoz-signoz-0", - "resource.aws_ecs_service.signoz_0.task_definition": "${aws_ecs_task_definition.signoz_0.arn}", - "resource.aws_ecs_service.signoz_0.desired_count": "1", - "resource.aws_ecs_service.signoz_0.deployment_minimum_healthy_percent": tt.expectedMinimumPercent, - "resource.aws_ecs_service.signoz_0.deployment_maximum_percent": tt.expectedMaximumPercent, - } - - if tt.expectedHostPath != "" { - expected["resource.aws_ecs_task_definition.signoz_0.volume.0.host_path"] = tt.expectedHostPath - } - - for path, want := range expected { - value, err := material.GetBytes(path) + require.NoError(t, tfarsTF.Execute(buf, templateDataFor(t, tt.casting))) - require.NoError(t, err, "reading %s", path) - assert.Equal(t, want, string(value), "at %s", path) - } - - if tt.expectedHostPath == "" { - _, err := material.GetBytes("resource.aws_ecs_task_definition.signoz_0.volume") - assert.Error(t, err) - } - - outputs := bytes.NewBuffer(nil) - require.NoError(t, outputsTF.Execute(outputs, templateDataFor(t, casting))) - - assert.Contains(t, outputs.String(), "${aws_ecs_service.signoz_0.name}") - assert.Contains(t, outputs.String(), "${aws_ecs_service.signoz_0.id}") - }) - } -} - -// A postgres holding a data directory takes longer to answer than the default -// start period allows. -func TestMetaStoreIsANodeGivenTimeToStart(t *testing.T) { - casting := statedCasting(&installation.Casting{}) - casting.Spec.MetaStore.Kind = installation.MetaStoreKindPostgres - - buf := bytes.NewBuffer(nil) - require.NoError(t, metaStoreTF.Execute(buf, templateDataFor(t, casting))) - - material, err := domain.NewJSONMaterial(buf.Bytes(), "metastore.tf.json") - require.NoError(t, err) - - tests := []struct { - name string - path string - expectedValue string - }{ - {name: "StartPeriod_Stated", path: "locals.containers_metastore_postgres_0.0.healthCheck.startPeriod", expectedValue: "300"}, - {name: "ContainerName_Stated", path: "locals.containers_metastore_postgres_0.0.name", expectedValue: "signoz-metastore-postgres-0"}, - {name: "Family_Stated", path: "resource.aws_ecs_task_definition.metastore_postgres_0.family", expectedValue: "signoz-metastore-postgres-0"}, - {name: "DNS_Stated", path: "resource.aws_service_discovery_service.metastore_postgres_0.name", expectedValue: "metastore-postgres-0"}, - {name: "ServiceName_Stated", path: "resource.aws_ecs_service.metastore_postgres_0.name", expectedValue: "signoz-metastore-postgres-0"}, - {name: "HostPath_Stated", path: "resource.aws_ecs_task_definition.metastore_postgres_0.volume.0.host_path", expectedValue: "/var/lib/foundry/signoz/metastore/postgres/0"}, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - value, err := material.GetBytes(tt.path) - require.NoError(t, err, "reading %s", tt.path) - assert.Equal(t, tt.expectedValue, string(value)) + assert.JSONEq(t, fmt.Sprintf(tfvars, tt.expectedRoles), buf.String()) }) } } @@ -763,31 +308,10 @@ func TestStatefulComponentsForgeAServicePerNode(t *testing.T) { casting *installation.Casting expectedServices []string }{ - { - name: "TelemetryStore_ServicePerNode", template: telemetryStoreTF, casting: store, - expectedServices: []string{ - "signoz-telemetrystore-clickhouse-0-0", - "signoz-telemetrystore-clickhouse-0-1", - "signoz-telemetrystore-clickhouse-1-0", - "signoz-telemetrystore-clickhouse-1-1", - }, - }, - { - name: "TelemetryKeeper_ServicePerNode", template: telemetryKeeperTF, casting: keeper, - expectedServices: []string{ - "signoz-telemetrykeeper-clickhousekeeper-0", - "signoz-telemetrykeeper-clickhousekeeper-1", - "signoz-telemetrykeeper-clickhousekeeper-2", - }, - }, - { - name: "MetaStore_ServicePerNode", template: metaStoreTF, casting: metaStore, - expectedServices: []string{"signoz-metastore-postgres-0", "signoz-metastore-postgres-1"}, - }, - { - name: "Signoz_ServicePerNode", template: signozTF, casting: signoz, - expectedServices: []string{"signoz-signoz-0", "signoz-signoz-1"}, - }, + {name: "TelemetryStore_ServicePerNode", template: telemetryStoreTF, casting: store, expectedServices: []string{"signoz-telemetrystore-clickhouse-0-0", "signoz-telemetrystore-clickhouse-0-1", "signoz-telemetrystore-clickhouse-1-0", "signoz-telemetrystore-clickhouse-1-1"}}, + {name: "TelemetryKeeper_ServicePerNode", template: telemetryKeeperTF, casting: keeper, expectedServices: []string{"signoz-telemetrykeeper-clickhousekeeper-0", "signoz-telemetrykeeper-clickhousekeeper-1", "signoz-telemetrykeeper-clickhousekeeper-2"}}, + {name: "MetaStore_ServicePerNode", template: metaStoreTF, casting: metaStore, expectedServices: []string{"signoz-metastore-postgres-0", "signoz-metastore-postgres-1"}}, + {name: "Signoz_ServicePerNode", template: signozTF, casting: signoz, expectedServices: []string{"signoz-signoz-0", "signoz-signoz-1"}}, } for _, tt := range tests { @@ -873,21 +397,13 @@ func TestOutputsEnumerateEveryNode(t *testing.T) { material, err := domain.NewJSONMaterial(buf.Bytes(), "outputs.tf.json") require.NoError(t, err) - tests := []struct { - name string - path string - expectedCount int - }{ - {name: "TelemetryStore_PerNode", path: "output.telemetrystore_service_names.value", expectedCount: 4}, - {name: "TelemetryKeeper_PerNode", path: "output.telemetrykeeper_service_names.value", expectedCount: 3}, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - names, err := material.GetStringSlice(tt.path) - require.NoError(t, err, "reading %s", tt.path) - assert.Len(t, names, tt.expectedCount) - }) + for path, expectedCount := range map[string]int{ + "output.telemetrystore_service_names.value": 4, + "output.telemetrykeeper_service_names.value": 3, + } { + names, err := material.GetStringSlice(path) + require.NoError(t, err, "reading %s", path) + assert.Len(t, names, expectedCount, path) } for _, path := range []string{"output.metastore_service_name.value", "output.signoz_service_name.value", "output.signoz_service_arn.value"} { From 36a91407aed0f7b7b9572a196ed614b7998b72c6 Mon Sep 17 00:00:00 2001 From: Nagesh Bansal Date: Fri, 11 Sep 2026 16:19:01 +0530 Subject: [PATCH 3/4] kick in ci From a7a15424ab9b1111088268f45b4305feb0dc6e3b Mon Sep 17 00:00:00 2001 From: Nagesh Bansal Date: Fri, 11 Sep 2026 16:29:50 +0530 Subject: [PATCH 4/4] fix(casting/ecs): read the ping body in the ClickHouse healthcheck wget --spider closes the connection before the body arrives, so ClickHouse logs a broken-pipe error with a stack trace on every probe while the check stays green. Reading the body with -O- ends the noise, and 0.0.0.0 avoids localhost resolving to ::1 against a server that listens on IPv4 only. Same shape as the coolify casting. Carries #194. Refs SigNoz/signoz#5140 --- .../ecs/ec2/terraform/pours/deployment/telemetrystore.tf.json | 2 +- .../ecsterraformcasting/templates/telemetrystore.tf.json.gotmpl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/examples/ecs/ec2/terraform/pours/deployment/telemetrystore.tf.json b/docs/examples/ecs/ec2/terraform/pours/deployment/telemetrystore.tf.json index 963a2614..a725d984 100644 --- a/docs/examples/ecs/ec2/terraform/pours/deployment/telemetrystore.tf.json +++ b/docs/examples/ecs/ec2/terraform/pours/deployment/telemetrystore.tf.json @@ -102,7 +102,7 @@ "cpu": 1024, "memoryReservation": 512, "healthCheck": { - "command": ["CMD-SHELL", "wget --spider -q http://localhost:8123/ping || exit 1"], + "command": ["CMD-SHELL", "wget -q -O- 0.0.0.0:8123/ping || exit 1"], "interval": 30, "timeout": 5, "retries": 3, diff --git a/internal/casting/ecsterraformcasting/templates/telemetrystore.tf.json.gotmpl b/internal/casting/ecsterraformcasting/templates/telemetrystore.tf.json.gotmpl index 78349244..37c00a26 100644 --- a/internal/casting/ecsterraformcasting/templates/telemetrystore.tf.json.gotmpl +++ b/internal/casting/ecsterraformcasting/templates/telemetrystore.tf.json.gotmpl @@ -125,7 +125,7 @@ "cpu": 1024, "memoryReservation": 512, "healthCheck": { - "command": ["CMD-SHELL", "wget --spider -q http://localhost:8123/ping || exit 1"], + "command": ["CMD-SHELL", "wget -q -O- 0.0.0.0:8123/ping || exit 1"], "interval": 30, "timeout": 5, "retries": 3,