diff --git a/docs/examples/ecs/ec2/terraform/README.md b/docs/examples/ecs/ec2/terraform/README.md index 36c8571e..a48063c3 100644 --- a/docs/examples/ecs/ec2/terraform/README.md +++ b/docs/examples/ecs/ec2/terraform/README.md @@ -18,12 +18,10 @@ Components: - OTel Collector (ingester) - Schema migrator, run once on Fargate -The example beside this file is [`byo/`](byo/): the cluster is yours, and every object it is made of is stated on the casting. - > [!IMPORTANT] > Mount a durable volume at `/var/lib/foundry` on the instances that run stateful services, otherwise data lives on the instance's root disk. This deployment sets no placement constraints; pin stateful services to their instances yourself. > -> One node per component: cluster shard and replica counts are not honoured yet. +> Cluster shard and replica counts are honoured: every node is its own service. ## Prerequisites @@ -64,6 +62,31 @@ Component configuration is delivered through AWS AppConfig and reloaded in place Stateful data lives on the instance running the task, under `/var/lib/foundry////`, for example `/var/lib/foundry/signoz/telemetrystore/clickhouse/0-0`. +Shard and replica counts become nodes, and every node is its own ECS service: + +```yaml +spec: + telemetrystore: + spec: + cluster: + shards: 2 + replicas: 1 # copies beside each shard, so four nodes here + telemetrykeeper: + spec: + cluster: + replicas: 3 # keep it odd, for quorum + signoz: + spec: + cluster: + replicas: 2 # under sqlite each node holds its own database + metastore: + spec: + cluster: + replicas: 2 +``` + +Nothing spreads the nodes across instances, so pin them yourself if that matters. + ## Deploy ```bash @@ -109,9 +132,9 @@ One root module, one file per component, and no child modules. It creates: | Resource | Count | | --- | --- | | `aws_service_discovery_private_dns_namespace` | 1, named `-installation.local` | -| `aws_service_discovery_service` | one per component | -| `aws_ecs_service` | one per component | -| `aws_ecs_task_definition` | one per component, plus the migrator | +| `aws_service_discovery_service` | one per node | +| `aws_ecs_service` | one per node | +| `aws_ecs_task_definition` | one per node, plus the migrator | | `aws_appconfig_*` | one set per config file | | `aws_iam_role` | 2, unless stated on the casting | @@ -145,11 +168,11 @@ Components resolve each other inside `-installation.local`: | Component | DNS name | Ports | | --- | --- | --- | -| ClickHouse Keeper | `telemetrykeeper-clickhousekeeper-0` | 9181 client, 9234 raft | -| ZooKeeper | `telemetrykeeper-zookeeper-0` | 2181 client, 2888 raft, 3888 election, 9141 metrics | -| ClickHouse | `telemetrystore-clickhouse-0-0` | 9000 native, 8123 HTTP, 9009 interserver, 9363 metrics | -| PostgreSQL | `metastore-postgres-0` | 5432 | -| SigNoz | `signoz-0` | 8080 API, 4320 OpAMP | +| ClickHouse Keeper | `telemetrykeeper-clickhousekeeper-` | 9181 client, 9234 raft | +| ZooKeeper | `telemetrykeeper-zookeeper-` | 2181 client, 2888 raft, 3888 election, 9141 metrics | +| ClickHouse | `telemetrystore-clickhouse--` | 9000 native, 8123 HTTP, 9009 interserver, 9363 metrics | +| PostgreSQL | `metastore-postgres-` | 5432 | +| SigNoz | `signoz-` | 8080 API, 4320 OpAMP | | Ingester | `ingester` | 4317 gRPC, 4318 HTTP | | MCP | `mcp` | 8000 | diff --git a/docs/examples/ecs/ec2/terraform/pours/deployment/telemetrystore.tf.json b/docs/examples/ecs/ec2/terraform/pours/deployment/telemetrystore.tf.json index 963a2614..a725d984 100644 --- a/docs/examples/ecs/ec2/terraform/pours/deployment/telemetrystore.tf.json +++ b/docs/examples/ecs/ec2/terraform/pours/deployment/telemetrystore.tf.json @@ -102,7 +102,7 @@ "cpu": 1024, "memoryReservation": 512, "healthCheck": { - "command": ["CMD-SHELL", "wget --spider -q http://localhost:8123/ping || exit 1"], + "command": ["CMD-SHELL", "wget -q -O- 0.0.0.0:8123/ping || exit 1"], "interval": 30, "timeout": 5, "retries": 3, diff --git a/internal/casting/ecsterraformcasting/embed_test.go b/internal/casting/ecsterraformcasting/embed_test.go index 603f7d58..cf145b9e 100644 --- a/internal/casting/ecsterraformcasting/embed_test.go +++ b/internal/casting/ecsterraformcasting/embed_test.go @@ -9,6 +9,7 @@ import ( "strings" "testing" + "github.com/signoz/foundry/api/v1alpha1" "github.com/signoz/foundry/api/v1alpha1/installation" "github.com/signoz/foundry/internal/domain" "github.com/stretchr/testify/assert" @@ -37,99 +38,114 @@ func templateDataFor(t *testing.T, casting *installation.Casting) templateData { return data } -func TestNotEmptyAndValid(t *testing.T) { +func TestEveryTemplateRendersValidJSON(t *testing.T) { + data := templateDataFor(t, installation.Default(&installation.Casting{})) + templates := map[string]*domain.Template{ - "versionsTF": versionsTF, - "backendTF": backendTF, - "providersTF": providersTF, - "mainTF": mainTF, - "variablesTF": variablesTF, - "outputsTF": outputsTF, - "telemetryKeeperTF": telemetryKeeperTF, - "telemetryStoreTF": telemetryStoreTF, - "migratorTF": migratorTF, - "metaStoreTF": metaStoreTF, - "signozTF": signozTF, - "ingesterTF": ingesterTF, - "mcpTF": mcpTF, + "versions.tf.json": versionsTF, + "backend.tf.json": backendTF, + "providers.tf.json": providersTF, + "main.tf.json": mainTF, + "variables.tf.json": variablesTF, + "outputs.tf.json": outputsTF, + "terraform.tfvars.json": tfarsTF, + "telemetrykeeper.tf.json": telemetryKeeperTF, + "telemetrystore.tf.json": telemetryStoreTF, + "migrator.tf.json": migratorTF, + "metastore.tf.json": metaStoreTF, + "signoz.tf.json": signozTF, + "ingester.tf.json": ingesterTF, + "mcp.tf.json": mcpTF, } for name, tmpl := range templates { - assert.NotEmpty(t, tmpl, "%s should not be empty", name) buf := bytes.NewBuffer(nil) - err := tmpl.Execute(buf, nil) - assert.NoError(t, err, "error executing %s", name) - assert.NotEmpty(t, buf.String(), "%s output should not be empty", name) + require.NoError(t, tmpl.Execute(buf, data), name) + + _, err := domain.NewJSONMaterial(buf.Bytes(), name) + require.NoError(t, err, name) } } -// A default in the variables would race the tfvars. -func TestTfvarsTemplateCarriesEveryValue(t *testing.T) { - buf := bytes.NewBuffer(nil) - require.NoError(t, tfarsTF.Execute(buf, templateDataFor(t, statedCasting(&installation.Casting{})))) +func TestTemplateDataResolution(t *testing.T) { + complete := statedCasting(&installation.Casting{}).Metadata.Annotations - assert.JSONEq(t, `{ - "aws_region": "us-east-1", - "cluster_arn": "arn:aws:ecs:us-east-1:123456789012:cluster/test", - "subnet_ids": ["subnet-abc123", "subnet-def456"], - "security_group_ids": ["sg-abc123"], - "vpc_id": "vpc-abc123", - "task_role_name": "signoz-installation-iam-task", - "execution_role_name": "signoz-installation-iam-exec" - }`, buf.String()) -} + malformed := map[string]string{} + maps.Copy(malformed, complete) + malformed[installation.ECSPrivateSubnetIDs.Key] = " , ," -// No name is computed for a role this stack does not create. -func TestTfvarsTemplateCarriesStatedRoles(t *testing.T) { - casting := statedCasting(&installation.Casting{}) - casting.Metadata.Annotations[installation.ECSTaskRoleARN.Key] = "arn:aws:iam::123456789012:role/task" - casting.Metadata.Annotations[installation.ECSTaskExecutionRoleARN.Key] = "arn:aws:iam::123456789012:role/exec" + tests := []struct { + name string + annotations map[string]string + expectedRegion string + expectedStated bool + expectedMessage string + pass bool + }{ + {name: "AllStated_Valid", annotations: complete, expectedRegion: "us-east-1", expectedStated: true, pass: true}, + {name: "NothingStated_Valid", annotations: nil, pass: true}, + {name: "SubnetIDsMalformed_Invalid", annotations: malformed, expectedMessage: "no ids found"}, + } - buf := bytes.NewBuffer(nil) - require.NoError(t, tfarsTF.Execute(buf, templateDataFor(t, casting))) + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + casting := installation.Default(&installation.Casting{}) + casting.Metadata.Annotations = tt.annotations - assert.JSONEq(t, `{ - "aws_region": "us-east-1", - "cluster_arn": "arn:aws:ecs:us-east-1:123456789012:cluster/test", - "subnet_ids": ["subnet-abc123", "subnet-def456"], - "security_group_ids": ["sg-abc123"], - "vpc_id": "vpc-abc123", - "task_role_arn": "arn:aws:iam::123456789012:role/task", - "execution_role_arn": "arn:aws:iam::123456789012:role/exec" - }`, buf.String()) + data, err := New(slog.New(slog.DiscardHandler)).templateData(*casting) + if !tt.pass { + assert.ErrorContains(t, err, tt.expectedMessage) + + return + } + + require.NoError(t, err) + assert.Equal(t, tt.expectedRegion, data.Region) + + for _, reference := range []Reference{data.Cluster, data.VPC, data.Subnets, data.SecurityGroup} { + assert.Equal(t, tt.expectedStated, reference.IsStated()) + } + + assert.False(t, data.TaskRole.IsStated()) + }) + } } -func TestStatedObjectsAreResolvedThroughLocals(t *testing.T) { - data := templateDataFor(t, statedCasting(&installation.Casting{})) +// Sqlite is a file the signoz task holds, so it is no service of its own. +func TestForgeSelectsComponents(t *testing.T) { + sqlite := statedCasting(&installation.Casting{}) + sqlite.Spec.MetaStore.Kind = installation.MetaStoreKindSQLite - variables := bytes.NewBuffer(nil) - require.NoError(t, variablesTF.Execute(variables, data)) + withMCP := statedCasting(&installation.Casting{}) + withMCP.Spec.MCP.Spec.Enabled = v1alpha1.BoolPtr(true) - for _, expected := range []string{ - `"cluster_arn"`, - `"subnet_ids"`, - `"security_group_ids"`, - `"vpc_id"`, - `"task_role_name"`, - `"execution_role_name"`, - } { - assert.Contains(t, variables.String(), expected) + tests := []struct { + name string + casting *installation.Casting + path string + expectedForged bool + }{ + {name: "Postgres_MetaStoreForged", casting: statedCasting(&installation.Casting{}), path: "metastore.tf.json", expectedForged: true}, + {name: "Sqlite_MetaStoreUnforged", casting: sqlite, path: "metastore.tf.json"}, + {name: "MCPEnabled_Forged", casting: withMCP, path: "mcp.tf.json", expectedForged: true}, + {name: "MCPDisabled_Unforged", casting: statedCasting(&installation.Casting{}), path: "mcp.tf.json"}, + {name: "NothingStated_Forged", casting: installation.Default(&installation.Casting{}), path: "main.tf.json", expectedForged: true}, } - assert.NotContains(t, variables.String(), `"default"`) + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + materials, err := New(slog.New(slog.DiscardHandler)).Forge(context.Background(), *tt.casting, "") + require.NoError(t, err) - main := bytes.NewBuffer(nil) - require.NoError(t, mainTF.Execute(main, data)) + forged := false + for _, material := range materials { + if strings.Contains(material.Path(), tt.path) { + forged = true + } + } - out := main.String() - for _, expected := range []string{ - `"cluster_arn": "${var.cluster_arn}"`, - `"subnet_ids": "${var.subnet_ids}"`, - `"security_group_ids": "${var.security_group_ids}"`, - `"vpc_id": "${var.vpc_id}"`, - `"name": "${var.task_role_name}"`, - } { - assert.Contains(t, out, expected) + assert.Equal(t, tt.expectedForged, forged) + }) } } @@ -137,21 +153,12 @@ func TestStatedObjectsAreResolvedThroughLocals(t *testing.T) { // cannot use. An empty "data" object is a root terraform refuses outright. func TestStatedSubnetsAreCheckedAtPlan(t *testing.T) { tests := []struct { - name string - subnetIDs string - expectedLookups []string - expectedConditions []string - expectedMessages []string - pass bool + name string + subnetIDs string + expectedIDs []string + pass bool }{ - { - name: "TwoSubnetsStated_Checked", - subnetIDs: "subnet-abc123, subnet-def456", - expectedLookups: []string{"${var.subnet_ids[0]}", "${var.subnet_ids[1]}"}, - expectedConditions: []string{"${!data.aws_subnet.tasks_0.map_public_ip_on_launch}", "${!data.aws_subnet.tasks_1.map_public_ip_on_launch}"}, - expectedMessages: []string{"subnet subnet-abc123 assigns public IPs on launch", "subnet subnet-def456 assigns public IPs on launch"}, - pass: true, - }, + {name: "TwoSubnetsStated_Checked", subnetIDs: "subnet-abc123, subnet-def456", expectedIDs: []string{"subnet-abc123", "subnet-def456"}, pass: true}, {name: "NoSubnetStated_NotChecked"}, } @@ -171,142 +178,34 @@ func TestStatedSubnetsAreCheckedAtPlan(t *testing.T) { material, err := domain.NewJSONMaterial(main.Bytes(), "main.tf.json") require.NoError(t, err) + preconditions, err := material.GetBytes("resource.aws_service_discovery_private_dns_namespace.main.lifecycle.precondition") if !tt.pass { + assert.Error(t, err, "an unstated subnet list emits no precondition") + _, err := material.GetBytes("data") assert.Error(t, err, "an unstated subnet list emits no lookup") - _, err = material.GetBytes("resource.aws_service_discovery_private_dns_namespace.main.lifecycle") - assert.Error(t, err, "an unstated subnet list emits no precondition") - return } - for index, expected := range tt.expectedLookups { - id, err := material.GetBytes(fmt.Sprintf("data.aws_subnet.tasks_%d.id", index)) - require.NoError(t, err) - assert.Equal(t, expected, string(id)) - } - - preconditions, err := material.GetBytes("resource.aws_service_discovery_private_dns_namespace.main.lifecycle.precondition") require.NoError(t, err) + assert.Equal(t, len(tt.expectedIDs), strings.Count(string(preconditions), `"condition"`)) - rendered := string(preconditions) - assert.Equal(t, len(tt.expectedConditions), strings.Count(rendered, `"condition"`)) - - for _, expected := range tt.expectedConditions { - assert.Contains(t, rendered, expected) - } - - for _, expected := range tt.expectedMessages { - assert.Contains(t, rendered, expected) - } - }) - } -} - -func TestStatedRoleIsNotCreated(t *testing.T) { - casting := statedCasting(&installation.Casting{}) - casting.Metadata.Annotations[installation.ECSTaskRoleARN.Key] = "arn:aws:iam::123456789012:role/task" - casting.Metadata.Annotations[installation.ECSTaskExecutionRoleARN.Key] = "arn:aws:iam::123456789012:role/exec" - - data := templateDataFor(t, casting) - - main := bytes.NewBuffer(nil) - require.NoError(t, mainTF.Execute(main, data)) - - out := main.String() - assert.NotContains(t, out, "aws_iam_role") - assert.NotContains(t, out, "appconfig:StartConfigurationSession") - assert.Contains(t, out, `"task_role_arn": "${var.task_role_arn}"`) -} - -func TestReferenceIsStated(t *testing.T) { - tests := []struct { - name string - reference Reference - pass bool - }{ - {name: "Stated_Valid", reference: Reference{Stated: "arn:aws:ecs:us-east-1:1:cluster/x"}, pass: true}, - {name: "StatedIDs_Valid", reference: Reference{StatedIDs: []string{"subnet-a"}}, pass: true}, - {name: "Empty_Invalid", reference: Reference{}, pass: false}, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - assert.Equal(t, tt.pass, tt.reference.IsStated()) - }) - } -} - -func TestTemplateDataResolution(t *testing.T) { - complete := statedCasting(&installation.Casting{}).Metadata.Annotations - - malformed := map[string]string{} - maps.Copy(malformed, complete) - malformed[installation.ECSPrivateSubnetIDs.Key] = " , ," - - tests := []struct { - name string - annotations map[string]string - expectedRegion string - expectedStated bool - expectedMessage string - pass bool - }{ - {name: "AllStated_Valid", annotations: complete, expectedRegion: "us-east-1", expectedStated: true, pass: true}, - {name: "NothingStated_Valid", annotations: nil, pass: true}, - {name: "SubnetIDsMalformed_Invalid", annotations: malformed, expectedMessage: "no ids found"}, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - casting := installation.Default(&installation.Casting{}) - casting.Metadata.Annotations = tt.annotations - - data, err := New(slog.New(slog.DiscardHandler)).templateData(*casting) - if !tt.pass { - assert.ErrorContains(t, err, tt.expectedMessage) - - return + for _, id := range tt.expectedIDs { + assert.Contains(t, string(preconditions), fmt.Sprintf("subnet %s assigns public IPs on launch", id)) } + lookups, err := material.GetBytes("data.aws_subnet") require.NoError(t, err) - assert.Equal(t, tt.expectedRegion, data.Region) - - for _, reference := range []Reference{data.Cluster, data.VPC, data.Subnets, data.SecurityGroup} { - assert.Equal(t, tt.expectedStated, reference.IsStated()) - } - - assert.False(t, data.TaskRole.IsStated()) + assert.Equal(t, len(tt.expectedIDs), strings.Count(string(lookups), `"id"`)) }) } } -// The platform judges its own inputs, so an unstated casting still forges. -func TestUnstatedCastingForges(t *testing.T) { - casting := installation.Default(&installation.Casting{}) - - materials, err := New(slog.New(slog.DiscardHandler)).Forge(context.Background(), *casting, "") - require.NoError(t, err) - assert.NotEmpty(t, materials) - - buf := bytes.NewBuffer(nil) - require.NoError(t, tfarsTF.Execute(buf, templateDataFor(t, casting))) - - assert.JSONEq(t, `{ - "aws_region": "", - "cluster_arn": "", - "subnet_ids": [], - "security_group_ids": [], - "vpc_id": "", - "task_role_name": "signoz-installation-iam-task", - "execution_role_name": "signoz-installation-iam-exec" - }`, buf.String()) -} - // Terraform refuses an empty or malformed value at plan, so every variable the -// casting does not judge carries its own condition. -func TestVariablesCarryValidations(t *testing.T) { +// casting does not judge carries its own condition, and a default would race +// the tfvars. +func TestVariablesCarryValidationsAndNoDefault(t *testing.T) { casting := statedCasting(&installation.Casting{}) casting.Metadata.Annotations[installation.ECSTaskRoleARN.Key] = "arn:aws:iam::123456789012:role/task" casting.Metadata.Annotations[installation.ECSTaskExecutionRoleARN.Key] = "arn:aws:iam::123456789012:role/exec" @@ -314,6 +213,8 @@ func TestVariablesCarryValidations(t *testing.T) { buf := bytes.NewBuffer(nil) require.NoError(t, variablesTF.Execute(buf, templateDataFor(t, casting))) + assert.NotContains(t, buf.String(), `"default"`) + material, err := domain.NewJSONMaterial(buf.Bytes(), "variables.tf.json") require.NoError(t, err) @@ -340,400 +241,173 @@ func TestVariablesCarryValidations(t *testing.T) { } } -// Sqlite is a file the signoz task holds, so it is no service of its own. -func TestSqliteForgesNoMetaStoreService(t *testing.T) { - casting := statedCasting(&installation.Casting{}) - casting.Spec.MetaStore.Kind = installation.MetaStoreKindSQLite - - materials, err := New(slog.New(slog.DiscardHandler)).Forge(context.Background(), *casting, "") - require.NoError(t, err) - - for _, material := range materials { - assert.NotContains(t, material.Path(), "metastore.tf.json") - } - - outputs := bytes.NewBuffer(nil) - require.NoError(t, outputsTF.Execute(outputs, templateDataFor(t, casting))) - - assert.NotContains(t, outputs.String(), "metastore_service_name") -} - -func TestPostgresForgesAMetaStoreService(t *testing.T) { - casting := statedCasting(&installation.Casting{}) - casting.Spec.MetaStore.Kind = installation.MetaStoreKindPostgres - - materials, err := New(slog.New(slog.DiscardHandler)).Forge(context.Background(), *casting, "") - require.NoError(t, err) - - forged := false - for _, material := range materials { - if strings.Contains(material.Path(), "metastore.tf.json") { - forged = true - } - } - - assert.True(t, forged) - - outputs := bytes.NewBuffer(nil) - require.NoError(t, outputsTF.Execute(outputs, templateDataFor(t, casting))) - - assert.Contains(t, outputs.String(), "metastore_service_name") -} - -// A container that logs without the labels reaches SigNoz carrying only a -// container id. Fargate refuses json-file, so the migrator carries no block. -func TestEveryEC2ContainerLogsWithLabels(t *testing.T) { - data := templateDataFor(t, statedCasting(&installation.Casting{})) - - for name, tmpl := range map[string]*domain.Template{ - "signozTF": signozTF, - "ingesterTF": ingesterTF, - "metaStoreTF": metaStoreTF, - "mcpTF": mcpTF, - "telemetryStoreTF": telemetryStoreTF, - "telemetryKeeperTF": telemetryKeeperTF, - } { - buf := bytes.NewBuffer(nil) - require.NoError(t, tmpl.Execute(buf, data), name) - - rendered := buf.String() - - assert.Equal(t, strings.Count(rendered, `"image":`), strings.Count(rendered, `"logConfiguration":`), - "%s: every container logs, or none of them is attributed", name) - assert.Contains(t, rendered, "com.amazonaws.ecs.task-definition-family,com.amazonaws.ecs.container-name,com.amazonaws.ecs.task-arn,com.amazonaws.ecs.cluster", name) - - // Without rotation docker fills the instance disk. - assert.Contains(t, rendered, `"max-size": "10m"`, name) - } - - migrator := bytes.NewBuffer(nil) - require.NoError(t, migratorTF.Execute(migrator, data)) - - assert.NotContains(t, migrator.String(), "logConfiguration") -} - -// Nothing else puts a previous revision back. -func TestEveryServiceRollsBackABadRevision(t *testing.T) { - data := templateDataFor(t, statedCasting(&installation.Casting{})) - - for name, tmpl := range map[string]*domain.Template{ - "signozTF": signozTF, - "ingesterTF": ingesterTF, - "metaStoreTF": metaStoreTF, - "mcpTF": mcpTF, - "telemetryStoreTF": telemetryStoreTF, - "telemetryKeeperTF": telemetryKeeperTF, - } { - buf := bytes.NewBuffer(nil) - require.NoError(t, tmpl.Execute(buf, data), name) - - rendered := buf.String() - - assert.Equal(t, strings.Count(rendered, `"launch_type": "EC2"`), strings.Count(rendered, `"deployment_circuit_breaker"`), name) - assert.NotContains(t, rendered, "capacity_provider") - } -} - -// A CollectionAgent of the same name on the same account holds its own. -func TestAppConfigApplicationCarriesTheKind(t *testing.T) { - data := templateDataFor(t, statedCasting(&installation.Casting{})) - - main := bytes.NewBuffer(nil) - require.NoError(t, mainTF.Execute(main, data)) - - material, err := domain.NewJSONMaterial(main.Bytes(), "main.tf.json") - require.NoError(t, err) - - for path, expected := range map[string]string{ - "resource.aws_appconfig_application.main.name": "signoz-installation-appconfig", - "resource.aws_appconfig_deployment_strategy.main.name": "signoz-installation-appconfig-strategy", - - "resource.aws_service_discovery_private_dns_namespace.main.name": "signoz-installation.local", - } { - value, err := material.GetBytes(path) - - assert.NoError(t, err, "reading %s", path) - assert.Equal(t, expected, string(value), "at %s", path) - } - - // The sidecar prefetches by the application name, so a rename must reach it. - ingester := bytes.NewBuffer(nil) - require.NoError(t, ingesterTF.Execute(ingester, data)) +// No name is computed for a role this stack does not create. +func TestTfvarsCarriesEveryStatedValue(t *testing.T) { + statedRoles := statedCasting(&installation.Casting{}) + statedRoles.Metadata.Annotations[installation.ECSTaskRoleARN.Key] = "arn:aws:iam::123456789012:role/task" + statedRoles.Metadata.Annotations[installation.ECSTaskExecutionRoleARN.Key] = "arn:aws:iam::123456789012:role/exec" - assert.Contains(t, ingester.String(), "signoz-installation-appconfig:default:ingester") -} + tfvars := `{ + "aws_region": "us-east-1", + "cluster_arn": "arn:aws:ecs:us-east-1:123456789012:cluster/test", + "subnet_ids": ["subnet-abc123", "subnet-def456"], + "security_group_ids": ["sg-abc123"], + "vpc_id": "vpc-abc123", + %s + }` -// A task bound to a disk cannot be replaced before the one holding it stops. -func TestServiceRollsBeforeStopping(t *testing.T) { tests := []struct { - name string - template *domain.Template - metaStoreKind installation.MetaStoreKind - service string - expectedMinimumPercent string - expectedMaximumPercent string + name string + casting *installation.Casting + expectedRoles string }{ - {name: "Ingester_Rolling", template: ingesterTF, metaStoreKind: installation.MetaStoreKindPostgres, service: "ingester", expectedMinimumPercent: "100", expectedMaximumPercent: "200"}, - {name: "MCP_Rolling", template: mcpTF, metaStoreKind: installation.MetaStoreKindPostgres, service: "mcp", expectedMinimumPercent: "100", expectedMaximumPercent: "200"}, - {name: "SignozOnPostgres_Rolling", template: signozTF, metaStoreKind: installation.MetaStoreKindPostgres, service: "signoz_0", expectedMinimumPercent: "100", expectedMaximumPercent: "200"}, - {name: "SignozOnSqlite_StopFirst", template: signozTF, metaStoreKind: installation.MetaStoreKindSQLite, service: "signoz_0", expectedMinimumPercent: "0", expectedMaximumPercent: "100"}, - {name: "TelemetryStore_StopFirst", template: telemetryStoreTF, metaStoreKind: installation.MetaStoreKindPostgres, service: "telemetrystore_clickhouse_0_0", expectedMinimumPercent: "0", expectedMaximumPercent: "100"}, + { + name: "CreatedRoles_Named", + casting: statedCasting(&installation.Casting{}), + expectedRoles: `"task_role_name": "signoz-installation-iam-task", "execution_role_name": "signoz-installation-iam-exec"`, + }, + { + name: "StatedRoles_Carried", + casting: statedRoles, + expectedRoles: `"task_role_arn": "arn:aws:iam::123456789012:role/task", "execution_role_arn": "arn:aws:iam::123456789012:role/exec"`, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - casting := statedCasting(&installation.Casting{}) - casting.Spec.MetaStore.Kind = tt.metaStoreKind - buf := bytes.NewBuffer(nil) - require.NoError(t, tt.template.Execute(buf, templateDataFor(t, casting))) + require.NoError(t, tfarsTF.Execute(buf, templateDataFor(t, tt.casting))) - material, err := domain.NewJSONMaterial(buf.Bytes(), "service.tf.json") - require.NoError(t, err) - - minimum, err := material.GetBytes("resource.aws_ecs_service." + tt.service + ".deployment_minimum_healthy_percent") - require.NoError(t, err) - assert.Equal(t, tt.expectedMinimumPercent, string(minimum)) - - maximum, err := material.GetBytes("resource.aws_ecs_service." + tt.service + ".deployment_maximum_percent") - require.NoError(t, err) - assert.Equal(t, tt.expectedMaximumPercent, string(maximum)) + assert.JSONEq(t, fmt.Sprintf(tfvars, tt.expectedRoles), buf.String()) }) } } -// Replication reaches a node on the interserver port, and a node holding data -// takes longer to answer than the default start period allows. -func TestTelemetryStoreContainerIsReachableAndGivenTimeToStart(t *testing.T) { - buf := bytes.NewBuffer(nil) - require.NoError(t, telemetryStoreTF.Execute(buf, templateDataFor(t, statedCasting(&installation.Casting{})))) +// A clustered fixture is one service per node, each named by the ordinal the +// molding gives it. +func TestStatefulComponentsForgeAServicePerNode(t *testing.T) { + shards, perShard, keepers, nodes := 2, 1, 3, 2 - material, err := domain.NewJSONMaterial(buf.Bytes(), "telemetrystore.tf.json") - require.NoError(t, err) + store := statedCasting(&installation.Casting{}) + store.Spec.TelemetryStore.Spec.Cluster.Shards = &shards + store.Spec.TelemetryStore.Spec.Cluster.Replicas = &perShard - container := `locals.containers_telemetrystore_clickhouse_0_0.#(name=="signoz-telemetrystore-clickhouse-0-0")` + keeper := statedCasting(&installation.Casting{}) + keeper.Spec.TelemetryKeeper.Kind = installation.TelemetryKeeperKindClickhouseKeeper + keeper.Spec.TelemetryKeeper.Spec.Cluster.Replicas = &keepers - tests := []struct { - name string - path string - expectedValue string - }{ - {name: "NativePort_Mapped", path: container + `.portMappings.#(containerPort==9000).name`, expectedValue: "native"}, - {name: "HTTPPort_Mapped", path: container + `.portMappings.#(containerPort==8123).name`, expectedValue: "http"}, - {name: "PrometheusPort_Mapped", path: container + `.portMappings.#(containerPort==9363).name`, expectedValue: "prometheus"}, - {name: "InterserverPort_Mapped", path: container + `.portMappings.#(containerPort==9009).name`, expectedValue: "interserver"}, - {name: "StartPeriod_Stated", path: container + `.healthCheck.startPeriod`, expectedValue: "300"}, - } + metaStore := statedCasting(&installation.Casting{}) + metaStore.Spec.MetaStore.Spec.Cluster.Replicas = &nodes - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - value, err := material.GetBytes(tt.path) - require.NoError(t, err) - assert.Equal(t, tt.expectedValue, string(value)) - }) - } -} - -// Nothing chowns the task volume, so the agent writes as root and the server -// stays root to read what it wrote. -func TestTelemetryStoreRunsAsRoot(t *testing.T) { - buf := bytes.NewBuffer(nil) - require.NoError(t, telemetryStoreTF.Execute(buf, templateDataFor(t, statedCasting(&installation.Casting{})))) - - material, err := domain.NewJSONMaterial(buf.Bytes(), "telemetrystore.tf.json") - require.NoError(t, err) - - agent := `locals.containers_telemetrystore_clickhouse_0_0.#(name=="signoz-telemetrystore-appconfig-agent")` - server := `locals.containers_telemetrystore_clickhouse_0_0.#(name=="signoz-telemetrystore-clickhouse-0-0")` - scripts := `locals.containers_telemetrystore_clickhouse_0_0.#(name=="signoz-telemetrystore-user-scripts")` - - t.Run("AgentUser_Unstated", func(t *testing.T) { - _, err := material.GetBytes(agent + ".user") - assert.Error(t, err) - }) - - t.Run("ServerRunAsRoot_Stated", func(t *testing.T) { - value, err := material.GetBytes(server + `.environment.#(name=="CLICKHOUSE_RUN_AS_ROOT").value`) - require.NoError(t, err) - assert.Equal(t, "1", string(value)) - }) - - t.Run("UserScriptsCommand_Chownless", func(t *testing.T) { - value, err := material.GetBytes(scripts + ".command.0") - require.NoError(t, err) - assert.NotContains(t, string(value), "chown") - }) -} - -// Nothing chowns the task volume, so the agent writes as root and the -// collector reads what it wrote. -func TestIngesterContainersRunAsRoot(t *testing.T) { - buf := bytes.NewBuffer(nil) - require.NoError(t, ingesterTF.Execute(buf, templateDataFor(t, statedCasting(&installation.Casting{})))) - - material, err := domain.NewJSONMaterial(buf.Bytes(), "ingester.tf.json") - require.NoError(t, err) + signoz := statedCasting(&installation.Casting{}) + signoz.Spec.Signoz.Spec.Cluster.Replicas = &nodes tests := []struct { - name string - path string - expectedValue string + name string + template *domain.Template + casting *installation.Casting + expectedServices []string }{ - {name: "ContainerCount_Stated", path: "locals.containers_ingester.#", expectedValue: "2"}, - {name: "AgentName_Stated", path: "locals.containers_ingester.0.name", expectedValue: "signoz-ingester-appconfig-agent"}, - {name: "CollectorName_Stated", path: "locals.containers_ingester.1.name", expectedValue: "signoz-ingester"}, - {name: "AgentUser_Root", path: `locals.containers_ingester.#(name=="signoz-ingester-appconfig-agent").user`, expectedValue: "0"}, - {name: "CollectorUser_Root", path: `locals.containers_ingester.#(name=="signoz-ingester").user`, expectedValue: "0"}, + {name: "TelemetryStore_ServicePerNode", template: telemetryStoreTF, casting: store, expectedServices: []string{"signoz-telemetrystore-clickhouse-0-0", "signoz-telemetrystore-clickhouse-0-1", "signoz-telemetrystore-clickhouse-1-0", "signoz-telemetrystore-clickhouse-1-1"}}, + {name: "TelemetryKeeper_ServicePerNode", template: telemetryKeeperTF, casting: keeper, expectedServices: []string{"signoz-telemetrykeeper-clickhousekeeper-0", "signoz-telemetrykeeper-clickhousekeeper-1", "signoz-telemetrykeeper-clickhousekeeper-2"}}, + {name: "MetaStore_ServicePerNode", template: metaStoreTF, casting: metaStore, expectedServices: []string{"signoz-metastore-postgres-0", "signoz-metastore-postgres-1"}}, + {name: "Signoz_ServicePerNode", template: signozTF, casting: signoz, expectedServices: []string{"signoz-signoz-0", "signoz-signoz-1"}}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - value, err := material.GetBytes(tt.path) - require.NoError(t, err) - assert.Equal(t, tt.expectedValue, string(value)) - }) - } -} - -// The keeper writes a host path owned by root, and a node holding a raft log -// takes longer to answer than the default start period allows. -func TestTelemetryKeeperContainerRunsAsRootAndIsGivenTimeToStart(t *testing.T) { - zookeeper := `locals.containers_telemetrykeeper_zookeeper_0.#(name=="signoz-telemetrykeeper-zookeeper-0")` - clickhouseKeeper := `locals.containers_telemetrykeeper_clickhousekeeper_0.#(name=="signoz-telemetrykeeper-clickhousekeeper-0")` - - tests := []struct { - name string - kind installation.TelemetryKeeperKind - path string - expectedValue string - }{ - {name: "ZookeeperUser_Root", kind: installation.TelemetryKeeperKindZookeeper, path: zookeeper + ".user", expectedValue: "0"}, - {name: "ZookeeperPrometheusPort_Mapped", kind: installation.TelemetryKeeperKindZookeeper, path: zookeeper + `.portMappings.#(containerPort==9141).name`, expectedValue: "prometheus"}, - {name: "ZookeeperStartPeriod_Stated", kind: installation.TelemetryKeeperKindZookeeper, path: zookeeper + ".healthCheck.startPeriod", expectedValue: "300"}, - {name: "ClickhouseKeeperStartPeriod_Stated", kind: installation.TelemetryKeeperKindClickhouseKeeper, path: clickhouseKeeper + ".healthCheck.startPeriod", expectedValue: "300"}, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - casting := statedCasting(&installation.Casting{}) - casting.Spec.TelemetryKeeper.Kind = tt.kind - buf := bytes.NewBuffer(nil) - require.NoError(t, telemetryKeeperTF.Execute(buf, templateDataFor(t, casting))) + require.NoError(t, tt.template.Execute(buf, templateDataFor(t, tt.casting))) - material, err := domain.NewJSONMaterial(buf.Bytes(), "telemetrykeeper.tf.json") + material, err := domain.NewJSONMaterial(buf.Bytes(), "component.tf.json") require.NoError(t, err) - value, err := material.GetBytes(tt.path) + services, err := material.GetStringSlice("resource.aws_ecs_service.@values.#.name") require.NoError(t, err) - assert.Equal(t, tt.expectedValue, string(value)) + assert.ElementsMatch(t, tt.expectedServices, services) }) } } -// SigNoz carries a node ordinal whatever the metadata store is, so only the -// sqlite disk and its rollout tell the two apart. -func TestSignozIsAlwaysANode(t *testing.T) { +// ZooKeeper numbers its own nodes from one, and an ensemble names itself +// 0.0.0.0 so the node does not dial its own service record. +func TestZookeeperEnsembleKnowsItsPeers(t *testing.T) { tests := []struct { - name string - metaStoreKind installation.MetaStoreKind - expectedHostPath string - expectedMinimumPercent string - expectedMaximumPercent string + name string + replicas int + node int + expectedID string + expectedServers string }{ + {name: "SingleNode_NoEnsemble", replicas: 1, node: 0, expectedID: "1"}, { - name: "Postgres_Diskless", - metaStoreKind: installation.MetaStoreKindPostgres, - expectedMinimumPercent: "100", - expectedMaximumPercent: "200", + name: "FirstOfThree_Ensemble", replicas: 3, node: 0, expectedID: "1", + expectedServers: "0.0.0.0:2888:3888,telemetrykeeper-zookeeper-1.signoz-installation.local:2888:3888,telemetrykeeper-zookeeper-2.signoz-installation.local:2888:3888", }, { - name: "Sqlite_HoldsADisk", - metaStoreKind: installation.MetaStoreKindSQLite, - expectedHostPath: "/var/lib/foundry/signoz/metastore/sqlite/0", - expectedMinimumPercent: "0", - expectedMaximumPercent: "100", + name: "LastOfThree_Ensemble", replicas: 3, node: 2, expectedID: "3", + expectedServers: "telemetrykeeper-zookeeper-0.signoz-installation.local:2888:3888,telemetrykeeper-zookeeper-1.signoz-installation.local:2888:3888,0.0.0.0:2888:3888", }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { casting := statedCasting(&installation.Casting{}) - casting.Spec.MetaStore.Kind = tt.metaStoreKind + casting.Spec.TelemetryKeeper.Kind = installation.TelemetryKeeperKindZookeeper + casting.Spec.TelemetryKeeper.Spec.Cluster.Replicas = &tt.replicas buf := bytes.NewBuffer(nil) - require.NoError(t, signozTF.Execute(buf, templateDataFor(t, casting))) + require.NoError(t, telemetryKeeperTF.Execute(buf, templateDataFor(t, casting))) - material, err := domain.NewJSONMaterial(buf.Bytes(), "signoz.tf.json") + material, err := domain.NewJSONMaterial(buf.Bytes(), "telemetrykeeper.tf.json") require.NoError(t, err) - expected := map[string]string{ - "locals.containers_signoz_0.0.name": "signoz-signoz-0", - "locals.containers_signoz_0.0.healthCheck.startPeriod": "300", - "resource.aws_ecs_task_definition.signoz_0.family": "signoz-signoz-0", - "resource.aws_service_discovery_service.signoz_0.name": "signoz-0", - "resource.aws_ecs_service.signoz_0.name": "signoz-signoz-0", - "resource.aws_ecs_service.signoz_0.task_definition": "${aws_ecs_task_definition.signoz_0.arn}", - "resource.aws_ecs_service.signoz_0.desired_count": "1", - "resource.aws_ecs_service.signoz_0.deployment_minimum_healthy_percent": tt.expectedMinimumPercent, - "resource.aws_ecs_service.signoz_0.deployment_maximum_percent": tt.expectedMaximumPercent, - } - - if tt.expectedHostPath != "" { - expected["resource.aws_ecs_task_definition.signoz_0.volume.0.host_path"] = tt.expectedHostPath - } + container := fmt.Sprintf(`locals.containers_telemetrykeeper_zookeeper_%d.#(name=="signoz-telemetrykeeper-zookeeper-%d")`, tt.node, tt.node) - for path, want := range expected { - value, err := material.GetBytes(path) + id, err := material.GetBytes(container + `.environment.#(name=="ZOO_SERVER_ID").value`) + require.NoError(t, err) + assert.Equal(t, tt.expectedID, string(id)) - require.NoError(t, err, "reading %s", path) - assert.Equal(t, want, string(value), "at %s", path) - } + servers, err := material.GetBytes(container + `.environment.#(name=="ZOO_SERVERS").value`) + if tt.expectedServers == "" { + assert.Error(t, err, "a single node stands alone") - if tt.expectedHostPath == "" { - _, err := material.GetBytes("resource.aws_ecs_task_definition.signoz_0.volume") - assert.Error(t, err) + return } - outputs := bytes.NewBuffer(nil) - require.NoError(t, outputsTF.Execute(outputs, templateDataFor(t, casting))) - - assert.Contains(t, outputs.String(), "${aws_ecs_service.signoz_0.name}") - assert.Contains(t, outputs.String(), "${aws_ecs_service.signoz_0.id}") + require.NoError(t, err) + assert.Equal(t, tt.expectedServers, string(servers)) }) } } -// A postgres holding a data directory takes longer to answer than the default -// start period allows. -func TestMetaStoreIsANodeGivenTimeToStart(t *testing.T) { +// A patch written against the singular output keeps working, so both shapes +// stand. +func TestOutputsEnumerateEveryNode(t *testing.T) { + shards, perShard, keepers := 2, 1, 3 + casting := statedCasting(&installation.Casting{}) - casting.Spec.MetaStore.Kind = installation.MetaStoreKindPostgres + casting.Spec.TelemetryStore.Spec.Cluster.Shards = &shards + casting.Spec.TelemetryStore.Spec.Cluster.Replicas = &perShard + casting.Spec.TelemetryKeeper.Spec.Cluster.Replicas = &keepers buf := bytes.NewBuffer(nil) - require.NoError(t, metaStoreTF.Execute(buf, templateDataFor(t, casting))) + require.NoError(t, outputsTF.Execute(buf, templateDataFor(t, casting))) - material, err := domain.NewJSONMaterial(buf.Bytes(), "metastore.tf.json") + material, err := domain.NewJSONMaterial(buf.Bytes(), "outputs.tf.json") require.NoError(t, err) - tests := []struct { - name string - path string - expectedValue string - }{ - {name: "StartPeriod_Stated", path: "locals.containers_metastore_postgres_0.0.healthCheck.startPeriod", expectedValue: "300"}, - {name: "ContainerName_Stated", path: "locals.containers_metastore_postgres_0.0.name", expectedValue: "signoz-metastore-postgres-0"}, - {name: "Family_Stated", path: "resource.aws_ecs_task_definition.metastore_postgres_0.family", expectedValue: "signoz-metastore-postgres-0"}, - {name: "DNS_Stated", path: "resource.aws_service_discovery_service.metastore_postgres_0.name", expectedValue: "metastore-postgres-0"}, - {name: "ServiceName_Stated", path: "resource.aws_ecs_service.metastore_postgres_0.name", expectedValue: "signoz-metastore-postgres-0"}, - {name: "HostPath_Stated", path: "resource.aws_ecs_task_definition.metastore_postgres_0.volume.0.host_path", expectedValue: "/var/lib/foundry/signoz/metastore/postgres/0"}, + for path, expectedCount := range map[string]int{ + "output.telemetrystore_service_names.value": 4, + "output.telemetrykeeper_service_names.value": 3, + } { + names, err := material.GetStringSlice(path) + require.NoError(t, err, "reading %s", path) + assert.Len(t, names, expectedCount, path) } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - value, err := material.GetBytes(tt.path) - require.NoError(t, err, "reading %s", tt.path) - assert.Equal(t, tt.expectedValue, string(value)) - }) + for _, path := range []string{"output.metastore_service_name.value", "output.signoz_service_name.value", "output.signoz_service_arn.value"} { + _, err := material.GetBytes(path) + assert.NoError(t, err, "reading %s", path) } } diff --git a/internal/casting/ecsterraformcasting/templates/metastore.tf.json.gotmpl b/internal/casting/ecsterraformcasting/templates/metastore.tf.json.gotmpl index 156bf7eb..c64da882 100644 --- a/internal/casting/ecsterraformcasting/templates/metastore.tf.json.gotmpl +++ b/internal/casting/ecsterraformcasting/templates/metastore.tf.json.gotmpl @@ -1,11 +1,12 @@ {{- $name := $.Metadata.Name -}} {{- $kind := $.Spec.MetaStore.Kind.String -}} -{{- $i := 0 -}} -{{- $label := printf "metastore_%s_%d" $kind $i -}} -{{- $dns := printf "metastore-%s-%d" $kind $i -}} -{{- $family := printf "%s-%s" $name $dns -}} +{{- $replicas := derefIntDefault $.Spec.MetaStore.Spec.Cluster.Replicas 1 -}} +{{- if lt $replicas 1 }}{{- $replicas = 1 -}}{{- end -}} { "locals": { + {{- range $i := until $replicas }}{{ if $i }},{{ end }} + {{- $label := printf "metastore_%s_%d" $kind $i }} + {{- $family := printf "%s-metastore-%s-%d" $name $kind $i }} "containers_{{ $label }}": [ { "name": "{{ $family }}", @@ -44,9 +45,13 @@ } } ] + {{- end }} }, "resource": { "aws_ecs_task_definition": { + {{- range $i := until $replicas }}{{ if $i }},{{ end }} + {{- $label := printf "metastore_%s_%d" $kind $i }} + {{- $family := printf "%s-metastore-%s-%d" $name $kind $i }} "{{ $label }}": { "family": "{{ $family }}", "tags": {{ toJson $.Labels }}, @@ -62,10 +67,13 @@ } ] } + {{- end }} }, "aws_service_discovery_service": { + {{- range $i := until $replicas }}{{ if $i }},{{ end }} + {{- $label := printf "metastore_%s_%d" $kind $i }} "{{ $label }}": { - "name": "{{ $dns }}", + "name": "metastore-{{ $kind }}-{{ $i }}", "tags": {{ toJson $.Labels }}, "dns_config": { "namespace_id": "${aws_service_discovery_private_dns_namespace.main.id}", @@ -78,8 +86,12 @@ "routing_policy": "MULTIVALUE" } } + {{- end }} }, "aws_ecs_service": { + {{- range $i := until $replicas }}{{ if $i }},{{ end }} + {{- $label := printf "metastore_%s_%d" $kind $i }} + {{- $family := printf "%s-metastore-%s-%d" $name $kind $i }} "{{ $label }}": { "name": "{{ $family }}", "cluster": "${local.cluster_arn}", @@ -101,6 +113,7 @@ "registry_arn": "${aws_service_discovery_service.{{ $label }}.arn}" } } + {{- end }} } } } diff --git a/internal/casting/ecsterraformcasting/templates/outputs.tf.json.gotmpl b/internal/casting/ecsterraformcasting/templates/outputs.tf.json.gotmpl index 946e069d..ebc720bd 100644 --- a/internal/casting/ecsterraformcasting/templates/outputs.tf.json.gotmpl +++ b/internal/casting/ecsterraformcasting/templates/outputs.tf.json.gotmpl @@ -1,3 +1,8 @@ +{{- $storeShards := derefIntDefault $.Spec.TelemetryStore.Spec.Cluster.Shards 1 -}} +{{- if lt $storeShards 1 }}{{- $storeShards = 1 -}}{{- end -}} +{{- $storePerShard := int (add (derefIntDefault $.Spec.TelemetryStore.Spec.Cluster.Replicas 0) 1) -}} +{{- $keeperReplicas := derefIntDefault $.Spec.TelemetryKeeper.Spec.Cluster.Replicas 1 -}} +{{- if lt $keeperReplicas 1 }}{{- $keeperReplicas = 1 -}}{{- end -}} { "output": { "cluster_arn": { @@ -43,13 +48,13 @@ {{- if derefBool $.Spec.TelemetryStore.Spec.Enabled }}, "telemetrystore_service_names": { "description": "TelemetryStore ECS service names (one per node)", - "value": ["${aws_ecs_service.telemetrystore_{{ $.Spec.TelemetryStore.Kind.String }}_0_0.name}"] + "value": [{{ $first := true }}{{ range $s := until $storeShards }}{{ range $r := until $storePerShard }}{{ if not $first }}, {{ end }}{{ $first = false }}"${aws_ecs_service.telemetrystore_{{ $.Spec.TelemetryStore.Kind.String }}_{{ $s }}_{{ $r }}.name}"{{ end }}{{ end }}] } {{- end }} {{- if derefBool $.Spec.TelemetryKeeper.Spec.Enabled }}, "telemetrykeeper_service_names": { "description": "TelemetryKeeper ECS service names (one per node)", - "value": ["${aws_ecs_service.telemetrykeeper_{{ $.Spec.TelemetryKeeper.Kind.String }}_0.name}"] + "value": [{{ range $i := until $keeperReplicas }}{{ if $i }}, {{ end }}"${aws_ecs_service.telemetrykeeper_{{ $.Spec.TelemetryKeeper.Kind.String }}_{{ $i }}.name}"{{ end }}] } {{- end }} {{- if and (derefBool $.Spec.MetaStore.Spec.Enabled) (eq $.Spec.MetaStore.Kind.String "postgres") }}, diff --git a/internal/casting/ecsterraformcasting/templates/signoz.tf.json.gotmpl b/internal/casting/ecsterraformcasting/templates/signoz.tf.json.gotmpl index a85b06db..3b3afe64 100644 --- a/internal/casting/ecsterraformcasting/templates/signoz.tf.json.gotmpl +++ b/internal/casting/ecsterraformcasting/templates/signoz.tf.json.gotmpl @@ -1,13 +1,13 @@ {{- $name := $.Metadata.Name -}} {{- $sqlite := eq $.Spec.MetaStore.Kind.String "sqlite" -}} -{{- $i := 0 -}} -{{- $label := printf "signoz_%d" $i -}} -{{- $dns := printf "signoz-%d" $i -}} -{{- $family := printf "%s-signoz-%d" $name $i -}} -{{- $local := printf "containers_%s" $label -}} +{{- $replicas := derefIntDefault $.Spec.Signoz.Spec.Cluster.Replicas 1 -}} +{{- if lt $replicas 1 }}{{- $replicas = 1 -}}{{- end -}} { "locals": { - "{{ $local }}": [ + {{- range $i := until $replicas }}{{ if $i }},{{ end }} + {{- $label := printf "signoz_%d" $i }} + {{- $family := printf "%s-signoz-%d" $name $i }} + "containers_{{ $label }}": [ { "name": "{{ $family }}", "image": "{{ $.Spec.Signoz.Spec.Image }}", @@ -50,9 +50,13 @@ } } ] + {{- end }} }, "resource": { "aws_ecs_task_definition": { + {{- range $i := until $replicas }}{{ if $i }},{{ end }} + {{- $label := printf "signoz_%d" $i }} + {{- $family := printf "%s-signoz-%d" $name $i }} "{{ $label }}": { "family": "{{ $family }}", "tags": {{ toJson $.Labels }}, @@ -60,7 +64,7 @@ "requires_compatibilities": ["EC2"], "task_role_arn": "${local.task_role_arn}", "execution_role_arn": "${local.execution_role_arn}", - "container_definitions": "${jsonencode(local.{{ $local }})}" + "container_definitions": "${jsonencode(local.containers_{{ $label }})}" {{- if $sqlite }}, "volume": [ { @@ -70,10 +74,13 @@ ] {{- end }} } + {{- end }} }, "aws_service_discovery_service": { + {{- range $i := until $replicas }}{{ if $i }},{{ end }} + {{- $label := printf "signoz_%d" $i }} "{{ $label }}": { - "name": "{{ $dns }}", + "name": "signoz-{{ $i }}", "tags": {{ toJson $.Labels }}, "dns_config": { "namespace_id": "${aws_service_discovery_private_dns_namespace.main.id}", @@ -86,8 +93,12 @@ "routing_policy": "MULTIVALUE" } } + {{- end }} }, "aws_ecs_service": { + {{- range $i := until $replicas }}{{ if $i }},{{ end }} + {{- $label := printf "signoz_%d" $i }} + {{- $family := printf "%s-signoz-%d" $name $i }} "{{ $label }}": { "name": "{{ $family }}", "cluster": "${local.cluster_arn}", @@ -109,6 +120,7 @@ "registry_arn": "${aws_service_discovery_service.{{ $label }}.arn}" } } + {{- end }} } } } diff --git a/internal/casting/ecsterraformcasting/templates/telemetrykeeper.tf.json.gotmpl b/internal/casting/ecsterraformcasting/templates/telemetrykeeper.tf.json.gotmpl index e943fb2c..22281d29 100644 --- a/internal/casting/ecsterraformcasting/templates/telemetrykeeper.tf.json.gotmpl +++ b/internal/casting/ecsterraformcasting/templates/telemetrykeeper.tf.json.gotmpl @@ -1,11 +1,14 @@ {{- $name := $.Metadata.Name -}} {{- $kind := $.Spec.TelemetryKeeper.Kind.String -}} +{{- $namespace := printf "%s-installation.local" $name -}} +{{- $replicas := derefIntDefault $.Spec.TelemetryKeeper.Spec.Cluster.Replicas 1 -}} +{{- if lt $replicas 1 }}{{- $replicas = 1 -}}{{- end -}} {{- /* The molding names its config file by the same ordinal. */ -}} -{{- $i := 0 -}} -{{- $label := printf "telemetrykeeper_%s_%d" $kind $i -}} {{- if eq $kind "zookeeper" -}} { "locals": { + {{- range $i := until $replicas }}{{ if $i }},{{ end }} + {{- $label := printf "telemetrykeeper_%s_%d" $kind $i }} "containers_{{ $label }}": [ { "name": "{{ $name }}-telemetrykeeper-{{ $kind }}-{{ $i }}", @@ -22,6 +25,17 @@ "user": "0", {{- /* One-based: ZooKeeper's own myid starts at 1. */}} {{- $env := list (dict "name" "ZOO_SERVER_ID" "value" (printf "%d" (add $i 1))) }} + {{- if gt $replicas 1 }} + {{- $servers := list }} + {{- range $j := until $replicas }} + {{- if eq $j $i }} + {{- $servers = append $servers "0.0.0.0:2888:3888" }} + {{- else }} + {{- $servers = append $servers (printf "telemetrykeeper-%s-%d.%s:2888:3888" $kind $j $namespace) }} + {{- end }} + {{- end }} + {{- $env = append $env (dict "name" "ZOO_SERVERS" "value" (join "," $servers)) }} + {{- end }} {{- range $key, $value := $.Spec.TelemetryKeeper.Spec.Env }} {{- $env = append $env (dict "name" $key "value" $value) }} {{- end }} @@ -49,9 +63,12 @@ } } ] + {{- end }} }, "resource": { "aws_ecs_task_definition": { + {{- range $i := until $replicas }}{{ if $i }},{{ end }} + {{- $label := printf "telemetrykeeper_%s_%d" $kind $i }} "{{ $label }}": { "family": "{{ $name }}-telemetrykeeper-{{ $kind }}-{{ $i }}", "tags": {{ toJson $.Labels }}, @@ -67,8 +84,11 @@ } ] } + {{- end }} }, "aws_service_discovery_service": { + {{- range $i := until $replicas }}{{ if $i }},{{ end }} + {{- $label := printf "telemetrykeeper_%s_%d" $kind $i }} "{{ $label }}": { "name": "telemetrykeeper-{{ $kind }}-{{ $i }}", "tags": {{ toJson $.Labels }}, @@ -83,8 +103,11 @@ "routing_policy": "MULTIVALUE" } } + {{- end }} }, "aws_ecs_service": { + {{- range $i := until $replicas }}{{ if $i }},{{ end }} + {{- $label := printf "telemetrykeeper_%s_%d" $kind $i }} "{{ $label }}": { "name": "{{ $name }}-telemetrykeeper-{{ $kind }}-{{ $i }}", "cluster": "${local.cluster_arn}", @@ -106,13 +129,16 @@ "registry_arn": "${aws_service_discovery_service.{{ $label }}.arn}" } } + {{- end }} } } } {{- else -}} -{{- $profile := printf "telemetrykeeper-%s-%d" $kind $i -}} { "locals": { + {{- range $i := until $replicas }}{{ if $i }},{{ end }} + {{- $label := printf "telemetrykeeper_%s_%d" $kind $i }} + {{- $profile := printf "telemetrykeeper-%s-%d" $kind $i }} "containers_{{ $label }}": [ {{- /* A config change reloads keeper rather than replacing the node that holds the raft log. */}} @@ -198,28 +224,37 @@ } } ] + {{- end }} }, "resource": { {{- /* Hosted versions are immutable, so editing the pour's YAML mints a new version and a new deployment. */}} "aws_appconfig_configuration_profile": { + {{- range $i := until $replicas }}{{ if $i }},{{ end }} + {{- $label := printf "telemetrykeeper_%s_%d" $kind $i }} "{{ $label }}": { "application_id": "${aws_appconfig_application.main.id}", - "name": "{{ $profile }}", + "name": "telemetrykeeper-{{ $kind }}-{{ $i }}", "location_uri": "hosted", "type": "AWS.Freeform", "tags": {{ toJson $.Labels }} } + {{- end }} }, "aws_appconfig_hosted_configuration_version": { + {{- range $i := until $replicas }}{{ if $i }},{{ end }} + {{- $label := printf "telemetrykeeper_%s_%d" $kind $i }} "{{ $label }}": { "application_id": "${aws_appconfig_application.main.id}", "configuration_profile_id": "${aws_appconfig_configuration_profile.{{ $label }}.configuration_profile_id}", "content_type": "application/x-yaml", "content": "${file(\"${path.module}/telemetrykeeper/{{ $kind }}/keeper-{{ $i }}.yaml\")}" } + {{- end }} }, "aws_appconfig_deployment": { + {{- range $i := until $replicas }}{{ if $i }},{{ end }} + {{- $label := printf "telemetrykeeper_%s_%d" $kind $i }} "{{ $label }}": { "application_id": "${aws_appconfig_application.main.id}", "environment_id": "${aws_appconfig_environment.main.environment_id}", @@ -228,8 +263,11 @@ "deployment_strategy_id": "${aws_appconfig_deployment_strategy.main.id}", "tags": {{ toJson $.Labels }} } + {{- end }} }, "aws_ecs_task_definition": { + {{- range $i := until $replicas }}{{ if $i }},{{ end }} + {{- $label := printf "telemetrykeeper_%s_%d" $kind $i }} "{{ $label }}": { "family": "{{ $name }}-telemetrykeeper-{{ $kind }}-{{ $i }}", "tags": {{ toJson $.Labels }}, @@ -253,8 +291,11 @@ ], "depends_on": ["aws_appconfig_deployment.{{ $label }}"] } + {{- end }} }, "aws_service_discovery_service": { + {{- range $i := until $replicas }}{{ if $i }},{{ end }} + {{- $label := printf "telemetrykeeper_%s_%d" $kind $i }} "{{ $label }}": { "name": "telemetrykeeper-{{ $kind }}-{{ $i }}", "tags": {{ toJson $.Labels }}, @@ -269,8 +310,11 @@ "routing_policy": "MULTIVALUE" } } + {{- end }} }, "aws_ecs_service": { + {{- range $i := until $replicas }}{{ if $i }},{{ end }} + {{- $label := printf "telemetrykeeper_%s_%d" $kind $i }} "{{ $label }}": { "name": "{{ $name }}-telemetrykeeper-{{ $kind }}-{{ $i }}", "cluster": "${local.cluster_arn}", @@ -292,6 +336,7 @@ "registry_arn": "${aws_service_discovery_service.{{ $label }}.arn}" } } + {{- end }} } } } diff --git a/internal/casting/ecsterraformcasting/templates/telemetrystore.tf.json.gotmpl b/internal/casting/ecsterraformcasting/templates/telemetrystore.tf.json.gotmpl index 61a7f5d8..37c00a26 100644 --- a/internal/casting/ecsterraformcasting/templates/telemetrystore.tf.json.gotmpl +++ b/internal/casting/ecsterraformcasting/templates/telemetrystore.tf.json.gotmpl @@ -1,15 +1,21 @@ {{- $name := $.Metadata.Name -}} {{- $kind := $.Spec.TelemetryStore.Kind.String -}} +{{- $shards := derefIntDefault $.Spec.TelemetryStore.Spec.Cluster.Shards 1 -}} +{{- if lt $shards 1 }}{{- $shards = 1 -}}{{- end -}} +{{- $perShard := int (add (derefIntDefault $.Spec.TelemetryStore.Spec.Cluster.Replicas 0) 1) -}} {{- /* The molding names its config file by the same id. */ -}} -{{- $id := "0-0" -}} -{{- $nodeKey := "0_0" -}} -{{- $label := printf "telemetrystore_%s_%s" $kind $nodeKey -}} +{{- $nodes := list -}} +{{- range $s := until $shards -}} +{{- range $r := until $perShard -}} +{{- $nodes = append $nodes (dict "id" (printf "%d-%d" $s $r) "label" (printf "telemetrystore_%s_%d_%d" $kind $s $r) "profile" (printf "telemetrystore-%s-%d-%d" $kind $s $r)) -}} +{{- end -}} +{{- end -}} {{- $fnLabel := printf "telemetrystore_%s_functions" $kind -}} -{{- $cfgProfile := printf "telemetrystore-%s-%s" $kind $id -}} {{- $fnProfile := printf "telemetrystore-%s-functions" $kind -}} { "locals": { - "containers_{{ $label }}": [ + {{- range $i, $node := $nodes }}{{ if $i }},{{ end }} + "containers_{{ $node.label }}": [ { "name": "{{ $name }}-telemetrystore-user-scripts", "image": "{{ $.Spec.TelemetryStore.Spec.Image }}", @@ -52,9 +58,9 @@ {"containerName": "{{ $name }}-telemetrystore-user-scripts", "condition": "SUCCESS"} ], "environment": [ - {"name": "PREFETCH_LIST", "value": "{{ $name }}-installation-appconfig:default:{{ $cfgProfile }},{{ $name }}-installation-appconfig:default:{{ $fnProfile }}"}, + {"name": "PREFETCH_LIST", "value": "{{ $name }}-installation-appconfig:default:{{ $node.profile }},{{ $name }}-installation-appconfig:default:{{ $fnProfile }}"}, {"name": "POLL_INTERVAL", "value": "45s"}, - {"name": "MANIFEST", "value": "{\"{{ $name }}-installation-appconfig:default:{{ $cfgProfile }}\":{\"writeTo\":{\"path\":\"/etc/clickhouse-server/config.d/config-{{ $id }}.yaml\"}},\"{{ $name }}-installation-appconfig:default:{{ $fnProfile }}\":{\"writeTo\":{\"path\":\"/etc/clickhouse-server/config.d/functions.yaml\"}}}"} + {"name": "MANIFEST", "value": "{\"{{ $name }}-installation-appconfig:default:{{ $node.profile }}\":{\"writeTo\":{\"path\":\"/etc/clickhouse-server/config.d/config-{{ $node.id }}.yaml\"}},\"{{ $name }}-installation-appconfig:default:{{ $fnProfile }}\":{\"writeTo\":{\"path\":\"/etc/clickhouse-server/config.d/functions.yaml\"}}}"} ], "mountPoints": [ { @@ -63,7 +69,7 @@ } ], "healthCheck": { - "command": ["CMD-SHELL", "test -s /etc/clickhouse-server/config.d/config-{{ $id }}.yaml && test -s /etc/clickhouse-server/config.d/functions.yaml"], + "command": ["CMD-SHELL", "test -s /etc/clickhouse-server/config.d/config-{{ $node.id }}.yaml && test -s /etc/clickhouse-server/config.d/functions.yaml"], "interval": 5, "timeout": 3, "retries": 10, @@ -72,7 +78,7 @@ "memoryReservation": 102 }, { - "name": "{{ $name }}-telemetrystore-{{ $kind }}-{{ $id }}", + "name": "{{ $name }}-telemetrystore-{{ $kind }}-{{ $node.id }}", "image": "{{ $.Spec.TelemetryStore.Spec.Image }}", "logConfiguration": { "logDriver": "json-file", @@ -84,7 +90,7 @@ }, "essential": true, {{- $env := list (dict "name" "CLICKHOUSE_SKIP_USER_SETUP" "value" "1") }} - {{- $env = append $env (dict "name" "CLICKHOUSE_CONFIG" "value" (printf "/etc/clickhouse-server/config.d/config-%s.yaml" $id)) }} + {{- $env = append $env (dict "name" "CLICKHOUSE_CONFIG" "value" (printf "/etc/clickhouse-server/config.d/config-%s.yaml" $node.id)) }} {{- /* The agent writes its files as root, so the server stays root through the image's own switch. */}} {{- $env = append $env (dict "name" "CLICKHOUSE_RUN_AS_ROOT" "value" "1") }} @@ -119,7 +125,7 @@ "cpu": 1024, "memoryReservation": 512, "healthCheck": { - "command": ["CMD-SHELL", "wget --spider -q http://localhost:8123/ping || exit 1"], + "command": ["CMD-SHELL", "wget -q -O- 0.0.0.0:8123/ping || exit 1"], "interval": 30, "timeout": 5, "retries": 3, @@ -128,16 +134,19 @@ } } ] + {{- end }} }, "resource": { "aws_appconfig_configuration_profile": { - "{{ $label }}": { + {{- range $i, $node := $nodes }}{{ if $i }},{{ end }} + "{{ $node.label }}": { "application_id": "${aws_appconfig_application.main.id}", - "name": "{{ $cfgProfile }}", + "name": "{{ $node.profile }}", "location_uri": "hosted", "type": "AWS.Freeform", "tags": {{ toJson $.Labels }} - }, + } + {{- end }}, "{{ $fnLabel }}": { "application_id": "${aws_appconfig_application.main.id}", "name": "{{ $fnProfile }}", @@ -147,12 +156,14 @@ } }, "aws_appconfig_hosted_configuration_version": { - "{{ $label }}": { + {{- range $i, $node := $nodes }}{{ if $i }},{{ end }} + "{{ $node.label }}": { "application_id": "${aws_appconfig_application.main.id}", - "configuration_profile_id": "${aws_appconfig_configuration_profile.{{ $label }}.configuration_profile_id}", + "configuration_profile_id": "${aws_appconfig_configuration_profile.{{ $node.label }}.configuration_profile_id}", "content_type": "application/x-yaml", - "content": "${file(\"${path.module}/telemetrystore/{{ $kind }}/config-{{ $id }}.yaml\")}" - }, + "content": "${file(\"${path.module}/telemetrystore/{{ $kind }}/config-{{ $node.id }}.yaml\")}" + } + {{- end }}, "{{ $fnLabel }}": { "application_id": "${aws_appconfig_application.main.id}", "configuration_profile_id": "${aws_appconfig_configuration_profile.{{ $fnLabel }}.configuration_profile_id}", @@ -161,14 +172,16 @@ } }, "aws_appconfig_deployment": { - "{{ $label }}": { + {{- range $i, $node := $nodes }}{{ if $i }},{{ end }} + "{{ $node.label }}": { "application_id": "${aws_appconfig_application.main.id}", "environment_id": "${aws_appconfig_environment.main.environment_id}", - "configuration_profile_id": "${aws_appconfig_configuration_profile.{{ $label }}.configuration_profile_id}", - "configuration_version": "${aws_appconfig_hosted_configuration_version.{{ $label }}.version_number}", + "configuration_profile_id": "${aws_appconfig_configuration_profile.{{ $node.label }}.configuration_profile_id}", + "configuration_version": "${aws_appconfig_hosted_configuration_version.{{ $node.label }}.version_number}", "deployment_strategy_id": "${aws_appconfig_deployment_strategy.main.id}", "tags": {{ toJson $.Labels }} - }, + } + {{- end }}, "{{ $fnLabel }}": { "application_id": "${aws_appconfig_application.main.id}", "environment_id": "${aws_appconfig_environment.main.environment_id}", @@ -179,14 +192,15 @@ } }, "aws_ecs_task_definition": { - "{{ $label }}": { - "family": "{{ $name }}-telemetrystore-{{ $kind }}-{{ $id }}", + {{- range $i, $node := $nodes }}{{ if $i }},{{ end }} + "{{ $node.label }}": { + "family": "{{ $name }}-telemetrystore-{{ $kind }}-{{ $node.id }}", "tags": {{ toJson $.Labels }}, "network_mode": "awsvpc", "requires_compatibilities": ["EC2"], "task_role_arn": "${local.task_role_arn}", "execution_role_arn": "${local.execution_role_arn}", - "container_definitions": "${jsonencode(local.containers_{{ $label }})}", + "container_definitions": "${jsonencode(local.containers_{{ $node.label }})}", "volume": [ { "name": "telemetrystore-user-scripts", @@ -204,15 +218,17 @@ }, { "name": "telemetrystore-data", - "host_path": "/var/lib/foundry/{{ $name }}/telemetrystore/{{ $kind }}/{{ $id }}" + "host_path": "/var/lib/foundry/{{ $name }}/telemetrystore/{{ $kind }}/{{ $node.id }}" } ], - "depends_on": ["aws_appconfig_deployment.{{ $label }}", "aws_appconfig_deployment.{{ $fnLabel }}"] + "depends_on": ["aws_appconfig_deployment.{{ $node.label }}", "aws_appconfig_deployment.{{ $fnLabel }}"] } + {{- end }} }, "aws_service_discovery_service": { - "{{ $label }}": { - "name": "telemetrystore-{{ $kind }}-{{ $id }}", + {{- range $i, $node := $nodes }}{{ if $i }},{{ end }} + "{{ $node.label }}": { + "name": "telemetrystore-{{ $kind }}-{{ $node.id }}", "tags": {{ toJson $.Labels }}, "dns_config": { "namespace_id": "${aws_service_discovery_private_dns_namespace.main.id}", @@ -225,12 +241,14 @@ "routing_policy": "MULTIVALUE" } } + {{- end }} }, "aws_ecs_service": { - "{{ $label }}": { - "name": "{{ $name }}-telemetrystore-{{ $kind }}-{{ $id }}", + {{- range $i, $node := $nodes }}{{ if $i }},{{ end }} + "{{ $node.label }}": { + "name": "{{ $name }}-telemetrystore-{{ $kind }}-{{ $node.id }}", "cluster": "${local.cluster_arn}", - "task_definition": "${aws_ecs_task_definition.{{ $label }}.arn}", + "task_definition": "${aws_ecs_task_definition.{{ $node.label }}.arn}", "desired_count": 1, {{- /* Two tasks cannot co-schedule over one data directory. */}} "deployment_minimum_healthy_percent": 0, @@ -246,9 +264,10 @@ "security_groups": "${local.security_group_ids}" }, "service_registries": { - "registry_arn": "${aws_service_discovery_service.{{ $label }}.arn}" + "registry_arn": "${aws_service_discovery_service.{{ $node.label }}.arn}" } } + {{- end }} } } }