SCC-5654: add alarms

[KaseyW31]

Adds lambda error and log error alarms. Existing error logging looks thorough
terraform plan output for BibDelete on qa:

Terraform used the selected providers to generate the following execution plan. Resource actions are indicated with the following symbols:
  + create
  ~ update in-place

Terraform will perform the following actions:

  # module.base.aws_cloudwatch_log_metric_filter.log_error_metric_filter will be created
  + resource "aws_cloudwatch_log_metric_filter" "log_error_metric_filter" {
      + apply_on_transformed_logs = (known after apply)
      + id                        = (known after apply)
      + log_group_name            = "/aws/lambda/SierraBibDeleteUpdatePoller-qa"
      + name                      = "SierraBibDeleteUpdatePollerLogError-qa"
      + pattern                   = "{ $.level = \"error\" }"
      + region                    = "us-east-1"

      + metric_transformation {
          + name      = "SierraBibDeleteUpdatePollerLogError-qa"
          + namespace = "LogMetrics"
          + unit      = "None"
          + value     = "1"
        }
    }

  # module.base.aws_cloudwatch_metric_alarm.lambda_errors will be created
  + resource "aws_cloudwatch_metric_alarm" "lambda_errors" {
      + actions_enabled                       = true
      + alarm_actions                         = [
          + "arn:aws:sns:us-east-1:946183545209:research-catalog-team-alarms-qa",
        ]
      + alarm_description                     = "Lambda function SierraBibDeleteUpdatePoller-qa has invocation errors"
      + alarm_name                            = "SierraBibDeleteUpdatePollerLambdaErrorAlarm-qa"
      + arn                                   = (known after apply)
      + comparison_operator                   = "GreaterThanOrEqualToThreshold"
      + dimensions                            = {
          + "FunctionName" = "SierraBibDeleteUpdatePoller-qa"
        }
      + evaluate_low_sample_count_percentiles = (known after apply)
      + evaluation_periods                    = 1
      + id                                    = (known after apply)
      + metric_name                           = "Errors"
      + namespace                             = "AWS/Lambda"
      + period                                = 300
      + region                                = "us-east-1"
      + statistic                             = "Sum"
      + tags_all                              = (known after apply)
      + threshold                             = 1
      + treat_missing_data                    = "notBreaching"
    }

  # module.base.aws_cloudwatch_metric_alarm.log_errors will be created
  + resource "aws_cloudwatch_metric_alarm" "log_errors" {
      + actions_enabled                       = true
      + alarm_actions                         = [
          + "arn:aws:sns:us-east-1:946183545209:research-catalog-team-alarms-qa",
        ]
      + alarm_description                     = "Lambda function SierraBibDeleteUpdatePoller-qa has error logs"
      + alarm_name                            = "SierraBibDeleteUpdatePollerLogErrorAlarm-qa"
      + arn                                   = (known after apply)
      + comparison_operator                   = "GreaterThanOrEqualToThreshold"
      + dimensions                            = {
          + "FunctionName" = "SierraBibDeleteUpdatePoller-qa"
        }
      + evaluate_low_sample_count_percentiles = (known after apply)
      + evaluation_periods                    = 1
      + id                                    = (known after apply)
      + metric_name                           = "SierraBibDeleteUpdatePollerLogError-qa"
      + namespace                             = "LogMetrics"
      + period                                = 300
      + region                                = "us-east-1"
      + statistic                             = "Sum"
      + tags_all                              = (known after apply)
      + threshold                             = 1
      + treat_missing_data                    = "notBreaching"
    }

  # module.base.aws_lambda_function.poller_lambda will be updated in-place
  ~ resource "aws_lambda_function" "poller_lambda" {
        id                             = "SierraBibDeleteUpdatePoller-qa"
      ~ last_modified                  = "2026-08-25T20:26:45.000+0000" -> (known after apply)
      ~ source_code_hash               = "/GghaNfjO0G9YAmscv9wLsTpxlR4YjWu9gPL2skp8XA=" -> "R6NCWEJMYXHc7orcTa3SANbdjks6fIPyI7lsfFcUJp4="
        tags                           = {}
        # (31 unchanged attributes hidden)

        # (4 unchanged blocks hidden)
    }

  # module.base.aws_s3_object.uploaded_zip will be updated in-place
  ~ resource "aws_s3_object" "uploaded_zip" {
      ~ etag                          = "2807b2bfa4fece7fecefe67851a1ae0f-3" -> "113cc3d05858f9144d29cd07379c1815"
        id                            = "SierraBibDeleteUpdatePoller-qa-dist.zip"
        tags                          = {}
      + version_id                    = (known after apply)
        # (26 unchanged attributes hidden)
    }

Plan: 3 to add, 2 to change, 0 to destroy.

[7emansell]

Lgtm why are these only on the qa instance (do we need alarms at all on qa for this) and do we need alarms for the other 3 pollers (BibUpdate, ItemDelete, ItemUpdate)? Or are those separate PRs ignore me I did not read closely! identical for all 4 on prod/qa. Only note is the threshold, I want to make sure these aren't permanently in alarm

[7emansell]

should probably be only greater than be the threshold is 1?

[KaseyW31]

I think we want to have it so that any error gets alarmed on, so it should be > 0 or >=1 ?