gitlab-ci.yml

variables:
  PROJECT: "datakit"
  CI_VERSION: "2.14.0"
  CI_TESTING_VERSION: $CI_VERSION-testing_$CI_COMMIT_BRANCH
  DCA_CI_VERSION: "0.2.0"
  DCA_CI_TESTING_VERSION: $DCA_CI_VERSION-testing_$CI_COMMIT_BRANCH
  DISABLED_MR_TARGET_BRANCHES: "/^master|main/"
  DISABLED_MR_TITLE: '/^(\[Draft\]|\(Draft\)|Draft:)/'
  FLAMESHOT_VERSION: "0.2.4"
  FLAMESHOT_TESTING_VERSION: $FLAMESHOT_VERSION-testing_$CI_COMMIT_BRANCH
  DK_GO_VERSION: "1.26.2"
  GOLANGCI_LINT_VERSION: "v2.11.4"
  GOLANGCI_LINT_BIN: "./bin/golangci-lint"
  # Release jobs enable glibc JIT packaging after downloading the runtime.
  PIPELINE_JIT_ENABLED: "0"
  PP_JIT_VERSION: "v0.1.3"
  PLATYPUS_JIT_RUNTIME_DIR: "$CI_PROJECT_DIR/.cache/pipeline-jit-runtime"

before_script:
  - uname -a
  - echo $DK_BPF_KERNEL_SRC_PATH
  - echo $CC_HTTP_PROXY
  - env | grep ^OSS_
  - |
    export GO_DIST_ARCH="$(uname -m)"
    if [ "$GO_DIST_ARCH" = "x86_64" ] || [ "$GO_DIST_ARCH" = "amd64" ]; then
      export GO_DIST_ARCH="amd64"
    elif [ "$GO_DIST_ARCH" = "aarch64" ] || [ "$GO_DIST_ARCH" = "arm64" ]; then
      export GO_DIST_ARCH="arm64"
    else
      echo "unsupported architecture: $GO_DIST_ARCH"
      exit 1
    fi
    export GO_VERSION_NORMALIZED="${DK_GO_VERSION#go}"
    export GO_DIST_TARBALL="go${GO_VERSION_NORMALIZED}.linux-${GO_DIST_ARCH}.tar.gz"
    export GO_DIST_URL="https://dl.google.com/go/${GO_DIST_TARBALL}"
    # Isolate concurrent jobs and keep Go sources outside repository-wide lint scans.
    export GO_CACHE_ROOT="${CI_PROJECT_DIR}.go-toolchain"
    export GOROOT="${GO_CACHE_ROOT}/${GO_VERSION_NORMALIZED}"
    mkdir -p "$GO_CACHE_ROOT"
    rm -rf "$GOROOT"
    echo "download golang from $GO_DIST_URL"
    https_proxy="$CC_HTTP_PROXY" curl -sSfL "$GO_DIST_URL" -o "${GO_CACHE_ROOT}/${GO_DIST_TARBALL}"
    tar -C "$GO_CACHE_ROOT" -xzf "${GO_CACHE_ROOT}/${GO_DIST_TARBALL}"
    mv "${GO_CACHE_ROOT}/go" "$GOROOT"
    export PATH="$GOROOT/bin:$PATH"
  - go version
  - https_proxy="$CC_HTTP_PROXY" curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | https_proxy="$CC_HTTP_PROXY" sh -s -- -b ./bin ${GOLANGCI_LINT_VERSION}
  # Install CGO development headers for journald and IBM i external collectors.
  # Use non-interactive mode and prevent service restarts to avoid CI job termination
  - if command -v apt-get >/dev/null 2>&1; then export DEBIAN_FRONTEND=noninteractive && printf '#!/bin/sh\nexit 101\n' > /usr/sbin/policy-rc.d && chmod +x /usr/sbin/policy-rc.d && apt-get update -qq && apt-get install -y -qq --no-install-recommends libsystemd-dev unixodbc-dev; rm -f /usr/sbin/policy-rc.d; elif command -v yum >/dev/null 2>&1; then yum install -y systemd-devel unixODBC-devel; elif command -v apk >/dev/null 2>&1; then apk add --no-cache systemd-dev unixodbc-dev; fi

stages:
  - _test_guance
  - _deploy_guance
  - _deploy_truewatch

# run lint and unit test for most of the branches
ci_lint_and_ut:
  stage: _deploy_guance
  resource_group: production
  variables:
    GOMODCACHE: "$CI_PROJECT_DIR/.cache/go/pkg/mod"
    GOLANGCI_LINT_CACHE: "$CI_PROJECT_DIR/.cache/golangci-lint"
  cache:
    key:
      prefix: ci-lint-and-ut
      files:
        - go.mod
        - go.sum
    paths:
      - .cache/go/pkg/mod
      - .cache/golangci-lint
    policy: pull-push
  rules:
    # No pipeline when there is a MR whose title starts with Draft
    - if: $CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_TITLE =~ /^(\[Draft\]|\(Draft\)|Draft:)/
      when: never

    # Trigger a pipeline in case of MR event, which means the MR is ready, since we did not go through the first case
    - if: $CI_PIPELINE_SOURCE == "merge_request_event"

  script:
    - nvm use v16.14.0
    - make clean
    - make detect_mr_target_branch MERGE_REQUEST_SOURCE_BRANCH="$CI_MERGE_REQUEST_SOURCE_BRANCH_NAME" MERGE_REQUEST_TARGET_BRANCH="$CI_MERGE_REQUEST_TARGET_BRANCH_NAME"
    - make local # build binary
    - make lint GOLINT_BINARY="$GOLANGCI_LINT_BIN" AUTO_FIX=false
    - GOOS=windows GOARCH=amd64 CGO_ENABLED=0 go test -c -o /tmp/winnetflow.test.exe ./internal/plugins/inputs/winnetflow # compile Windows-only collector and tests
    - make ut DATAWAY_URL="$TESTING_METRIC_DATAWAY" UT_EXCLUDE="$UT_EXCLUDE" UT_ONLY="$UT_ONLY" UT_IMPACTED="${UT_IMPACTED:-true}" UT_BASE_BRANCH="$CI_MERGE_REQUEST_TARGET_BRANCH_NAME" UT_PARALLEL="$UT_PARALLEL"
    - bash scripts/test-pipeline-jit-ci.sh
  tags:
    - cloudcare-ft

ci_darwin_arm64_build:
  stage: _deploy_guance
  resource_group: production
  rules:
    - if: $CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_TITLE =~ /^(\[Draft\]|\(Draft\)|Draft:)/
      when: never
    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
  script:
    # Reuse release compilation, including all DataKit variants and installer/upgrader.
    # The local target builds only; publishing stays in the testing/production jobs.
    - env -u ROBOT_TOKEN make local LOCAL_ARCHS=darwin/arm64 DIST_DIR=dist/darwin-arm64-ci SKIP_HELM=1 BRAND=guance VERSION="$CI_VERSION-ci.$CI_COMMIT_SHORT_SHA"
  tags:
    - cloudcare-ft

ci_ebpf_nocgo_build:
  stage: _deploy_guance
  rules:
    # No pipeline when there is a MR whose title starts with Draft
    # Disable merge request to master/main branch
    - if: $CI_PIPELINE_SOURCE == "merge_request_event" && $CI_MERGE_REQUEST_TITLE =~ /^(\[Draft\]|\(Draft\)|Draft:)/
      when: never

    # Trigger on MR changes that can affect datakit-ebpf nocgo build.
    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
      changes:
        - Makefile
        - externals/**/*
        - gitlab-ci.yml
        - cmd/make/build/external.go
        - cmd/make/build/external_test.go
        - internal/plugins/externals/ebpf/**/*
        - internal/plugins/inputs/ebpf/**/*

  script:
    - docker version
    - make ebpf_nocgo_ci
  tags:
    - cloudcare-ft

ut-datakit-production:
  stage: _test_guance
  resource_group: production
  only:
    - master
  script:
    - nvm use v16.14.0
    - make clean
    - make production_notify GIT_BRANCH=$CI_COMMIT_BRANCH VERSION="$CI_VERSION"
    - make lint GOLINT_BINARY="$GOLANGCI_LINT_BIN" AUTO_FIX=false
    - make ut DATAWAY_URL="$TESTING_METRIC_DATAWAY" UT_EXCLUDE="$UT_EXCLUDE" UT_PARALLEL="$UT_PARALLEL"
    - bash scripts/test-pipeline-jit-ci.sh
  tags:
    - cloudcare-ft

ut-datakit-testing:
  stage: _test_guance
  resource_group: production
  only:
    - testing
    - /^testing-.*$/
  script:
    - nvm use v16.14.0
    - make clean
    - make testing_notify GIT_BRANCH=$CI_COMMIT_BRANCH VERSION="$CI_TESTING_VERSION"
    #- make lint GOLINT_BINARY="$GOLANGCI_LINT_BIN" AUTO_FIX=false
    #- make ut DATAWAY_URL="$TESTING_METRIC_DATAWAY" UT_EXCLUDE="$UT_EXCLUDE"
    - bash scripts/test-pipeline-jit-ci.sh
  tags:
    - cloudcare-ft

release-testing-guance: # release testing packages to guance
  variables:
    PIPELINE_JIT_ENABLED: "1"
  stage: _deploy_guance
  resource_group: production
  needs: [ut-datakit-testing]
  only:
    - testing
    - /^testing-.*$/
  script:
    - export AWS_ACCESS_KEY_ID="$TESTING_AWS_ACCESS_KEY_ID"
    - export AWS_SECRET_ACCESS_KEY="$TESTING_AWS_SECRET_ACCESS_KEY"
    # building && publish testing
    - make clean
    - export OSS_HOST="$OSS_HOST_TESTING"
    - export OSS_BUCKET="$OSS_BUCKET_TESTING"
    - export OSS_ACCESS_KEY="$OSS_ACCESS_KEY_TESTING"
    - export OSS_SECRET_KEY="$OSS_SECRET_KEY_TESTING"
    - export OSS_PATH="datakit-v2"
    - |
      bash scripts/download-pipeline-jit.sh || exit $?
      case "${PIPELINE_JIT_ENABLED:-0}" in
        1|true|on|yes)
          PLATYPUS_JIT_EXPECTED_REVISION=$(cat "$PLATYPUS_JIT_RUNTIME_DIR/source-revision")
          export PLATYPUS_JIT_EXPECTED_REVISION
          ;;
      esac
    - make testing GIT_BRANCH=$CI_COMMIT_BRANCH \
        VERSION="$CI_TESTING_VERSION" \
        BRAND="guance"
    - make testing_image VERSION="$CI_TESTING_VERSION" \
        BRAND="guance" \
        DOCKER_IMAGE_REPO="pubrepo.guance.com/datakit" \
        DOCKERFILE_SUFFIX="guance" \
        DOCKER_BUILDX_CACHE_REPO="pubrepo.guance.com/datakit/build-cache" \
        FLAMESHOT_VERSION="$FLAMESHOT_TESTING_VERSION"
    - make uos_image_testing VERSION="$CI_TESTING_VERSION" \
        DOCKER_IMAGE_REPO="pubrepo.guance.com/uos-dataflux" \
        DOCKER_BUILDX_CACHE_REPO="pubrepo.guance.com/uos-dataflux/build-cache"
  tags:
    - cloudcare-ft

release-testing-tw: # release testing packages to truewatch
  variables:
    PIPELINE_JIT_ENABLED: "1"
  stage: _deploy_truewatch
  resource_group: production
  needs: [ut-datakit-testing]
  only:
    - testing
    - /^testing-.*$/
  script:
    - export AWS_ACCESS_KEY_ID="$TESTING_AWS_ACCESS_KEY_ID"
    - export AWS_SECRET_ACCESS_KEY="$TESTING_AWS_SECRET_ACCESS_KEY"
    # building && publish testing
    - make clean
    - export OSS_HOST="$OSS_HOST_TESTING"
    - export OSS_BUCKET="$OSS_BUCKET_TESTING"
    - export OSS_ACCESS_KEY="$OSS_ACCESS_KEY_TESTING"
    - export OSS_SECRET_KEY="$OSS_SECRET_KEY_TESTING"
    - export OSS_PATH="datakit-tw"
    - |
      bash scripts/download-pipeline-jit.sh || exit $?
      case "${PIPELINE_JIT_ENABLED:-0}" in
        1|true|on|yes)
          PLATYPUS_JIT_EXPECTED_REVISION=$(cat "$PLATYPUS_JIT_RUNTIME_DIR/source-revision")
          export PLATYPUS_JIT_EXPECTED_REVISION
          ;;
      esac
    - make testing GIT_BRANCH=$CI_COMMIT_BRANCH \
      VERSION="$CI_TESTING_VERSION" \
      BRAND="truewatch"
    - make testing_image VERSION="$CI_TESTING_VERSION" \
      BRAND="truewatch" \
      DOCKER_IMAGE_REPO="pubrepo.guance.com/truewatch" \
      DOCKERFILE_SUFFIX="tw" \
      DOCKER_BUILDX_CACHE_REPO="pubrepo.guance.com/truewatch/build-cache" \
      FLAMESHOT_VERSION="$FLAMESHOT_TESTING_VERSION"
  tags:
    - cloudcare-ft

release-prod-guance: # CI for guance packages
  variables:
    PIPELINE_JIT_ENABLED: "1"
  stage: _deploy_guance
  resource_group: production
  needs: [ut-datakit-production]
  only:
    - master
  except: # disable CI on tag push
    - tags
  script:
    # For guance building
    - make clean
    - export OSS_HOST="$OSS_HOST_PROD"
    - export OSS_BUCKET="$OSS_BUCKET_PROD"
    - export OSS_ACCESS_KEY="$OSS_ACCESS_KEY_PROD"
    - export OSS_SECRET_KEY="$OSS_SECRET_KEY_PROD"
    - export OSS_PATH="datakit-v2"
    - |
      bash scripts/download-pipeline-jit.sh || exit $?
      case "${PIPELINE_JIT_ENABLED:-0}" in
        1|true|on|yes)
          PLATYPUS_JIT_EXPECTED_REVISION=$(cat "$PLATYPUS_JIT_RUNTIME_DIR/source-revision")
          export PLATYPUS_JIT_EXPECTED_REVISION
          ;;
      esac
    - make production GIT_BRANCH=$CI_COMMIT_BRANCH \
      VERSION="$CI_VERSION" \
      BRAND="guance" \
      HELM_CHART_REPO="datakit-chart-cn"
    - make production_image VERSION="$CI_VERSION" \
      FLAMESHOT_VERSION="$FLAMESHOT_VERSION" \
      BRAND="guance" \
      DOCKER_IMAGE_REPO="pubrepo.guance.com/datakit" \
      DOCKERFILE_SUFFIX="guance"
    - make uos_image_production VERSION="$CI_VERSION" \
      DOCKER_IMAGE_REPO="pubrepo.guance.com/uos-dataflux"
    - make push_new_version VERSION="$CI_VERSION" \
      NEW_VERSION_PUSH_TOKEN=$NEW_VERSION_PUSH_TOKEN \
      NEW_VERSION_PUSH_URL=$NEW_VERSION_PUSH_URL
  tags:
    - cloudcare-ft

release-prod-tw: # CI for truewatch packages
  variables:
    PIPELINE_JIT_ENABLED: "1"
  stage: _deploy_truewatch
  resource_group: production
  needs: [ut-datakit-production]
  only:
    - master
  except: # disable CI on tag push
    - tags
  script:
    - export OSS_HOST="$OSS_HOST_TW"
    - export OSS_BUCKET="$OSS_BUCKET_TW"
    - export OSS_ACCESS_KEY="$OSS_ACCESS_KEY_TW"
    - export OSS_SECRET_KEY="$OSS_SECRET_KEY_TW"
    - export OSS_PATH="datakit-v2"
    - |
      bash scripts/download-pipeline-jit.sh || exit $?
      case "${PIPELINE_JIT_ENABLED:-0}" in
        1|true|on|yes)
          PLATYPUS_JIT_EXPECTED_REVISION=$(cat "$PLATYPUS_JIT_RUNTIME_DIR/source-revision")
          export PLATYPUS_JIT_EXPECTED_REVISION
          ;;
      esac
    - make production GIT_BRANCH=$CI_COMMIT_BRANCH \
      VERSION="$CI_VERSION" \
      BRAND="truewatch" \
      HELM_CHART_REPO="datakit-chart-intl"
    - make production_image GIT_BRANCH=$CI_COMMIT_BRANCH \
      VERSION="$CI_VERSION" \
      BRAND="truewatch" \
      DOCKER_IMAGE_REPO="pubrepo.guance.com/truewatch" \
      DOCKERFILE_SUFFIX="tw"
  tags:
    - cloudcare-ft

build-profiling-image-guance:
  stage: _deploy_guance
  resource_group: production
  only:
    - /^k8s_profilers[-_].*$/
  script:
    - pwd
    - cd k8s-profilers && make build DOCKER_IMAGE_REPO="pubrepo.guance.com/datakit-operator"
    - pwd
  tags:
    - cloudcare-ft

build-profiling-image-tw:
  stage: _deploy_truewatch
  resource_group: production
  only:
    - /^k8s_profilers[-_].*$/
  script:
    - pwd
    - cd k8s-profilers && make build DOCKER_IMAGE_REPO="pubrepo.guance.com/truewatch/datakit-operator"
    - pwd
  tags:
    - cloudcare-ft

build-dca-testing-image-guance:
  stage: _deploy_guance
  resource_group: production
  only:
    - /^dca-testing.*$/
  script:
    - nvm use v22.19.0
    - make clean
    - export OSS_HOST="$OSS_HOST_TESTING"
    - export OSS_BUCKET="$OSS_BUCKET_TESTING"
    - export OSS_ACCESS_KEY="$OSS_ACCESS_KEY_TESTING"
    - export OSS_SECRET_KEY="$OSS_SECRET_KEY_TESTING"
    - export OSS_PATH="dca"
    - make build_dca DCA_VERSION="$DCA_CI_TESTING_VERSION" BRAND="guance"
    - make build_dca_image_test \
      DOCKER_IMAGE_REPO="pubrepo.guance.com/tools/dca" \
      DOCKERFILE_SUFFIX="guance" \
      DOCKER_BUILDX_CACHE_REPO="pubrepo.guance.com/tools/dca-build-cache" \
      DCA_VERSION="$DCA_CI_TESTING_VERSION"
    - make build_dca_image_test \
      DOCKER_IMAGE_REPO="pubrepo.guance.com/tools/dca-uos" \
      DOCKERFILE_SUFFIX="uos" \
      DOCKER_BUILDX_CACHE_REPO="pubrepo.guance.com/tools/dca-uos-build-cache" \
      DCA_VERSION="$DCA_CI_TESTING_VERSION"
  tags:
    - cloudcare-ft

build-dca-testing-image-tw:
  stage: _deploy_truewatch
  resource_group: production
  only:
    - /^dca-testing.*$/
  script:
    - nvm use v22.19.0
    - make clean
    - export OSS_HOST="$OSS_HOST_TESTING" # we also upload truewatch DCA testing dist to the same OSS as guance, but with different subdir(dca-tw).
    - export OSS_BUCKET="$OSS_BUCKET_TESTING"
    - export OSS_ACCESS_KEY="$OSS_ACCESS_KEY_TESTING"
    - export OSS_SECRET_KEY="$OSS_SECRET_KEY_TESTING"
    - export OSS_PATH="dca-tw"
    - make build_dca DCA_VERSION="$DCA_CI_TESTING_VERSION" BRAND="truewatch"
    - make build_dca_image_test \
      DOCKER_IMAGE_REPO="pubrepo.guance.com/truewatch/dca" \
      DOCKERFILE_SUFFIX="tw" \
      DOCKER_BUILDX_CACHE_REPO="pubrepo.guance.com/truewatch/dca-build-cache" \
      DCA_VERSION="$DCA_CI_TESTING_VERSION"
      # NOTE: no UOS image for truewatch
  tags:
    - cloudcare-ft

build-dca-production-image-guance:
  stage: _deploy_guance
  resource_group: production
  rules: # We do not replease DCA related image on each release of datakit.
    - if: "$CI_COMMIT_BRANCH == 'master'"
      changes: # And if one of these files changed, we are going to release new version of DCA images.
        - dca/package.json
        - dca/Dockerfile.guance
        - dca/Dockerfile.uos
    - if: "$CI_COMMIT_BRANCH =~ /^dca-master-.*$/"
  script:
    - nvm use v22.19.0
    - make clean
    - export OSS_HOST="$OSS_HOST_PROD"
    - export OSS_BUCKET="$OSS_BUCKET_PROD"
    - export OSS_ACCESS_KEY="$OSS_ACCESS_KEY_PROD"
    - export OSS_SECRET_KEY="$OSS_SECRET_KEY_PROD"
    - export OSS_PATH="datakit/dca"
    - make build_dca DCA_VERSION="$DCA_CI_VERSION" BRAND="guance"
    - export DCA_MANIFEST_BASE_URL="https://${OSS_BUCKET}.${OSS_HOST}/${OSS_PATH}"
    - curl --fail --silent --show-error --retry 3 "${DCA_MANIFEST_BASE_URL}/dca.yaml" | grep -F "pubrepo.guance.com/tools/dca:${DCA_CI_VERSION}"
    - curl --fail --silent --show-error --retry 3 "${DCA_MANIFEST_BASE_URL}/dca-${DCA_CI_VERSION}.yaml" | grep -F "pubrepo.guance.com/tools/dca:${DCA_CI_VERSION}"
    - make build_dca_image DOCKER_IMAGE_REPO="pubrepo.guance.com/tools/dca" \
      DOCKERFILE_SUFFIX="guance" \
      DCA_VERSION="$DCA_CI_VERSION"
    - make build_dca_image DOCKER_IMAGE_REPO="pubrepo.guance.com/tools/dca-uos" \
      DOCKERFILE_SUFFIX="uos" \
      DCA_VERSION="$DCA_CI_VERSION"
  tags:
    - cloudcare-ft

build-dca-production-image-tw:
  stage: _deploy_truewatch
  resource_group: production
  rules:
    - if: "$CI_COMMIT_BRANCH == 'master'"
      changes:
        - dca/package.json
        - dca/Dockerfile.tw
    - if: "$CI_COMMIT_BRANCH =~ /^dca-master-.*$/"
  script:
    - nvm use v22.19.0
    - make clean
    - export OSS_HOST="$OSS_HOST_TW"
    - export OSS_BUCKET="$OSS_BUCKET_TW"
    - export OSS_ACCESS_KEY="$OSS_ACCESS_KEY_TW"
    - export OSS_SECRET_KEY="$OSS_SECRET_KEY_TW"
    - export OSS_PATH="datakit/dca"
    - make build_dca DCA_VERSION="$DCA_CI_VERSION" BRAND="truewatch"
    - export DCA_MANIFEST_BASE_URL="https://${OSS_BUCKET}.${OSS_HOST}/${OSS_PATH}"
    - curl --fail --silent --show-error --retry 3 "${DCA_MANIFEST_BASE_URL}/dca.yaml" | grep -F "pubrepo.truewatch.com/truewatch/dca:${DCA_CI_VERSION}"
    - curl --fail --silent --show-error --retry 3 "${DCA_MANIFEST_BASE_URL}/dca-${DCA_CI_VERSION}.yaml" | grep -F "pubrepo.truewatch.com/truewatch/dca:${DCA_CI_VERSION}"
    - make build_dca_image DOCKER_IMAGE_REPO="pubrepo.guance.com/truewatch/dca" \
      DOCKERFILE_SUFFIX="tw" \
      DCA_VERSION="$DCA_CI_VERSION"
      # NOTE: no UOS image for truewatch
  tags:
    - cloudcare-ft