diff --git a/.build/build-bench.xml b/.build/build-bench.xml
index b10fc644250e..5cd171b8a271 100644
--- a/.build/build-bench.xml
+++ b/.build/build-bench.xml
@@ -20,6 +20,11 @@
xmlns:if="ant:if" xmlns:unless="ant:unless">
+
+
+
+
+
@@ -81,6 +86,7 @@
+
@@ -103,7 +109,7 @@
-
+
@@ -111,10 +117,9 @@
-
-
+
-
+
diff --git a/.build/build-rat.xml b/.build/build-rat.xml
index 7219e0cf8aa6..4a3080918d78 100644
--- a/.build/build-rat.xml
+++ b/.build/build-rat.xml
@@ -50,12 +50,16 @@
+
+
+
+
@@ -68,8 +72,10 @@
+
+
@@ -79,6 +85,8 @@
+
+
@@ -93,6 +101,9 @@
+
+
+
diff --git a/.build/build-resolver.xml b/.build/build-resolver.xml
index 09263d42aa6e..0392dabbb406 100644
--- a/.build/build-resolver.xml
+++ b/.build/build-resolver.xml
@@ -53,11 +53,16 @@
+
+
+
+
-
-
+
+
+
+
+
+
+
+
+
-
+
+
-
+
+
@@ -269,24 +277,29 @@
+
+
+
-
-
+
+
+
-
+
+
-
+
@@ -308,6 +321,9 @@
+
+
+
diff --git a/.build/cassandra-build-deps-template.xml b/.build/cassandra-build-deps-template.xml
index cc1a25a8c1fc..95481db29c58 100644
--- a/.build/cassandra-build-deps-template.xml
+++ b/.build/cassandra-build-deps-template.xml
@@ -17,8 +17,8 @@
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
4.0.0
- cassandra-parent
- org.apache.cassandra
+ dse-db-parent
+ com.datastax.dse
@version@
@final.name@-parent.pom
@@ -135,10 +135,6 @@
com.github.tomakehurst
wiremock-jre8
-
- de.jflex
- jflex
-
com.carrotsearch.randomizedtesting
randomizedtesting-runner
@@ -155,5 +151,9 @@
org.bouncycastle
bcutil-jdk18on
+
+ com.bpodgursky
+ jbool_expressions
+
diff --git a/.build/cassandra-deps-template.xml b/.build/cassandra-deps-template.xml
index cfbc5eddaea1..281a306d9e87 100644
--- a/.build/cassandra-deps-template.xml
+++ b/.build/cassandra-deps-template.xml
@@ -17,12 +17,12 @@
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
4.0.0
- org.apache.cassandra
- cassandra-parent
+ com.datastax.dse
+ dse-db-parent
@version@
@final.name@-parent.pom
- cassandra-all
+ dse-db-all
@version@
Apache Cassandra
The Apache Cassandra Project develops a highly scalable second-generation distributed database, bringing together Dynamo's fully distributed design and Bigtable's ColumnFamily-based data model.
@@ -35,9 +35,9 @@
- scm:https://gitbox.apache.org/repos/asf/cassandra.git
- scm:https://gitbox.apache.org/repos/asf/cassandra.git
- https://gitbox.apache.org/repos/asf?p=cassandra.git
+ scm:git:ssh://git@github.com:datastax/cassandra.git
+ scm:git:ssh://git@github.com:datastax/cassandra.git
+ scm:git:ssh://git@github.com:datastax/cassandra.git
@@ -104,6 +104,14 @@
com.fasterxml.jackson.datatype
jackson-datatype-jsr310
+
+ org.msgpack
+ jackson-dataformat-msgpack
+
+
+ com.googlecode.json-simple
+ json-simple
+
com.boundary
high-scale-lib
@@ -136,6 +144,10 @@
com.clearspring.analytics
stream
+
+ com.esri.geometry
+ esri-geometry-api
+
ch.qos.logback
logback-core
@@ -201,6 +213,10 @@
net.openhft
chronicle-threads
+
+ net.openhft
+ chronicle-map
+
net.openhft
@@ -320,6 +336,10 @@
org.hdrhistogram
HdrHistogram
+
+ com.dynatrace.hash4j
+ hash4j
+
com.googlecode.concurrent-trees
concurrent-trees
@@ -368,6 +388,10 @@
org.apache.lucene
lucene-analysis-common
+
+ org.apache.lucene
+ lucene-backward-codecs
+
io.github.jbellis
jvector
@@ -376,5 +400,17 @@
com.vdurmont
semver4j
+
+ io.micrometer
+ micrometer-core
+
+
+ org.latencyutils
+ LatencyUtils
+
+
+ de.huxhorn.sulky
+ de.huxhorn.sulky.ulid
+
diff --git a/.build/checkstyle.xml b/.build/checkstyle.xml
index 2ec5ecab1610..aa5cac13d0be 100644
--- a/.build/checkstyle.xml
+++ b/.build/checkstyle.xml
@@ -169,9 +169,9 @@
-
-
-
+
+
+
diff --git a/.build/docker/_create_user.sh b/.build/docker/_create_user.sh
index 2da9f4913eec..7922e7a3565d 100755
--- a/.build/docker/_create_user.sh
+++ b/.build/docker/_create_user.sh
@@ -51,6 +51,12 @@ echo "${username} ALL=(root) NOPASSWD:ALL" > /etc/sudoers.d/${username}
chmod 0440 /etc/sudoers.d/${username}
mkdir -p ${BUILD_HOME}/docker ${DIST_DIR} ${BUILD_HOME}/.ssh
+# rsync in cached maven dependencies
+echo "Syncing maven dependencies and gradle wrapper"
+rsync -a /home/image-cache/.m2/repository/ ${BUILD_HOME}/.m2/repository/
+cp -a /home/image-cache/.gradle ${BUILD_HOME}/
+chown -R ${username}:${username} ${BUILD_HOME}/.gradle ${BUILD_HOME}/.m2
+
# we need to make SSH less strict to prevent various dtests from failing when they attempt to
# git clone a given commit/tag/etc
echo 'Host *\n UserKnownHostsFile /dev/null\n StrictHostKeyChecking no' > ${BUILD_HOME}/.ssh/config
diff --git a/.build/docker/_docker_run.sh b/.build/docker/_docker_run.sh
index 1a67840bde2e..a5da381f1385 100755
--- a/.build/docker/_docker_run.sh
+++ b/.build/docker/_docker_run.sh
@@ -31,6 +31,8 @@
# variables, with defaults
[ "x${cassandra_dir}" != "x" ] || cassandra_dir="$(readlink -f $(dirname -- "$0")/../..)"
[ "x${build_dir}" != "x" ] || build_dir="${cassandra_dir}/build"
+# parameterise the maven repository host directory, as it cannot be shared across containers
+# m2_dir fails under /tmp on macos
[ "x${m2_dir}" != "x" ] || m2_dir="${HOME}/.m2/repository"
[ -d "${build_dir}" ] || { mkdir -p "${build_dir}" ; }
[ -d "${m2_dir}" ] || { mkdir -p "${m2_dir}" ; }
@@ -99,7 +101,7 @@ if ! ( [[ "$(docker images -q ${image_name} 2>/dev/null)" != "" ]] ) ; then
if ! ( docker pull -q ${image_name} >/dev/null 2>/dev/null ) ; then
# Create build images containing the build tool-chain, Java and an Apache Cassandra git working directory, with retry
echo "Building docker image..."
- until docker build -t ${image_name} -f docker/${dockerfile} . ; do
+ until docker build -t ${image_name} -f docker/${dockerfile} --load . ; do
echo "docker build failed… trying again in 10s… "
sleep 10
done
diff --git a/.build/docker/_prepopulate_maven_deps.sh b/.build/docker/_prepopulate_maven_deps.sh
new file mode 100755
index 000000000000..34f2ccec2305
--- /dev/null
+++ b/.build/docker/_prepopulate_maven_deps.sh
@@ -0,0 +1,95 @@
+#!/bin/bash
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+# Script to prepopulate Maven repository with dependencies from multiple Cassandra branches
+# This will download all dependencies to a custom Maven repository directory
+
+# pre-conditions
+command -v ant >/dev/null 2>&1 || { error 1 "ant needs to be installed"; }
+command -v git >/dev/null 2>&1 || { error 1 "git needs to be installed"; }
+
+
+error() {
+ echo >&2 $2;
+ set -x
+ exit $1
+}
+
+# Function to download dependencies for a branch
+download_deps_for_branch() {
+ local branch=$1
+ local branch_name=$(echo "$branch" | sed 's|origin/||')
+
+ # Check if branch exists
+ if ! git rev-parse --verify "$branch" >/dev/null 2>&1; then
+ echo "WARNING: Branch $branch does not exist, skipping..."
+ return
+ fi
+
+ git checkout "$branch"
+
+ echo ""
+ echo "Downloading dependencies for $branch to $CUSTOM_M2_REPO..."
+ echo ""
+
+ # ensure git modules are initialised
+ ant init
+ # download all dependencies
+ ant -Dmaven.repo.local="$CUSTOM_M2_REPO" -Dlocal.repository="$CUSTOM_M2_REPO" resolver-dist-lib
+}
+
+CUSTOM_M2_REPO="${1:-$HOME/.m2/repository}"
+TMP_DIR=${TMP_DIR:-/tmp}
+
+cd $TMP_DIR
+git clone https://github.com/apache/cassandra.git
+cd cassandra
+git config advice.detachedHead false
+
+# Automatically detect branches from cassandra-5.0 onwards to trunk
+echo "Detecting branches..."
+BRANCHES=()
+
+# Get all origin branches matching cassandra-5.x+, cassandra-6.x+, etc., and trunk
+# Pattern matches: cassandra-5.0, cassandra-5.0.0, cassandra-10.0, cassandra-10.0.1, trunk
+while IFS= read -r branch; do
+ BRANCHES+=("$branch")
+done < <(git branch -r | grep -E "^\s*origin/(cassandra-[5-9][0-9]*\.[0-9]+(\.[0-9]+)?|trunk)$" | sed 's/^[[:space:]]*//' | sort -V)
+
+# If no branches found, fail
+if [ ${#BRANCHES[@]} -eq 0 ]; then
+ echo "ERROR: No branches auto-detected matching pattern origin/cassandra-[5+].x or origin/trunk"
+ echo "Please ensure you have fetched remote branches: git fetch origin"
+ exit 1
+fi
+
+echo "Branches to process:"
+for branch in "${BRANCHES[@]}"; do
+ echo " - $branch"
+done
+echo "=========================================="
+echo ""
+
+# Create custom Maven repository directory
+mkdir -p "$CUSTOM_M2_REPO"
+
+# Process each branch
+for branch in "${BRANCHES[@]}"; do
+ download_deps_for_branch "$branch"
+done
+
+cd -
+rm -rf $TMP_DIR/cassandra
\ No newline at end of file
diff --git a/.build/docker/almalinux-build.docker b/.build/docker/almalinux-build.docker
index 89832b67b346..cb612dc32398 100644
--- a/.build/docker/almalinux-build.docker
+++ b/.build/docker/almalinux-build.docker
@@ -26,7 +26,7 @@ ENV CASSANDRA_DIR=$BUILD_HOME/cassandra
ARG UID_ARG=1000
ARG GID_ARG=1000
-LABEL org.cassandra.buildenv=almalinux
+LABEL org.cassandra.buildenv=almalinux_build
RUN echo "Building with arguments:" \
&& echo " - DIST_DIR=${DIST_DIR}" \
@@ -55,3 +55,12 @@ RUN rpm -i --nodeps ant-junit-1.9.4-2.el7.noarch.rpm
# python3 is needed for the gen-doc target
RUN pip3 install --upgrade pip
+
+# Prepopulate Maven repository with dependencies from all branches. see _create_user.sh
+COPY docker/_prepopulate_maven_deps.sh /tmp/_prepopulate_maven_deps.sh
+RUN alternatives --set java $(alternatives --display java | grep "family java-11-openjdk" | cut -d' ' -f1)
+RUN alternatives --set javac $(alternatives --display javac | grep "family java-11-openjdk" | cut -d' ' -f1)
+RUN mkdir -p /home/image-cache && chmod -R a+rwx /home/image-cache
+RUN JAVA_HOME=$(readlink -f /usr/bin/javac | sed "s:/bin/javac::") \
+ bash /tmp/_prepopulate_maven_deps.sh /home/image-cache/.m2/repository && rm /tmp/_prepopulate_maven_deps.sh
+RUN cp -a /root/.gradle /home/image-cache/.gradle
diff --git a/.build/docker/bullseye-build.docker b/.build/docker/bullseye-build.docker
index b31bf03b3a75..fbb5654f6e98 100644
--- a/.build/docker/bullseye-build.docker
+++ b/.build/docker/bullseye-build.docker
@@ -23,7 +23,7 @@ ENV DIST_DIR=/dist
ENV BUILD_HOME=/home/build
ENV CASSANDRA_DIR=$BUILD_HOME/cassandra
-LABEL org.cassandra.buildenv=bullseye
+LABEL org.cassandra.buildenv=debian_build
RUN echo "Building with arguments:" \
&& echo " - DIST_DIR=${DIST_DIR}" \
@@ -53,21 +53,8 @@ RUN pip install --upgrade pip
# dependencies for .build/ci/ci_parser.py
RUN pip install beautifulsoup4==4.12.3 jinja2==3.1.3
-# install golang. GO_VERSION_SHA must be updated with VERSION
-RUN sh -c '\
- GO_VERSION="1.24.3" ;\
- GO_VERSION_SHAS="3333f6ea53afa971e9078895eaa4ac7204a8c6b5c68c10e6bc9a33e8e391bdd8 a463cb59382bd7ae7d8f4c68846e73c4d589f223c589ac76871b66811ded7836 13e6fe3fcf65689d77d40e633de1e31c6febbdbcb846eb05fc2434ed2213e92b 64a3fa22142f627e78fac3018ce3d4aeace68b743eff0afda8aae0411df5e4fb" ;\
- GO_OS=linux ;\
- [ $(uname) = "Darwin" ] && GO_OS=darwin ;\
- GO_PLATFORM=amd64 ;\
- [ $(uname -m) = "aarch64" ] && GO_PLATFORM=arm64 ;\
- GO_TAR="go${GO_VERSION}.${GO_OS}-${GO_PLATFORM}.tar.gz" ;\
- curl -L --fail --silent --retry 2 --retry-delay 5 --max-time 30 https://go.dev/dl/$GO_TAR -o $GO_TAR ;\
- GO_SHA="$(sha256sum $GO_TAR | cut -d" " -f2)" ;\
- echo "$GO_VERSION_SHAS" | sed "s/ /\n/g" | grep -q "$GO_SHA" || { echo "SHA256 mismatch for $GO_TAR $GO_SHA"; exit 1; } ;\
- tar -C /usr/local -xzf $GO_TAR ;\
- rm $GO_TAR'
-
-ENV GOROOT="/usr/local/go"
-ENV GOPATH="$BUILD_HOME/go"
-ENV PATH="$PATH:/usr/local/go/bin"
\ No newline at end of file
+# Prepopulate Maven repository with dependencies from all branches. see _create_user.sh
+COPY docker/_prepopulate_maven_deps.sh /tmp/_prepopulate_maven_deps.sh
+RUN mkdir -p /home/image-cache && chmod -R a+rwx /home/image-cache
+RUN bash /tmp/_prepopulate_maven_deps.sh /home/image-cache/.m2/repository && rm /tmp/_prepopulate_maven_deps.sh
+RUN cp -a /root/.gradle /home/image-cache/.gradle
\ No newline at end of file
diff --git a/.build/docker/run-tests.sh b/.build/docker/run-tests.sh
index ffade4adc8cc..a8c6a7354932 100755
--- a/.build/docker/run-tests.sh
+++ b/.build/docker/run-tests.sh
@@ -25,6 +25,8 @@
[ "x${cassandra_dir}" != "x" ] || cassandra_dir="$(readlink -f $(dirname -- "$0")/../..)"
[ "x${cassandra_dtest_dir}" != "x" ] || cassandra_dtest_dir="${cassandra_dir}/../cassandra-dtest"
[ "x${build_dir}" != "x" ] || build_dir="${cassandra_dir}/build"
+# parameterise the maven repository host directory, as it cannot be shared across containers.
+# m2_dir fails under /tmp on macos
[ "x${m2_dir}" != "x" ] || m2_dir="${HOME}/.m2/repository"
[ "x${docker_timeout_hours}" != "x" ] || docker_timeout_hours="1"
[ -d "${build_dir}" ] || { mkdir -p "${build_dir}" ; }
@@ -134,7 +136,7 @@ docker --version
pushd ${cassandra_dir}/.build >/dev/null
# build test image
-dockerfile="ubuntu2004_test.docker"
+dockerfile="ubuntu-test.docker"
image_tag="$(md5sum docker/${dockerfile} | cut -d' ' -f1)"
image_name="apache/cassandra-${dockerfile/.docker/}:${image_tag}"
docker_mounts="-v ${cassandra_dir}:/home/cassandra/cassandra -v "${build_dir}":/home/cassandra/cassandra/build -v ${m2_dir}:/home/cassandra/.m2/repository"
@@ -147,7 +149,7 @@ if ! ( [[ "$(docker images -q ${image_name} 2>/dev/null)" != "" ]] ) ; then
if ! ( docker pull -q ${image_name} >/dev/null 2>/dev/null ) ; then
# Create build images containing the build tool-chain, Java and an Apache Cassandra git working directory, with retry
echo "Building docker image..."
- until docker build -t ${image_name} -f docker/${dockerfile} . ; do
+ until docker build -t ${image_name} -f docker/${dockerfile} --load . ; do
echo "docker build failed… trying again in 10s… "
sleep 10
done
@@ -294,10 +296,11 @@ docker_command="source \${CASSANDRA_DIR}/.build/docker/_set_java.sh ${java_versi
# start the container, timeout after 4 hours
docker_id=$(docker run --name ${container_name} ${docker_flags} ${docker_envs} ${docker_mounts} ${docker_volume_opt} ${image_name} sleep ${docker_timeout_hours}h)
-echo "Running container ${container_name} ${docker_id}"
+echo "Running container ${container_name} ${docker_id} using image ${image_name}"
docker exec --user root ${container_name} bash -c "\${CASSANDRA_DIR}/.build/docker/_create_user.sh cassandra $(id -u) $(id -g)" | tee -a ${logfile}
docker exec --user root ${container_name} update-alternatives --set python /usr/bin/python${python_version} | tee -a ${logfile}
+docker exec --user root ${container_name} update-alternatives --set python3 /usr/bin/python${python_version} | tee -a ${logfile}
if [ -n "${DTEST_TMPDIR_LOCAL}" ] && [[ "${target}" =~ ^dtest-upgrade ]] ; then
# prepopulate a tmp ccm repository directory, if running dtest-upgrade tests
diff --git a/.build/docker/ubuntu-test.docker b/.build/docker/ubuntu-test.docker
new file mode 100644
index 000000000000..96039b93d96a
--- /dev/null
+++ b/.build/docker/ubuntu-test.docker
@@ -0,0 +1,219 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM ubuntu:22.04
+LABEL org.opencontainers.image.authors="Apache Cassandra "
+
+# CONTEXT is expected to be cassandra/.build
+
+ENV BUILD_HOME=/home/cassandra
+ENV CASSANDRA_DIR=$BUILD_HOME/cassandra
+ENV DIST_DIR=$CASSANDRA_DIR/build
+ENV LANG=en_US.UTF-8
+ENV LC_CTYPE=en_US.UTF-8
+ENV PYTHONIOENCODING=utf-8
+ENV PYTHONUNBUFFERED=true
+
+LABEL org.cassandra.buildenv=ubuntu_test
+
+RUN echo "Building with arguments:" \
+ && echo " - DIST_DIR=${DIST_DIR}" \
+ && echo " - BUILD_HOME=${BUILD_HOME}" \
+ && echo " - CASSANDRA_DIR=${CASSANDRA_DIR}" \
+ && echo " - UID_ARG=${UID_ARG}" \
+ && echo " - GID_ARG=${GID_ARG}"
+
+# configure apt to retry downloads
+RUN echo 'APT::Acquire::Retries "99";' > /etc/apt/apt.conf.d/80-retries
+RUN echo 'Acquire::http::Timeout "60";' > /etc/apt/apt.conf.d/80proxy.conf
+RUN echo 'Acquire::ftp::Timeout "60";' >> /etc/apt/apt.conf.d/80proxy.conf
+
+# install our python dependencies and some other stuff we need
+# libev4 libev-dev are for the python driver
+
+RUN export DEBIAN_FRONTEND=noninteractive && \
+ apt-get update && \
+ apt-get install -y --no-install-recommends software-properties-common apt-utils gnupg
+
+RUN export DEBIAN_FRONTEND=noninteractive && \
+ add-apt-repository -y ppa:deadsnakes/ppa && \
+ apt-get update && \
+ apt-get install -y curl git-core python3-pip \
+ python3.8 python3.8-venv python3.8-dev \
+ python3.10 python3.10-venv python3.10-dev \
+ python3.11 python3.11-venv python3.11-dev \
+ python3.12 python3.12-venv python3.12-dev \
+ python3.13 python3.13-venv python3.13-dev \
+ virtualenv net-tools libev4 libev-dev wget gcc libxml2 libxslt1-dev \
+ vim lsof sudo libjemalloc2 dumb-init locales rsync \
+ openjdk-8-jdk openjdk-11-jdk openjdk-17-jdk ant ant-optional
+
+RUN update-alternatives --remove java /usr/lib/jvm/java-8-openjdk-$(dpkg --print-architecture)/jre/bin/java
+RUN update-alternatives --install /usr/bin/java java /usr/lib/jvm/java-8-openjdk-$(dpkg --print-architecture)/bin/java 1081
+RUN update-alternatives --install /usr/bin/python python /usr/bin/python3.8 1
+RUN update-alternatives --install /usr/bin/python python /usr/bin/python3.10 2
+RUN update-alternatives --install /usr/bin/python python /usr/bin/python3.11 3
+RUN update-alternatives --install /usr/bin/python python /usr/bin/python3.12 4
+RUN update-alternatives --install /usr/bin/python python /usr/bin/python3.13 5
+RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.8 1
+RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.10 2
+RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.11 3
+RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.12 4
+RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.13 5
+RUN python3.8 -m pip install --upgrade pip
+
+# generate locales for the standard en_US.UTF8 value we use for testing
+RUN locale-gen en_US.UTF-8
+
+# as we only need the requirements.txt file from the dtest repo, let's just get it from GitHub as a raw asset
+# so we can avoid needing to clone the entire repo just to get this file
+RUN curl https://raw.githubusercontent.com/apache/cassandra-dtest/trunk/requirements.txt --output /opt/requirements.txt
+RUN chmod 0644 /opt/requirements.txt
+
+# now setup python via virtualenv with all of the python dependencies we need according to requirements.txt
+RUN pip3 install virtualenv virtualenv-clone
+RUN pip3 install --upgrade wheel
+
+# make Java 8 the default executable (we use to run all tests against Java 8)
+RUN update-java-alternatives --set java-1.8.0-openjdk-$(dpkg --print-architecture)
+
+# enable legacy TLSv1 and TLSv1.1 (CASSANDRA-16848)
+RUN find /etc -type f -name java.security -exec sed -i 's/TLSv1, TLSv1.1//' {} \;
+RUN find /etc -type f -name java.security -exec sed -i 's/3DES_EDE_CBC$/3DES_EDE_CBC, TLSv1, TLSv1.1/' {} \;
+
+# create and change to cassandra-tmp user, use an rare uid to avoid collision later on
+RUN mkdir -p /home/image-cache && chmod -R a+rwx /home/image-cache
+RUN adduser --disabled-login --uid 901743 --lastuid 901743 --gecos cassandra cassandra-tmp
+RUN gpasswd -a cassandra-tmp sudo
+RUN echo "cassandra-tmp ALL=(root) NOPASSWD:ALL" > /etc/sudoers.d/build
+RUN chmod 0440 /etc/sudoers.d/build
+
+# switch to the cassandra user
+RUN mkdir -p ${BUILD_HOME} && chmod a+rwx ${BUILD_HOME}
+USER cassandra-tmp
+ENV HOME=${BUILD_HOME}
+WORKDIR ${BUILD_HOME}
+
+ENV ANT_HOME=/usr/share/ant
+
+# Prepopulate Maven repository with dependencies from all branches. see _create_user.sh
+COPY docker/_prepopulate_maven_deps.sh /tmp/_prepopulate_maven_deps.sh
+RUN bash /tmp/_prepopulate_maven_deps.sh /home/image-cache/.m2/repository
+RUN cp -a /home/cassandra-tmp/.gradle /home/image-cache/.gradle
+
+# run pip commands and setup virtualenv (note we do this after we switch to cassandra user so we
+# setup the virtualenv for the cassandra user, not root) for Python 3.8-3.13
+# Don't build cython extensions when installing cassandra-driver. During test execution the driver
+# dependency is refreshed via pip install --upgrade, so that driver changes can be pulled in without
+# requiring the image to be rebuilt. Rebuilding compiled extensions is costly and is disabled by
+# default in test jobs using the CASS_DRIVER_X env vars below. However, if the extensions are
+# included in the base image, the compiled objects are not updated by pip at run time, which can
+# cause errors if the tests rely on new driver functionality or bug fixes.
+
+RUN virtualenv --python=python3.8 ${BUILD_HOME}/env3.8
+RUN chmod +x ${BUILD_HOME}/env3.8/bin/activate
+
+RUN /bin/bash -c "export CASS_DRIVER_NO_CYTHON=1 CASS_DRIVER_NO_EXTENSIONS=1 \
+ && source ${BUILD_HOME}/env3.8/bin/activate \
+ && pip3 install --upgrade pip \
+ && pip3 install -r /opt/requirements.txt \
+ && pip3 freeze --user"
+
+RUN virtualenv --python=python3.10 ${BUILD_HOME}/env3.10
+RUN chmod +x ${BUILD_HOME}/env3.10/bin/activate
+
+RUN /bin/bash -c "export CASS_DRIVER_NO_CYTHON=1 CASS_DRIVER_NO_EXTENSIONS=1 \
+ && source ${BUILD_HOME}/env3.10/bin/activate \
+ && curl -sS https://bootstrap.pypa.io/get-pip.py | python3.10 \
+ && pip3 install --upgrade \"pip<25.0\" \"setuptools==60.8.2\" wheel \
+ && pip3 install --no-build-isolation -r /opt/requirements.txt \
+ && pip3 freeze --user"
+
+RUN python3.11 -m venv ${BUILD_HOME}/env3.11
+RUN chmod +x ${BUILD_HOME}/env3.11/bin/activate
+
+RUN /bin/bash -c "export CASS_DRIVER_NO_CYTHON=1 CASS_DRIVER_NO_EXTENSIONS=1 \
+ && source ${BUILD_HOME}/env3.11/bin/activate \
+ && curl -sS https://bootstrap.pypa.io/get-pip.py | python3.11 \
+ && pip3 install --upgrade \"pip<25.0\" \"setuptools==60.8.2\" wheel \
+ && pip3 install --no-build-isolation -r /opt/requirements.txt \
+ && pip3 freeze --user"
+
+RUN virtualenv --python=python3.12 ${BUILD_HOME}/env3.12
+RUN chmod +x ${BUILD_HOME}/env3.12/bin/activate
+
+RUN /bin/bash -c "export CASS_DRIVER_NO_CYTHON=1 CASS_DRIVER_NO_EXTENSIONS=1 \
+ && source ${BUILD_HOME}/env3.12/bin/activate \
+ && curl -sS https://bootstrap.pypa.io/get-pip.py | python3.12 \
+ && pip3 install --upgrade \"pip<25.0\" \"setuptools>=65.5.0,<70.0.0\" wheel \
+ && sed -i 's/pkgutil.ImpImporter/type(\"ImpImporter\", (object,), {})/g' ${BUILD_HOME}/env3.12/lib/python3.12/site-packages/pkg_resources/__init__.py \
+ && pip3 install --no-build-isolation -r /opt/requirements.txt \
+ && pip3 freeze --user"
+
+RUN virtualenv --python=python3.13 ${BUILD_HOME}/env3.13
+RUN chmod +x ${BUILD_HOME}/env3.13/bin/activate
+
+RUN /bin/bash -c "export CASS_DRIVER_NO_CYTHON=1 CASS_DRIVER_NO_EXTENSIONS=1 \
+ && source ${BUILD_HOME}/env3.13/bin/activate \
+ && curl -sS https://bootstrap.pypa.io/get-pip.py | python3.13 \
+ && pip3 install --upgrade \"pip<25.0\" \"setuptools>=65.5.0,<70.0.0\" wheel \
+ && sed -i 's/pkgutil.ImpImporter/type(\"ImpImporter\", (object,), {})/g' ${BUILD_HOME}/env3.13/lib/python3.13/site-packages/pkg_resources/__init__.py \
+ && pip3 install --no-build-isolation -r /opt/requirements.txt \
+ && pip3 freeze --user"
+
+# 4* requires java8, sudo doesn't work on cross-platform builds
+USER root
+RUN update-alternatives --set java /usr/lib/jvm/java-8-openjdk-$(dpkg --print-architecture)/bin/java
+RUN update-alternatives --set javac /usr/lib/jvm/java-8-openjdk-$(dpkg --print-architecture)/bin/javac
+USER cassandra-tmp
+
+# Initialize the CCM git repo as well as this also can fail to clone
+RUN /bin/bash -c "source ${BUILD_HOME}/env3.8/bin/activate && \
+ ccm create -n 1 -v git:cassandra-4.1 test && ccm remove test && \
+ ccm create -n 1 -v git:cassandra-4.0 test && ccm remove test"
+
+# Initialize ccm versions. branch heads and all versions iterating through to the latest version found on downloads.apache.org/cassandra
+RUN bash -c 'source ${BUILD_HOME}/env3.8/bin/activate && \
+ latest_4_0=$(curl -s https://downloads.apache.org/cassandra/ | grep -oP "(?<=href=\")4\.0\.[0-9]+" | sort -V | tail -1 | cut -d"." -f3) && \
+ for i in $(seq 1 $latest_4_0); do echo $i ; ccm create --quiet -n 1 -v binary:4.0.$i test && ccm remove test ; done && \
+ latest_4_1=$(curl -s https://downloads.apache.org/cassandra/ | grep -oP "(?<=href=\")4\.1\.[0-9]+" | sort -V | tail -1 | cut -d"." -f3) && \
+ for i in $(seq 1 $latest_4_1); do echo $i ; ccm create --quiet -n 1 -v binary:4.1.$i test && ccm remove test ; done'
+
+# 5+ requires java11, sudo doesn't work on cross-platform builds
+USER root
+RUN update-alternatives --set java /usr/lib/jvm/java-11-openjdk-$(dpkg --print-architecture)/bin/java
+RUN update-alternatives --set javac /usr/lib/jvm/java-11-openjdk-$(dpkg --print-architecture)/bin/javac
+USER cassandra-tmp
+
+# Initialize ccm versions. branch heads and all versions iterating through to the latest version found on downloads.apache.org/cassandra
+RUN rm -fr ${BUILD_HOME}/.ccm/repository/_git_cache_apache
+RUN /bin/bash -c 'source ${BUILD_HOME}/env3.8/bin/activate && \
+ ccm create --quiet -n 1 -v git:cassandra-5.0 test && ccm remove test && \
+ ccm create --quiet -n 1 -v git:cassandra-6.0 test && ccm remove test && \
+ ccm create --quiet -n 1 -v git:trunk test && ccm remove test && \
+ latest_5_0=$(curl -s https://downloads.apache.org/cassandra/ | grep -oP "(?<=href=\")5\.0\.[0-9]+" | sort -V | tail -1 | cut -d"." -f3) && \
+ for i in $(seq 1 $latest_5_0); do echo $i ; ccm create --quiet -n 1 -v binary:5.0.$i test && ccm remove test ; done'
+ # TODO uncomment when 6.0.0 is released
+ #latest_6_0=$(curl -s https://downloads.apache.org/cassandra/ | grep -oP "(?<=href=\")6\.0\.[0-9]+" | sort -V | tail -1 | cut -d"." -f3) && \
+ #for i in $(seq 1 $latest_6_0); do echo $i ; ccm create --quiet -n 1 -v binary:6.0.$i test && ccm remove test ; done'
+
+# the .git subdirectories to pip installed cassandra-driver breaks virtualenv-clone, so just remove them
+# and other directories we don't need in image
+RUN rm -rf ${BUILD_HOME}/env*/src/cassandra-driver/.git /home/cassandra-tmp/.m2 /tmp/ccm-*.tar.gz
+# fix permissions, runtime user has different uid/gid
+RUN chmod -R og+wx ${BUILD_HOME}/.ccm ${BUILD_HOME}/env* ${BUILD_HOME}/.cache
+
+# mark "/tmp" as a volume so it will get mounted as an ext4 mount and not
+# the stupid aufs/CoW stuff that the actual docker container mounts will have.
+# we've been seeing 3+ minute hangs when calling sync on an aufs backed mount
+# so it greatly makes tests flaky as things can hang basically anywhere
+VOLUME ["/tmp"]
diff --git a/.build/docker/ubuntu2004_test.docker b/.build/docker/ubuntu2004_test.docker
deleted file mode 100644
index 9d19baef18b6..000000000000
--- a/.build/docker/ubuntu2004_test.docker
+++ /dev/null
@@ -1,155 +0,0 @@
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-FROM ubuntu:20.04
-MAINTAINER Apache Cassandra
-
-# CONTEXT is expected to be cassandra/.build
-
-ENV BUILD_HOME=/home/cassandra
-ENV CASSANDRA_DIR=$BUILD_HOME/cassandra
-ENV DIST_DIR=$CASSANDRA_DIR/build
-ENV LANG=en_US.UTF-8
-ENV LC_CTYPE=en_US.UTF-8
-ENV PYTHONIOENCODING=utf-8
-ENV PYTHONUNBUFFERED=true
-
-LABEL org.cassandra.buildenv=ubuntu_2004
-
-RUN echo "Building with arguments:" \
- && echo " - DIST_DIR=${DIST_DIR}" \
- && echo " - BUILD_HOME=${BUILD_HOME}" \
- && echo " - CASSANDRA_DIR=${CASSANDRA_DIR}" \
- && echo " - UID_ARG=${UID_ARG}" \
- && echo " - GID_ARG=${GID_ARG}"
-
-# configure apt to retry downloads
-RUN echo 'APT::Acquire::Retries "99";' > /etc/apt/apt.conf.d/80-retries
-RUN echo 'Acquire::http::Timeout "60";' > /etc/apt/apt.conf.d/80proxy.conf
-RUN echo 'Acquire::ftp::Timeout "60";' >> /etc/apt/apt.conf.d/80proxy.conf
-
-# install our python dependencies and some other stuff we need
-# libev4 libev-dev are for the python driver
-
-RUN export DEBIAN_FRONTEND=noninteractive && \
- apt-get update && \
- apt-get install -y --no-install-recommends software-properties-common apt-utils
-
-RUN export DEBIAN_FRONTEND=noninteractive && \
- add-apt-repository -y ppa:deadsnakes/ppa && \
- apt-get update && \
- apt-get install -y curl git-core python3-pip \
- python3.8 python3.8-venv python3.8-dev \
- python3.11 python3.11-venv python3.11-dev \
- virtualenv net-tools libev4 libev-dev wget gcc libxml2 libxslt1-dev \
- vim lsof sudo libjemalloc2 dumb-init locales rsync \
- openjdk-8-jdk openjdk-11-jdk openjdk-17-jdk ant ant-optional
-
-
-RUN update-alternatives --install /usr/bin/python python /usr/bin/python3.8 2
-RUN update-alternatives --install /usr/bin/python python /usr/bin/python3.11 3
-RUN python3.8 -m pip install --upgrade pip
-
-# generate locales for the standard en_US.UTF8 value we use for testing
-RUN locale-gen en_US.UTF-8
-
-# as we only need the requirements.txt file from the dtest repo, let's just get it from GitHub as a raw asset
-# so we can avoid needing to clone the entire repo just to get this file
-RUN curl https://raw.githubusercontent.com/apache/cassandra-dtest/trunk/requirements.txt --output /opt/requirements.txt
-RUN chmod 0644 /opt/requirements.txt
-
-# now setup python via virtualenv with all of the python dependencies we need according to requirements.txt
-RUN pip3 install virtualenv virtualenv-clone
-RUN pip3 install --upgrade wheel
-
-# make Java 8 the default executable (we use to run all tests against Java 8)
-RUN update-java-alternatives --set java-1.8.0-openjdk-$(dpkg --print-architecture)
-
-# enable legacy TLSv1 and TLSv1.1 (CASSANDRA-16848)
-RUN find /etc -type f -name java.security -exec sed -i 's/TLSv1, TLSv1.1//' {} \;
-RUN find /etc -type f -name java.security -exec sed -i 's/3DES_EDE_CBC$/3DES_EDE_CBC, TLSv1, TLSv1.1/' {} \;
-
-# create and change to cassandra-tmp user, use an rare uid to avoid collision later on
-RUN adduser --disabled-login --uid 901743 --lastuid 901743 --gecos cassandra cassandra-tmp
-RUN gpasswd -a cassandra-tmp sudo
-RUN echo "cassandra-tmp ALL=(root) NOPASSWD:ALL" > /etc/sudoers.d/build
-RUN chmod 0440 /etc/sudoers.d/build
-
-# switch to the cassandra user
-RUN mkdir -p ${BUILD_HOME} && chmod a+rwx ${BUILD_HOME}
-USER cassandra-tmp
-ENV HOME ${BUILD_HOME}
-WORKDIR ${BUILD_HOME}
-
-ENV ANT_HOME=/usr/share/ant
-
-# run pip commands and setup virtualenv (note we do this after we switch to cassandra user so we
-# setup the virtualenv for the cassandra user and not the root user by accident) for Python 3.8/3.11
-# Don't build cython extensions when installing cassandra-driver. During test execution the driver
-# dependency is refreshed via pip install --upgrade, so that driver changes can be pulled in without
-# requiring the image to be rebuilt. Rebuilding compiled extensions is costly and is disabled by
-# default in test jobs using the CASS_DRIVER_X env vars below. However, if the extensions are
-# included in the base image, the compiled objects are not updated by pip at run time, which can
-# cause errors if the tests rely on new driver functionality or bug fixes.
-
-RUN virtualenv --python=python3.8 ${BUILD_HOME}/env3.8
-RUN chmod +x ${BUILD_HOME}/env3.8/bin/activate
-
-RUN /bin/bash -c "export CASS_DRIVER_NO_CYTHON=1 CASS_DRIVER_NO_EXTENSIONS=1 \
- && source ${BUILD_HOME}/env3.8/bin/activate \
- && pip3 install --upgrade pip \
- && pip3 install -r /opt/requirements.txt \
- && pip3 freeze --user"
-
-RUN virtualenv --python=python3.11 ${BUILD_HOME}/env3.11
-RUN chmod +x ${BUILD_HOME}/env3.11/bin/activate
-
-RUN /bin/bash -c "export CASS_DRIVER_NO_CYTHON=1 CASS_DRIVER_NO_EXTENSIONS=1 \
- && source ${BUILD_HOME}/env3.11/bin/activate \
- && curl -sS https://bootstrap.pypa.io/get-pip.py | python3.11 \
- && pip3 install -r /opt/requirements.txt \
- && pip3 freeze --user"
-
-# Initialize the CCM git repo as well as this also can fail to clone
-RUN /bin/bash -c "source ${BUILD_HOME}/env3.8/bin/activate && \
- ccm create -n 1 -v git:cassandra-4.1 test && ccm remove test && \
- ccm create -n 1 -v git:cassandra-4.0 test && ccm remove test"
-
-# Initialize ccm versions. branch heads and all versions iterating through to the latest version found on downloads.apache.org/cassandra
-RUN bash -c 'source ${BUILD_HOME}/env3.8/bin/activate && \
- latest_4_0=$(curl -s https://downloads.apache.org/cassandra/ | grep -oP "(?<=href=\")4\.0\.[0-9]+(?=\")" | sort -V | tail -1 | cut -d"." -f3) && \
- for i in $(seq 1 $latest_4_0); do echo $i ; ccm create --quiet -n 1 -v binary:4.0.$i test && ccm remove test ; done && \
- latest_4_1=$(curl -s https://downloads.apache.org/cassandra/ | grep -oP "(?<=href=\")4\.1\.[0-9]+(?=\")" | sort -V | tail -1 | cut -d"." -f3) && \
- for i in $(seq 1 $latest_4_1); do echo $i ; ccm create --quiet -n 1 -v binary:4.1.$i test && ccm remove test ; done'
-
-# 5+ requires java11
-RUN sudo update-java-alternatives --set java-1.11.0-openjdk-$(dpkg --print-architecture)
-
-# Initialize ccm versions. branch heads and all versions iterating through to the latest version found on downloads.apache.org/cassandra
-RUN rm -fr ${BUILD_HOME}/.ccm/repository/_git_cache_apache
-RUN /bin/bash -c 'source ${BUILD_HOME}/env3.8/bin/activate && \
- ccm create --quiet -n 1 -v git:cassandra-5.0 test && ccm remove test && \
- ccm create --quiet -n 1 -v git:trunk test && ccm remove test && \
- latest_5_0=$(curl -s https://downloads.apache.org/cassandra/ | grep -oP "(?<=href=\")5\.0\.[0-9]+(?=\")" | sort -V | tail -1 | cut -d"." -f3) && \
- for i in $(seq 1 $latest_5_0); do echo $i ; ccm create --quiet -n 1 -v binary:5.0.$i test && ccm remove test ; done'
-
-# the .git subdirectories to pip installed cassandra-driver breaks virtualenv-clone, so just remove them
-# and other directories we don't need in image
-RUN rm -rf ${BUILD_HOME}/env*/src/cassandra-driver/.git /home/cassandra-tmp/.m2 /tmp/ccm-*.tar.gz
-# fix permissions, runtime user has different uid/gid
-RUN chmod -R og+wx ${BUILD_HOME}/.ccm ${BUILD_HOME}/env* ${BUILD_HOME}/.cache
-
-# mark "/tmp" as a volume so it will get mounted as an ext4 mount and not
-# the stupid aufs/CoW stuff that the actual docker container mounts will have.
-# we've been seeing 3+ minute hangs when calling sync on an aufs backed mount
-# so it greatly makes tests flaky as things can hang basically anywhere
-VOLUME ["/tmp"]
diff --git a/.build/parent-pom-template.xml b/.build/parent-pom-template.xml
index 782b1957ec40..972873921503 100644
--- a/.build/parent-pom-template.xml
+++ b/.build/parent-pom-template.xml
@@ -21,13 +21,13 @@
org.apache
22
- org.apache.cassandra
- cassandra-parent
+ com.datastax.dse
+ dse-db-parent
@version@
pom
- Apache Cassandra
- The Apache Cassandra Project develops a highly scalable second-generation distributed database, bringing together Dynamo's fully distributed design and Bigtable's ColumnFamily-based data model.
- https://cassandra.apache.org
+ Datastax DB
+ The Apache Cassandra Project develops a highly scalable second-generation distributed database. DataStax, Inc. provides additional improvements on top of Apache Cassandra
+ https://datastax.com
2009
@@ -36,8 +36,8 @@
- 1.12.13
- 4.0.20
+ 1.14.17
+ 4.0.23
0.5.1
@@ -46,7 +46,6 @@
@allocation-instrumenter.version@
@ecj.version@
@jacoco.version@
- @jflex.version@
@@ -239,9 +238,9 @@
- scm:https://gitbox.apache.org/repos/asf/cassandra.git
- scm:https://gitbox.apache.org/repos/asf/cassandra.git
- https://gitbox.apache.org/repos/asf?p=cassandra.git
+ scm:git:ssh://git@github.com:datastax/cassandra.git
+ scm:git:ssh://git@github.com:datastax/cassandra.git
+ scm:git:ssh://git@github.com:datastax/cassandra.git
@@ -291,12 +290,12 @@
org.xerial.snappy
snappy-java
- 1.1.10.4
+ 1.1.10.7
at.yawk.lz4
lz4-java
- 1.10.1
+ 1.10.2
com.github.luben
@@ -306,7 +305,7 @@
com.google.guava
guava
- 32.0.1-jre
+ 33.4.0-jre
jsr305
@@ -346,6 +345,11 @@
HdrHistogram
2.1.12
+
+ com.dynatrace.hash4j
+ hash4j
+ 0.30.0
+
commons-cli
commons-cli
@@ -412,37 +416,47 @@
ch.qos.logback
logback-core
- 1.5.18
+ 1.5.35
ch.qos.logback
logback-classic
- 1.5.18
+ 1.5.35
com.fasterxml.jackson.core
jackson-core
- 2.19.2
+ 2.21.4
com.fasterxml.jackson.core
jackson-databind
- 2.19.2
+ 2.21.4
com.fasterxml.jackson.core
jackson-annotations
- 2.19.2
+ 2.21
+
+
+ com.googlecode.json-simple
+ json-simple
+ 1.1
com.fasterxml.jackson.datatype
jackson-datatype-jsr310
- 2.19.2
+ 2.21.4
+
+
+ org.msgpack
+ jackson-dataformat-msgpack
+ 0.9.11
com.fasterxml.jackson.dataformat
jackson-dataformat-yaml
- 2.19.2
+ 2.21.4
test
@@ -465,12 +479,12 @@
org.yaml
snakeyaml
- 2.1
+ 2.4
junit
junit
- 4.12
+ 4.13
test
@@ -552,7 +566,7 @@
com.puppycrawl.tools
checkstyle
- 10.12.1
+ 10.26.1
test
@@ -746,7 +760,7 @@
io.netty
netty-all
- 4.1.130.Final
+ 4.1.136.Final
io.netty
@@ -760,10 +774,6 @@
io.netty
netty-codec-http2
-
- io.netty
- netty-codec-http
-
io.netty
netty-codec-memcache
@@ -817,7 +827,7 @@
io.netty
netty-tcnative-boringssl-static
- 2.0.70.Final
+ 2.0.74.Final
org.bouncycastle
@@ -840,18 +850,18 @@
io.netty
netty-transport-native-epoll
- 4.1.130.Final
+ 4.1.136.Final
io.netty
netty-transport-native-epoll
- 4.1.130.Final
+ 4.1.136.Final
linux-x86_64
io.netty
netty-transport-native-epoll
- 4.1.130.Final
+ 4.1.136.Final
linux-aarch_64
@@ -859,7 +869,7 @@
net.openhft
chronicle-queue
- 5.23.37
+ 5.24ea27
tools
@@ -875,7 +885,7 @@
net.openhft
chronicle-core
- 2.23.36
+ 2.24ea28
chronicle-analytics
@@ -890,7 +900,7 @@
net.openhft
chronicle-bytes
- 2.23.33
+ 2.24ea20
annotations
@@ -901,7 +911,7 @@
net.openhft
chronicle-wire
- 2.23.39
+ 2.24ea27
compiler
@@ -917,7 +927,19 @@
net.openhft
chronicle-threads
- 2.23.25
+ 2.24ea14
+
+
+
+ net.openhft
+ affinity
+
+
+
+
+ net.openhft
+ chronicle-map
+ 3.24ea4
@@ -943,7 +965,7 @@
com.google.code.findbugs
jsr305
- 2.0.2
+ 3.0.0
com.clearspring.analytics
@@ -956,6 +978,11 @@
+
+ com.esri.geometry
+ esri-geometry-api
+ 2.2.4
+
org.apache.cassandra
cassandra-driver-core
@@ -1039,17 +1066,6 @@
hppc
0.8.1
-
- de.jflex
- jflex
- ${jflex.version}
-
-
- ant
- org.apache.ant
-
-
-
com.googlecode.concurrent-trees
concurrent-trees
@@ -1218,27 +1234,32 @@
com.github.seancfoley
ipaddress
- 5.3.3
+ 5.6.2
org.agrona
agrona
- 1.17.1
+ 1.20.0
org.apache.lucene
lucene-core
- 9.7.0
+ 9.8.0
org.apache.lucene
lucene-analysis-common
- 9.7.0
+ 9.8.0
+
+
+ org.apache.lucene
+ lucene-backward-codecs
+ 9.8.0
io.github.jbellis
jvector
- 1.0.2
+ 4.0.0-rc.8-hf1
com.carrotsearch.randomizedtesting
@@ -1262,6 +1283,27 @@
semver4j
3.1.0
+
+ com.bpodgursky
+ jbool_expressions
+ 1.24
+ test
+
+
+ io.micrometer
+ micrometer-core
+ 1.5.5
+
+
+ org.latencyutils
+ LatencyUtils
+ 2.0.3
+
+
+ de.huxhorn.sulky
+ de.huxhorn.sulky.ulid
+ 8.2.0
+
diff --git a/.build/run-ci b/.build/run-ci
index 04a257d85850..2cb0f6cb61ad 100755
--- a/.build/run-ci
+++ b/.build/run-ci
@@ -203,10 +203,10 @@ def parse_arguments() -> argparse.Namespace:
"""
args = argument_parser().parse_args()
- assert args.repository.startswith("https://github.com/") and args.repository.endswith("cassandra.git"),\
+ assert args.repository.startswith("https://github.com/") and args.repository.removesuffix(".git").endswith("cassandra"),\
f"Only github apache/cassandra (forked) repository supported, got: {args.repository}"
- assert args.dtest_repository.startswith("https://github.com/") and args.dtest_repository.endswith("cassandra-dtest.git"),\
- f"Only github apache/cassandra (forked) repository supported, got: {args.dtest_repository}"
+ assert args.dtest_repository.startswith("https://github.com/") and args.dtest_repository.removesuffix(".git").endswith("cassandra-dtest"),\
+ f"Only github apache/cassandra-dtest (forked) repository supported, got: {args.dtest_repository}"
assert not (args.setup and args.only_setup), "Both --setup or --only-setup cannot be specified."
assert not (args.tear_down and args.only_tear_down), "Both --tear-down or --only-tear-down cannot be specified."
assert not ("custom" == args.profile and not args.profile_custom_regexp), "Custom profile requires --profile-custom-regexp."
@@ -305,27 +305,36 @@ def get_jenkins(k8s_client: client.CoreV1Api, args, kube_ns: str) -> Tuple[str,
return ip, server
-def trigger_jenkins_build(server: jenkins.Jenkins, job_name: str, **build_params) -> dict:
- """Triggers a Jenkins build with specified parameters and returns the queue item."""
+def ensure_job_parameters_visible(server: jenkins.Jenkins, job_name: str):
+ """
+ If necessary, triggers a non-parameter build to make parameterised builds visible.
+ """
+ job_info = server.get_job_info(job_name)
+ if any(param.get("parameterDefinitions") for param in job_info.get("property", [])):
+ return
- def check_for_parameter_build(server: jenkins.Jenkins, job_name: str):
- """
- If necessary, triggers a non-parameter build (which makes the parameterised build visible).
- """
- job_info = server.get_job_info(job_name)
- if not any(param.get("parameterDefinitions") for param in job_info.get("property", [])):
- print("Parameters are not visible; initiating non-parameter build.")
- queue_item = server.build_job(job_name)
- build_number = wait_for_build_number(server, queue_item)
- time.sleep(6)
- try:
- server.stop_build(job_name, build_number)
- except client.exceptions.ApiException as e:
- print(f"Failed to stop non-parameter build {job_name} {build_number} for job : {e}")
- print("Parameters should now be available.")
+ print(f"Parameters are not visible for job {job_name}; initiating non-parameter build.")
+ queue_item = server.build_job(job_name)
+ build_number = wait_for_build_number(server, queue_item)
+ time.sleep(6)
+ try:
+ server.stop_build(job_name, build_number)
+ except client.exceptions.ApiException as e:
+ print(f"Failed to stop non-parameter build {job_name} {build_number}: {e}")
+ print(f"Parameters should now be available for job {job_name}.")
+
+
+def ensure_cassandra_job_parameters_visible(server: jenkins.Jenkins):
+ """Ensures parameterised builds are visible for all cassandra* jobs."""
+ for job in server.get_jobs():
+ job_name = job.get("name", "")
+ if job_name.startswith("cassandra"):
+ ensure_job_parameters_visible(server, job_name)
- # Check and trigger non-parameter build if parameters are not visible
- check_for_parameter_build(server, job_name)
+
+def trigger_jenkins_build(server: jenkins.Jenkins, job_name: str, **build_params) -> dict:
+ """Triggers a Jenkins build with specified parameters and returns the queue item."""
+ ensure_job_parameters_visible(server, job_name)
print("Triggering Jenkins build… ")
return server.build_job(job_name, parameters=build_params)
@@ -818,6 +827,8 @@ def main():
install_jenkins(args.kubeconfig, args.kubecontext, DEFAULT_KUBE_NS)
(ip, server) = get_jenkins(k8s_client, args, DEFAULT_KUBE_NS)
+ if args.setup or args.only_setup:
+ ensure_cassandra_job_parameters_visible(server)
if args.only_setup:
return
if args.download_results:
diff --git a/.build/run-python-dtests.sh b/.build/run-python-dtests.sh
index f57f69668a57..be092b2c3d93 100755
--- a/.build/run-python-dtests.sh
+++ b/.build/run-python-dtests.sh
@@ -105,6 +105,7 @@ ALLOWED_DTEST_VARIANTS="large|latest|upgrade|novnode|latest"
[[ "${DTEST_TARGET}" =~ ^dtest(-(${ALLOWED_DTEST_VARIANTS}))*$ ]] || { echo >&2 "Unknown dtest target: ${DTEST_TARGET}. Allowed variants are ${ALLOWED_DTEST_VARIANTS}"; exit 1; }
java_version=$(java -version 2>&1 | awk -F '"' '/version/ {print $2}' | awk -F. '{print $1}')
+project_name=$(grep '/dev/null
- until git clone --quiet --depth 1 --no-single-branch https://github.com/apache/cassandra.git cassandra-dtest-jars ; do echo "git clone failed… trying again… " ; done
+ until git clone --quiet --depth 1 --no-single-branch --tags https://github.com/apache/cassandra.git cassandra-dtest-jars ; do echo "git clone failed… trying again… " ; done
popd >/dev/null
fi
@@ -202,7 +200,10 @@ _build_all_dtest_jars() {
[ "${java_version}" -eq 11 ] && export CASSANDRA_USE_JDK11=true
pushd ${TMP_DIR}/cassandra-dtest-jars >/dev/null
- for branch in cassandra-4.0 cassandra-4.1 cassandra-5.0 ; do
+ # Converged Core skips its corresponding branch (e.g. cassandra-5.0) as its always behind it
+ # Note: cassandra-5.0.7 tag is used instead of cassandra-5.0 branch to enable
+ # testing upgrades from 5.0.7 to the current local build for autorepair feature
+ for branch in cassandra-4.0 cassandra-4.1 cassandra-5.0.7 ; do
git clean -qxdff && git reset --hard HEAD || echo "failed to reset/clean ${TMP_DIR}/cassandra-dtest-jars… continuing…"
git checkout --quiet $branch
dtest_jar_version=$(grep 'property\s*name=\"base.version\"' build.xml |sed -ne 's/.*value=\"\([^"]*\)\".*/\1/p')
@@ -279,16 +280,85 @@ _run_testlist() {
[ "${_test_iterations}" -eq 1 ] || printf "––––\nfailure rate: ${failures}/${_test_iterations}\n"
}
+_list_microbench_tests() {
+ # Extract blacklist from build-bench.xml property (see CASSANDRA-18873)
+ local blacklist_pattern=$(grep 'name="microbench.exclude.pattern"' .build/build-bench.xml | sed -n 's/.*value="\([^"]*\)".*/\1/p')
+
+ # Find all *Bench.java files, strip prefix, sort, and filter out blacklisted ones
+ find "test/microbench" -name '*Bench.java' | \
+ sed "s;^test/microbench/;;g" | \
+ sort | \
+ grep -vE "(${blacklist_pattern})\.java$"
+}
+
+_run_microbench() {
+ local _target=$1
+ local _test_name_regexp=$2
+ local _split_chunk=$3
+ local testlist=""
+
+ # Assert no *Test.java files exist under test/microbench
+ # uncomment once CachingBenchTest and GcCompactionBenchTest are rewritten to JMH benchmarks
+ #_list_tests "microbench" | grep -q 'Test\.java$' && error 1 "Found *Test.java files under test/microbench, these should be moved to test/unit"
+
+ # Build test list from either regexp or split
+ if [ -n "${_test_name_regexp}" ]; then
+ echo "Running tests: ${_test_name_regexp}"
+ # test regexp can come in csv
+ for i in ${_test_name_regexp//,/ }; do
+ [ -n "${testlist}" ] && testlist="${testlist}"$'\n'
+ testlist="${testlist}$( _list_microbench_tests | _split_tests "${i}")"
+ done
+ [[ -z "${testlist}" ]] && error 1 "No tests found in test name regexp: ${_test_name_regexp}"
+ else
+ [ -n "${_split_chunk}" ] || { error 1 "Neither name regexp or split chunk defined"; }
+ echo "Running split: ${_split_chunk}"
+ testlist="$( _list_microbench_tests | _split_tests "${_split_chunk}")"
+ if [[ -z "${testlist}" ]]; then
+ echo "No microbench tests in split ${_split_chunk}, skipping"
+ return 0
+ fi
+ fi
+
+ # Convert file paths to the JMH classname pattern
+ local benchmark_pattern=$(echo "${testlist}" | sed 's/\.java$//g' | sed 's|^org/apache/cassandra/test/microbench/||g' | sed 's/\//./g' | tr '\n' '|' | sed 's/|$//')
+ echo "Running benchmarks: ${benchmark_pattern}"
+
+ # override build.test.output.dir, adding jdk and arch to output path for report separation
+ local -r java_version="$(java -version 2>&1 | awk -F '"' '/version/ {print $2}' | awk -F. '{print $1}')"
+ local -r arch="$(uname -m)"
+ local -r output_dir="${DIST_DIR}/test/output/${_target}/jdk${java_version}/${arch}/${_split_chunk//\//_}"
+
+ ant $_target ${ANT_TEST_OPTS} -Dbuild.test.output.dir=${output_dir} -Dbenchmark.name="${benchmark_pattern}" -Dmaven.test.failure.ignore=true
+
+ # Post-process jmh-result.json to add jdk and arch parameters
+ local jmh_result="${output_dir}/jmh-result.json"
+ if [ -f "${jmh_result}" ]; then
+ python3 -c "
+import json,sys
+with open('${jmh_result}','r') as f:
+ data=json.load(f)
+for r in (data if isinstance(data,list) else [data]):
+ if 'params' not in r:
+ r['params']={}
+ r['params']['jdk']='${java_version}'
+ r['params']['arch']='${arch}'
+with open('${jmh_result}','w') as f:
+ json.dump(data,f)
+"
+ fi
+}
+
_main() {
# parameters
local -r target="${test_target/-repeat/}"
- local -r split_chunk="${chunk:-'1/1'}" # Chunks formatted as "K/N" for the Kth chunk of N chunks
+ local -r split_chunk="${chunk:-1/1}" # Chunks formatted as "K/N" for the Kth chunk of N chunks
# check split_chunk is compatible with target (if not a regexp)
if [[ "${_split_chunk}" =~ ^\d+/\d+$ ]] && [[ "1/1" != "${split_chunk}" ]] ; then
case ${target} in
- "stress-test" | "fqltool-test" | "microbench" | "cqlsh-test" | "simulator-dtest")
- error 1 "Target ${target} does not suport splits."
+ "stress-test" | "fqltool-test" | "cqlsh-test" | "simulator-dtest")
+ error 1 "Target ${target} does not support splits."
;;
*)
;;
@@ -310,6 +380,7 @@ _main() {
# jdk check
local -r java_version=$(java -version 2>&1 | awk -F '"' '/version/ {print $2}' | awk -F. '{print $1}')
+ local -r project_name=$(grep '
-
-
-
-patch by ; reviewed by for CASSANDRA-#####
-
-Co-authored-by: Name1
-Co-authored-by: Name2
-
-```
-
-The [Cassandra Jira](https://issues.apache.org/jira/projects/CASSANDRA/issues/)
+### What is the issue
+...
+### What does this PR fix and why was it fixed
+...
diff --git a/.github/scripts/run_sonar_analysis.sh b/.github/scripts/run_sonar_analysis.sh
new file mode 100755
index 000000000000..2a66577ed2cd
--- /dev/null
+++ b/.github/scripts/run_sonar_analysis.sh
@@ -0,0 +1,152 @@
+#!/bin/bash
+set +e
+
+# Get Git/GitHub context
+GIT_BRANCH="${GITHUB_HEAD_REF:-$GITHUB_REF_NAME}"
+GIT_REPO_URL="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY"
+GIT_BASE_BRANCH="${GITHUB_BASE_REF:-main}"
+
+# SonarQube configuration (from job env vars)
+PROJECT_KEY="${SONAR_PROJECT_KEY}"
+PROJECT_NAME="${SONAR_PROJECT_NAME}"
+SONAR_HOST_URL="${SONAR_HOST}"
+
+# Retry configuration
+MAX_RETRIES=3
+RETRY_DELAY=30
+
+echo "=========================================="
+echo "SonarQube Analysis Configuration"
+echo "=========================================="
+echo "Project Key: $PROJECT_KEY"
+echo "Project Name: $PROJECT_NAME"
+echo "Branch: $GIT_BRANCH"
+echo "SonarQube Host: $SONAR_HOST_URL"
+echo "Max Retries: $MAX_RETRIES"
+echo "=========================================="
+
+# Note: Authentication uses the SONAR_TOKEN environment variable.
+# This is the standard SonarQube practice - sonar-scanner automatically
+# reads SONAR_TOKEN from the environment.
+#
+# Note: No truststore configuration needed. The whitewater.ibm.com endpoint
+# uses Cloudflare with public DigiCert certificates trusted by system CA bundles.
+
+# Build sonar-scanner arguments
+SONAR_ARGS=(
+ -Dsonar.projectKey="$PROJECT_KEY"
+ -Dsonar.projectName="$PROJECT_NAME"
+ -Dsonar.host.url="$SONAR_HOST_URL"
+ -Dsonar.token="$SONAR_TOKEN"
+ -Dsonar.links.homepage="$GIT_REPO_URL"
+ -Dsonar.qualitygate.wait=true
+)
+
+# Add PR-specific or branch-specific arguments
+if [[ "$GITHUB_EVENT_NAME" == "pull_request" ]]; then
+ GIT_SHA="${GITHUB_EVENT_PULL_REQUEST_HEAD_SHA}"
+ GIT_PR="${GITHUB_EVENT_PULL_REQUEST_NUMBER}"
+
+ SONAR_ARGS+=(
+ -Dsonar.pullrequest.key="$GIT_PR"
+ -Dsonar.pullrequest.branch="$GIT_BRANCH"
+ -Dsonar.pullrequest.base="$GIT_BASE_BRANCH"
+ -Dsonar.scm.revision="$GIT_SHA"
+ -Dsonar.links.scm="$GIT_REPO_URL/pull/$GIT_PR"
+ )
+else
+ SONAR_ARGS+=(
+ -Dsonar.branch.name="$GIT_BRANCH"
+ -Dsonar.links.scm="$GIT_REPO_URL/tree/$GIT_BRANCH"
+ )
+
+ # Only set reference branch when analyzing a branch that is different from the base branch
+ # Sonar rejects setting the reference branch to the same branch being analyzed
+ if [[ "$GIT_BRANCH" != "$GIT_BASE_BRANCH" ]]; then
+ SONAR_ARGS+=(-Dsonar.newCode.referenceBranch="$GIT_BASE_BRANCH")
+ fi
+fi
+
+# Add debug flag if enabled
+if [[ "$DEBUG_MODE" == "true" ]]; then
+ echo "Debug mode enabled: Sonar verbose output"
+ SONAR_ARGS+=(-Dsonar.verbose=true)
+fi
+
+# Retry loop
+for attempt in $(seq 1 $MAX_RETRIES); do
+ echo ""
+ echo "=========================================="
+ echo "Attempt $attempt of $MAX_RETRIES"
+ echo "=========================================="
+
+ if [[ $attempt -gt 1 ]]; then
+ echo "Waiting ${RETRY_DELAY}s before retry..."
+ sleep $RETRY_DELAY
+ RETRY_DELAY=$((RETRY_DELAY * 2))
+ fi
+
+ echo "Starting SonarQube analysis..."
+ set -x
+ sonar-scanner "${SONAR_ARGS[@]}" 2>&1 | tee sonar-output.log
+ RESULT=$?
+ set +x
+
+ # Check for Quality Gate failure first (this is NOT an error to retry)
+ if grep -q "QUALITY GATE STATUS: FAILED" sonar-output.log; then
+ echo ""
+ echo "=========================================="
+ echo "⚠️ Quality Gate FAILED"
+ echo "=========================================="
+ echo "Dashboard: $SONAR_HOST_URL/dashboard?id=$PROJECT_KEY&branch=$GIT_BRANCH"
+ echo "This is a code quality issue - fix the issues reported and re-run."
+ echo "result=quality_gate_failed" >> $GITHUB_OUTPUT
+ exit 1
+ fi
+
+ # Check for errors in log even if exit code is 0 (sonar-scanner bug in newer versions)
+ if grep -Eq "ERROR|FAILURE|BUILD FAILURE|Failed to|IllegalStateException|EXECUTION FAILURE" sonar-output.log; then
+ echo "ERROR detected in scanner output, treating as failure"
+ RESULT=1
+ fi
+
+ if [[ $RESULT -eq 0 ]]; then
+ echo ""
+ echo "=========================================="
+ echo "✓ SonarQube analysis completed successfully!"
+ echo "=========================================="
+
+ # Extract dashboard URL from report-task.txt (like Jenkins does)
+ if [[ -f ".scannerwork/report-task.txt" ]]; then
+ DASHBOARD_URL=$(grep "^dashboardUrl=" .scannerwork/report-task.txt | cut -d'=' -f2-)
+ if [[ -n "$DASHBOARD_URL" ]]; then
+ echo "Dashboard URL: $DASHBOARD_URL"
+ echo "dashboard_url=$DASHBOARD_URL" >> $GITHUB_OUTPUT
+ fi
+ fi
+
+ echo "result=success" >> $GITHUB_OUTPUT
+ exit 0
+ else
+ echo ""
+ echo "=========================================="
+ echo "✗ SonarQube analysis failed with exit code: $RESULT"
+ echo "=========================================="
+
+ # Check if this is a transient error (retry these)
+ if grep -Eqi "503|Service Unavailable|Timeout|Connection reset|temporarily unavailable|ConnectException|SocketTimeoutException" sonar-output.log; then
+ if [[ $attempt -lt $MAX_RETRIES ]]; then
+ echo "Transient error detected. Will retry..."
+ else
+ echo "Max retries reached. Failing."
+ echo "result=failure" >> $GITHUB_OUTPUT
+ exit $RESULT
+ fi
+ else
+ # Other errors - don't retry
+ echo "Non-transient error. Not retrying."
+ echo "result=failure" >> $GITHUB_OUTPUT
+ exit $RESULT
+ fi
+ fi
+done
diff --git a/.github/workflows/checklist_comment_on_new_pr.yml b/.github/workflows/checklist_comment_on_new_pr.yml
new file mode 100644
index 000000000000..c7af619972e4
--- /dev/null
+++ b/.github/workflows/checklist_comment_on_new_pr.yml
@@ -0,0 +1,18 @@
+name: Comment on new Pull Request with checklist
+on:
+ pull_request:
+ types: opened
+
+jobs:
+ checklist-comment:
+ runs-on: ubuntu-latest
+ steps:
+ - name: Checkout Repository
+ uses: actions/checkout@v4
+ - name: Add comment
+ run: |
+ sed "s/{{PR_NUMBER}}/$PRNUM/" .github/workflows/pr_checklist.md | gh pr comment $PRNUM --body-file -
+ env:
+ GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ GH_REPO: ${{ github.repository }}
+ PRNUM: ${{ github.event.pull_request.number }}
diff --git a/.github/workflows/pr_checklist.md b/.github/workflows/pr_checklist.md
new file mode 100644
index 000000000000..e32f5998d9a4
--- /dev/null
+++ b/.github/workflows/pr_checklist.md
@@ -0,0 +1,12 @@
+### Checklist before you submit for review
+- [ ] This PR adheres to [the Definition of Done](https://github.com/riptano/cndb/blob/main/DEFINITION_OF_DONE.md)
+- [ ] Make sure there is a PR and ticket in the CNDB project updating the Converged Cassandra version
+- [ ] Use `NoSpamLogger` for log lines that may appear frequently in the logs
+- [ ] Verify test results on Butler
+- [ ] Test coverage for new/modified code is > 80%, check manually at [SonarCloud page](https://sonarcloud.io/summary/new_code?id=cassandra-stargazer&pullRequest={{PR_NUMBER}})
+- [ ] Proper code formatting
+- [ ] Proper title for each commit staring with the project-issue number, like CNDB-1234
+- [ ] Each commit has a meaningful description
+- [ ] Each commit is not very long and contains related changes
+- [ ] Renames, moves and reformatting are in distinct commits
+- [ ] All new files should contain the IBM copyright header instead of the Apache License one (no DataStax copyright any longer)
diff --git a/.github/workflows/sonarqube-scan.yaml b/.github/workflows/sonarqube-scan.yaml
new file mode 100644
index 000000000000..bbcbea154657
--- /dev/null
+++ b/.github/workflows/sonarqube-scan.yaml
@@ -0,0 +1,143 @@
+name: SonarQube Code Quality Scan
+
+on:
+ workflow_dispatch:
+ inputs:
+ branch:
+ description: 'Branch to scan (default: main)'
+ required: false
+ default: 'main'
+ type: string
+ debug:
+ description: 'Enable debug output (Sonar verbose)'
+ required: false
+ default: false
+ type: boolean
+
+concurrency:
+ group: sonarqube-${{ github.ref_name }}
+ cancel-in-progress: true
+
+jobs:
+ sonar-analysis:
+ runs-on: ubuntu-latest
+ name: SonarQube Analysis
+ env:
+ SONAR_HOST: https://sonarqube-prod.whitewater.ibm.com
+ SONAR_DASHBOARD_HOST: https://sonarqube-prod.apps.wdc-sonarqube-prod.core.cirrus.ibm.com
+ SONAR_PROJECT_KEY: 544478-247111689
+ SONAR_PROJECT_NAME: datastax/cassandra
+ steps:
+ - name: Set target branch
+ id: set-branch
+ run: |
+ if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then
+ echo "branch=${{ inputs.branch }}" >> $GITHUB_OUTPUT
+ else
+ echo "branch=${{ github.ref_name }}" >> $GITHUB_OUTPUT
+ fi
+
+ - name: Checkout code
+ uses: actions/checkout@v4
+ with:
+ fetch-depth: 0
+ ref: ${{ steps.set-branch.outputs.branch }}
+
+ - name: Set up JDK 11
+ uses: actions/setup-java@v4
+ with:
+ distribution: 'temurin'
+ java-version: '11'
+
+ - name: Cache Ant dependencies
+ uses: actions/cache@v4
+ with:
+ path: |
+ ~/.ant
+ lib
+ key: ant-${{ hashFiles('build.xml') }}
+ restore-keys: ant-
+
+ - name: Build Cassandra
+ run: |
+ echo "Building Cassandra..."
+ ant clean jar
+ echo "Build completed"
+
+ - name: Check SonarQube server availability
+ id: health-check
+ continue-on-error: true
+ run: |
+ echo "Checking SonarQube server health..."
+ response=$(curl -s -o /dev/null -w "%{http_code}" $SONAR_HOST/api/system/status 2>&1)
+ echo "Server response code: $response"
+ if [[ "$response" == "200" ]]; then
+ echo "server_available=true" >> $GITHUB_OUTPUT
+ else
+ echo "server_available=false" >> $GITHUB_OUTPUT
+ fi
+
+ - name: Install SonarQube Scanner
+ run: |
+ echo "Installing SonarQube Scanner CLI..."
+ wget -q https://binaries.sonarsource.com/Distribution/sonar-scanner-cli/sonar-scanner-cli-8.1.0.6389-linux-x64.zip
+ unzip -q sonar-scanner-cli-8.1.0.6389-linux-x64.zip
+ echo "$(pwd)/sonar-scanner-8.1.0.6389-linux-x64/bin" >> $GITHUB_PATH
+
+ - name: Run SonarQube analysis
+ id: sonar-scan
+ env:
+ SONAR_TOKEN: ${{ secrets.IBM_SONARQUBE_API_TOKEN }}
+ GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+ DEBUG_MODE: ${{ inputs.debug }}
+ run: |
+ chmod +x .github/scripts/run_sonar_analysis.sh
+ .github/scripts/run_sonar_analysis.sh
+
+ - name: Publish analysis summary
+ if: always()
+ run: |
+ echo "## SonarQube Analysis Summary" >> $GITHUB_STEP_SUMMARY
+ echo "" >> $GITHUB_STEP_SUMMARY
+
+ if [[ "${{ steps.sonar-scan.outputs.result }}" == "success" ]]; then
+ echo "[OK] **Status**: Analysis completed successfully" >> $GITHUB_STEP_SUMMARY
+ elif [[ "${{ steps.sonar-scan.outputs.result }}" == "quality_gate_failed" ]]; then
+ echo "[WARN] **Status**: Quality Gate Failed" >> $GITHUB_STEP_SUMMARY
+ echo "" >> $GITHUB_STEP_SUMMARY
+ echo "**This is a code quality issue, not an infrastructure problem.**" >> $GITHUB_STEP_SUMMARY
+ echo "" >> $GITHUB_STEP_SUMMARY
+ echo "**Next steps:**" >> $GITHUB_STEP_SUMMARY
+ echo "1. Click the dashboard link below to see which quality conditions failed" >> $GITHUB_STEP_SUMMARY
+ echo "2. Common issues: low test coverage, code smells, bugs, security hotspots, duplicated code" >> $GITHUB_STEP_SUMMARY
+ echo "3. Fix the code quality issues and push again" >> $GITHUB_STEP_SUMMARY
+ else
+ echo "[ERROR] **Status**: Analysis failed" >> $GITHUB_STEP_SUMMARY
+ fi
+
+ echo "" >> $GITHUB_STEP_SUMMARY
+ echo "**Branch**: \`${{ steps.set-branch.outputs.branch }}\`" >> $GITHUB_STEP_SUMMARY
+ echo "**Run ID**: ${{ github.run_id }}" >> $GITHUB_STEP_SUMMARY
+ echo "" >> $GITHUB_STEP_SUMMARY
+
+ if [[ "${{ steps.health-check.outputs.server_available }}" == "false" ]]; then
+ echo "[WARN] **Warning**: SonarQube server health check failed before analysis" >> $GITHUB_STEP_SUMMARY
+ echo "" >> $GITHUB_STEP_SUMMARY
+ fi
+
+ echo "### Links" >> $GITHUB_STEP_SUMMARY
+
+ # Use extracted dashboard URL if available, otherwise construct it
+ DASHBOARD_URL="${{ steps.sonar-scan.outputs.dashboard_url }}"
+ if [[ -n "$DASHBOARD_URL" ]]; then
+ echo "- [SonarQube Dashboard]($DASHBOARD_URL)" >> $GITHUB_STEP_SUMMARY
+ else
+ echo "- [SonarQube Dashboard]($SONAR_DASHBOARD_HOST/dashboard?id=$SONAR_PROJECT_KEY&branch=${{ steps.set-branch.outputs.branch }})" >> $GITHUB_STEP_SUMMARY
+ fi
+
+ echo "- [Workflow Run](https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }})" >> $GITHUB_STEP_SUMMARY
+ echo "" >> $GITHUB_STEP_SUMMARY
+
+ - name: Fail job if analysis failed
+ if: steps.sonar-scan.outputs.result != 'success'
+ run: exit 1
diff --git a/.gitignore b/.gitignore
index 4531bc1e6773..3210784b8286 100644
--- a/.gitignore
+++ b/.gitignore
@@ -10,6 +10,7 @@ logs/
data/
!test/data
conf/hotspot_compiler
+doc/antora.yml
doc/cql3/CQL.html
doc/build/
lib/
@@ -60,15 +61,22 @@ nbactions.xml
#VS code
.vscode/
+# Aider (aider.chat)
+.aider*
+
# Maven, etc.
out/
target/
+# AI agents
+.ai/
+
# General
**/__pycache__
*.pyc
*~
*.bak
+*.log
*.sw[o,p]
*.tmp
.DS_Store
@@ -93,3 +101,6 @@ cassandra-builds/
cassandra-dtest/
conf/triggers/trigger-example.jar
+
+agent_log
+.bob
\ No newline at end of file
diff --git a/.jenkins/Jenkinsfile b/.jenkins/Jenkinsfile
index faf3f410cf24..8db95dec44c3 100644
--- a/.jenkins/Jenkinsfile
+++ b/.jenkins/Jenkinsfile
@@ -1,375 +1,85 @@
-#!/usr/bin/env groovy
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-//
-//
-// Jenkins CI declaration.
-//
-// The declarative pipeline is presented first as a high level view.
-//
-// Build and Test Stages are dynamic, the full possible list defined by the `tasks()` function.
-// There is a choice of pipeline profles with sets of tasks that are run, see `pipelineProfiles()`.
-//
-// All tasks use the dockerised CI-agnostic scripts found under `.build/docker/`
-// The `type: test` always `.build/docker/run-tests.sh`
-//
-//
-// This Jenkinsfile is expected to work on any Jenkins infrastructure.
-// The controller should have 4 cpu, 12GB ram (and be configured to use `-XX:+UseG1GC -Xmx8G`)
-//
-// It is required to have agents providing 6+ labels, each that can provide docker and the following capabilities:
-//
-// - cassandra-small + cassandra-${arch}-small : 1 cpu, 1GB ram (alias for above but for any arch)
-// - cassandra-medium + cassandra-${arch}-medium : 3 cpu, 5GB ram
-// - cassandra-large + cassandra-${arch}-large : 7 cpu, 16GB ram
-//
-// Performance targets required a `cassandra-${arch}-large-dedicated` labelled nodes.
-//
-// When running builds parameterised to other architectures the corresponding labels are expected.
-// For example 'arm64' requires the labels: cassandra-arm64-small, cassandra-arm64-medium, cassandra-arm64-large.
-//
-// Plugins required are:
-// git, workflow-job, workflow-cps, junit, workflow-aggregator, ws-cleanup, pipeline-build-step, test-stability, copyartifact, jmh-report.
-// See .jenkins/k8s/jenkins-deployment.yaml for up to date list of plugins.
-//
-// Any functionality that depends upon ASF Infra ( i.e. the canonical ci-cassandra.a.o )
-// will be ignored when run on other environments.
-// Note there are also differences when CI is being run pre- or post-commit.
-//
-// CAUTION! When running CI with changes in this file, ensure the "Pipeline script from SCM" scm details match
-// the brances being tested. These details don't honour the per-build repository and branch parameterisation.
-//
-// Validate/lint this file using the following command
-// `curl -X POST -F "jenkinsfile=<.jenkins/Jenkinsfile" https://ci-cassandra.apache.org/pipeline-model-converter/validate`
-//
-
-/** CONSTANTS for both the pipeline and scripting **/
-import groovy.transform.Field
-@Field List archsSupported = ["amd64", "arm64"]
-@Field List pythonsSupported = ["3.8", "3.11"]
-@Field String pythonDefault = "3.8"
-/** CONSTANTS end **********************************/
-
-pipeline {
- agent { label 'cassandra-small' }
- options {
- // must have: avoids agents waste in idle time on controller bottleneck
- durabilityHint('PERFORMANCE_OPTIMIZED')
- disableResume()
- }
- parameters {
- string(name: 'repository', defaultValue: params.repository ?: scm.userRemoteConfigs[0].url, description: 'Cassandra Repository')
- string(name: 'branch', defaultValue: params.branch ?: scm.userRemoteConfigs[0].refspec, description: 'Branch')
-
- choice(name: 'profile', choices: pipelineProfileNames(params.profile ?: ''), description: 'Pick a pipeline profile.')
- string(name: 'profile_custom_regexp', defaultValue: params.profile_custom_regexp ?: '', description: 'Regexp for stages when using custom profile. See `testSteps` in Jenkinsfile for list of stages. Example: stress.*|jvm-dtest.*')
-
- choice(name: 'architecture', choices: archsSupported + "all", description: 'Pick architecture. The ARM64 is disabled by default at the moment.')
- string(name: 'jdk', defaultValue: params.jdk ?: '', description: 'Restrict JDK versions. (e.g. "11", "17", etc)')
-
- string(name: 'dtest_repository', defaultValue: params.dtest_repository ?: 'https://github.com/apache/cassandra-dtest', description: 'Cassandra DTest Repository')
- string(name: 'dtest_branch', defaultValue: params.dtest_branch ?: 'trunk', description: 'DTest Branch')
- }
- stages {
- stage('init') {
- steps {
- script {
- // this helps assure folk their parameters are correct and will be used (despite the earlier output about the configured job coordinates)
- echo "Printing parameters used for this build"
- ["Repository: ${params.repository}", "Branch: ${params.branch}", "Profile: ${params.profile}", "Custom Profile Regexp: ${params.profile_custom_regexp}", "Architecture: ${params.architecture}", "JDK: ${params.jdk}", "DTest Repository: ${params.dtest_repository}", "DTest Branch: ${params.dtest_branch}"].each { println it }
- }
- }
- }
- stage('jar') {
- // the jar stage executes only the 'jar' build step, via the build(…) function
- // the results of these (per jdk, per arch) are then stashed and used for every other build and test step
- steps {
- script {
- parallel(getJarTasks())
- }
- }
- }
- stage('Tests') {
- // the Tests stage executes all other build and task steps.
- // build steps are sent to the build(…) function, test steps sent to the test(…) function
- // these steps are parameterised and split by the tasks() function
- when {
- expression { hasNonJarTasks() }
- }
- steps {
- script {
- parallel(tasks()['tests'])
- }
- }
- }
- stage('Summary') {
- // generate the ci_summary.html and results_details.tar.xz artefacts
- steps {
- generateTestReports()
- }
- }
- }
- post {
- failure {
- echo "ERROR pipeline failed – not all tests were run"
- }
- always {
- sendNotifications()
- }
- }
+#!groovy
+
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+properties([
+ buildDiscarder(logRotator(daysToKeepStr: '14', numToKeepStr: '512')),
+ disableConcurrentBuilds(),
+ parameters([
+ string(defaultValue: 'https://github.com/apache/cassandra.git', description: 'What git repository should be used to build and test from?', name: 'repository'),
+ string(defaultValue: 'trunk', description: 'What git branch should be used to build and test from?', name: 'branch'),
+ string(defaultValue: 'https://github.com/apache/cassandra-dtest.git', description: 'What git repository should be used to build and test Python dtests from?', name: 'dtest_repository'),
+ string(defaultValue: 'trunk', description: 'What git branch should be used to build and test Python dtests from?', name: 'dtest_branch'),
+ text(defaultValue: '{"steps": ["jar", "jvm-dtest-upgrade", "jvm-dtest", "jvm-dtest-upgrade-40", "jvm-dtest-upgrade-41", "jvm-upgrade-dtest", "jvm-upgrade-dtest-vnode", "jvm-upgrade-dtest-no-vnode", "jvm-upgrade-dtest-large", "jvm-upgrade-dtest-40", "jvm-upgrade-dtest-41", "jvm-upgrade-dtest-ssl", "jvm-upgrade-dtest-no-preview", "jvm-upgrade-dtest-storage", "jvm-upgrade-dtest-latest-killer", "jvm-upgrade-dtest-tls", "jvm-upgrade-dtest-auth", "jvm-dtest-latest", "jvm-dtest-latest-vnode", "jvm-dtest-latest-cdc", "jvm-dtest-latest-auth", "jvm-dtest-latest-compression", "jvm-dtest-latest-legacy-sstable", "jvm-dtest-latest-oa", "jvm-dtest-latest-large", "jvm-dtest-latest-no-vnode", "jvm-dtest-latest-sai", "jvm-dtest-latest-ssl", "jvm-dtest-latest-tls", "jvm-dtest-latest-upgrade", "jvm-dtest-latest-vnode-upgrade", "jvm-dtest-latest-trigger", "jvm-dtest-latest-materialized-view", "jvm-dtest-latest-transient-replication", "jvm-dtest-latest-counters", "jvm-dtest-latest-repair", "jvm-dtest-latest-secondary-index", "jvm-dtest-latest-paging", "jvm-dtest-latest-topology", "jvm-dtest-latest-system-keyspace-directory", "jvm-dtest-latest-stress", "jvm-dtest-latest-fqltool", "jvm-dtest-latest-reverse-query", "jvm-dtest-latest-cql", "jvm-dtest-latest-jmx", "jvm-dtest-latest-sql", "jvm-dtest-latest-bti", "jvm-dtest-latest-accord", "microbench", "cqlsh-test"], "cells": [{"step": "jar", "arch": "amd64", "jdk": "11", "python": "3.8", "cython": "no"}, {"step": "jvm-dtest-upgrade", "arch": "amd64", "jdk": "11", "python": "3.8", "cython": "no"}, {"step": "jvm-dtest", "arch": "amd64", "jdk": "11", "python": "3.8", "cython": "no"}, {"step": "jvm-dtest-upgrade-40", "arch": "amd64", "jdk": "11", "python": "3.8", "cython": "no"}, {"step": "jvm-dtest-upgrade-41", "arch": "amd64", "jdk": "11", "python": "3.8", "cython": "no"}, {"step": "jvm-upgrade-dtest", "arch": "amd64", "jdk": "11", "python": "3.8", "cython": "no"}, {"step": "jvm-upgrade-dtest-vnode", "arch": "amd64", "jdk": "11", "python": "3.8", "cython": "no"}, {"step": "jvm-upgrade-dtest-no-vnode", "arch": "amd64", "jdk": "11", "python": "3.8", "cython": "no"}, {"step": "jvm-upgrade-dtest-large", "arch": "amd64", "jdk": "11", "python": "3.8", "cython": "no"}, {"step": "jvm-upgrade-dtest-40", "arch": "amd64", "jdk": "11", "python": "3.8", "cython": "no"}, {"step": "jvm-upgrade-dtest-41", "arch": "amd64", "jdk": "11", "python": "3.8", "cython": "no"}, {"step": "jvm-upgrade-dtest-ssl", "arch": "amd64", "jdk": "11", "python": "3.8", "cython": "no"}, {"step": "jvm-upgrade-dtest-no-preview", "arch": "amd64", "jdk": "11", "python": "3.8", "cython": "no"}, {"step": "jvm-upgrade-dtest-storage", "arch": "amd64", "jdk": "11", "python": "3.8", "cython": "no"}, {"step": "jvm-upgrade-dtest-latest-killer", "arch": "amd64", "jdk": "11", "python": "3.8", "cython": "no"}, {"step": "jvm-upgrade-dtest-tls", "arch": "amd64", "jdk": "11", "python": "3.8", "cython": "no"}, {"step": "jvm-upgrade-dtest-auth", "arch": "amd64", "jdk": "11", "python": "3.8", "cython": "no"}, {"step": "jvm-dtest-latest", "arch": "amd64", "jdk": "11", "python": "3.8", "cython": "no"}, {"step": "jvm-dtest-latest-vnode", "arch": "amd64", "jdk": "11", "python": "3.8", "cython": "no"}, {"step": "jvm-dtest-latest-cdc", "arch": "amd64", "jdk": "11", "python": "3.8", "cython": "no"}, {"step": "jvm-dtest-latest-auth", "arch": "amd64", "jdk": "11", "python": "3.8", "cython": "no"}, {"step": "jvm-dtest-latest-compression", "arch": "amd64", "jdk": "11", "python": "3.8", "cython": "no"}, {"step": "jvm-dtest-latest-legacy-sstable", "arch": "amd64", "jdk": "11", "python": "3.8", "cython": "no"}, {"step": "jvm-dtest-latest-oa", "arch": "amd64", "jdk": "11", "python": "3.8", "cython": "no"}, {"step": "jvm-dtest-latest-large", "arch": "amd64", "jdk": "11", "python": "3.8", "cython": "no"}, {"step": "jvm-dtest-latest-no-vnode", "arch": "amd64", "jdk": "11", "python": "3.8", "cython": "no"}, {"step": "jvm-dtest-latest-sai", "arch": "amd64", "jdk": "11", "python": "3.8", "cython": "no"}, {"step": "jvm-dtest-latest-ssl", "arch": "amd64", "jdk": "11", "python": "3.8", "cython": "no"}, {"step": "jvm-dtest-latest-tls", "arch": "amd64", "jdk": "11", "python": "3.8", "cython": "no"}, {"step": "jvm-dtest-latest-upgrade", "arch": "amd64", "jdk": "11", "python": "3.8", "cython": "no"}, {"step": "jvm-dtest-latest-vnode-upgrade", "arch": "amd64", "jdk": "11", "python": "3.8", "cython": "no"}, {"step": "jvm-dtest-latest-trigger", "arch": "amd64", "jdk": "11", "python": "3.8", "cython": "no"}, {"step": "jvm-dtest-latest-materialized-view", "arch": "amd64", "jdk": "11", "python": "3.8", "cython": "no"}, {"step": "jvm-dtest-latest-transient-replication", "arch": "amd64", "jdk": "11", "python": "3.8", "cython": "no"}, {"step": "jvm-dtest-latest-counters", "arch": "amd64", "jdk": "11", "python": "3.8", "cython": "no"}, {"step": "jvm-dtest-latest-repair", "arch": "amd64", "jdk": "11", "python": "3.8", "cython": "no"}, {"step": "jvm-dtest-latest-secondary-index", "arch": "amd64", "jdk": "11", "python": "3.8", "cython": "no"}, {"step": "jvm-dtest-latest-paging", "arch": "amd64", "jdk": "11", "python": "3.8", "cython": "no"}, {"step": "jvm-dtest-latest-topology", "arch": "amd64", "jdk": "11", "python": "3.8", "cython": "no"}, {"step": "jvm-dtest-latest-system-keyspace-directory", "arch": "amd64", "jdk": "11", "python": "3.8", "cython": "no"}, {"step": "jvm-dtest-latest-stress", "arch": "amd64", "jdk": "11", "python": "3.8", "cython": "no"}, {"step": "jvm-dtest-latest-fqltool", "arch": "amd64", "jdk": "11", "python": "3.8", "cython": "no"}, {"step": "jvm-dtest-latest-reverse-query", "arch": "amd64", "jdk": "11", "python": "3.8", "cython": "no"}, {"step": "jvm-dtest-latest-cql", "arch": "amd64", "jdk": "11", "python": "3.8", "cython": "no"}, {"step": "jvm-dtest-latest-jmx", "arch": "amd64", "jdk": "11", "python": "3.8", "cython": "no"}, {"step": "jvm-dtest-latest-sql", "arch": "amd64", "jdk": "11", "python": "3.8", "cython": "no"}, {"step": "jvm-dtest-latest-bti", "arch": "amd64", "jdk": "11", "python": "3.8", "cython": "no"}, {"step": "jvm-dtest-latest-accord", "arch": "amd64", "jdk": "11", "python": "3.8", "cython": "no"}, {"step": "microbench", "arch": "amd64", "jdk": "11", "python": "3.8", "cython": "no"}, {"step": "cqlsh-test", "arch": "amd64", "jdk": "11", "python": "3.8", "cython": "yes"}, {"step": "cqlsh-test", "arch": "amd64", "jdk": "11", "python": "3.8", "cython": "no"}, {"step": "cqlsh-test", "arch": "amd64", "jdk": "11", "python": "3.10", "cython": "yes"}, {"step": "cqlsh-test", "arch": "amd64", "jdk": "11", "python": "3.10", "cython": "no"}, {"step": "cqlsh-test", "arch": "amd64", "jdk": "11", "python": "3.11", "cython": "yes"}, {"step": "cqlsh-test", "arch": "amd64", "jdk": "11", "python": "3.11", "cython": "no"}, {"step": "cqlsh-test", "arch": "amd64", "jdk": "11", "python": "3.12", "cython": "yes"}, {"step": "cqlsh-test", "arch": "amd64", "jdk": "11", "python": "3.12", "cython": "no"}, {"step": "cqlsh-test", "arch": "amd64", "jdk": "11", "python": "3.13", "cython": "yes"}, {"step": "cqlsh-test", "arch": "amd64", "jdk": "11", "python": "3.13", "cython": "no"}], "splits": 4}', description: 'Describe all test runs', name: 'config')
+ ])
+])
+
+def getConfig() {
+ return readJSON(text: params.config)
}
-///////////////////////////
-//// scripting support ////
-///////////////////////////
-
-@NonCPS
-def pipelineProfiles() {
- return [
- 'packaging': ['artifacts', 'lint', 'debian', 'redhat'],
- 'skinny': ['lint', 'cqlsh-test', 'test', 'jvm-dtest', 'simulator-dtest', 'dtest'],
- 'pre-commit': ['artifacts', 'lint', 'debian', 'redhat', 'fqltool-test', 'cqlsh-test', 'test', 'test-latest', 'stress-test', 'test-burn', 'jvm-dtest', 'simulator-dtest', 'dtest', 'dtest-latest', 'microbench-test'],
- 'pre-commit w/ upgrades': ['artifacts', 'lint', 'debian', 'redhat', 'fqltool-test', 'cqlsh-test', 'test', 'test-latest', 'stress-test', 'test-burn', 'jvm-dtest', 'jvm-dtest-upgrade', 'simulator-dtest', 'dtest', 'dtest-novnode', 'dtest-latest', 'dtest-upgrade', 'microbench-test'],
- 'post-commit': ['artifacts', 'lint', 'debian', 'redhat', 'fqltool-test', 'cqlsh-test', 'test-cdc', 'test', 'test-latest', 'test-compression', 'stress-test', 'test-burn', 'long-test', 'test-oa', 'test-system-keyspace-directory', 'jvm-dtest', 'jvm-dtest-upgrade', 'simulator-dtest', 'dtest', 'dtest-novnode', 'dtest-latest', 'dtest-large', 'dtest-large-novnode', 'dtest-large-latest', 'dtest-upgrade', 'dtest-upgrade-novnode', 'dtest-upgrade-large', 'dtest-upgrade-large-novnode', 'microbench-test'],
- 'performance': ['microbench'],
- 'custom': []
- ]
-}
-
-@NonCPS
-def pipelineProfileNames(putFirst) {
- set = pipelineProfiles().keySet() as List
- set = set - putFirst
- set.add(0, putFirst)
- return set
-}
-
-@Field Map cachedTasks = null
-
-def tasks() {
- if (null != cachedTasks) return cachedTasks
-
- // Steps config
- def buildSteps = [
- 'jar': [script: 'build-jars.sh', toCopy: null],
- 'artifacts': [script: 'build-artifacts.sh', toCopy: 'apache-cassandra-*.tar.gz,apache-cassandra-*.jar,apache-cassandra-*.pom'],
- 'lint': [script: 'check-code.sh', toCopy: null],
- 'debian': [script: 'build-debian.sh', toCopy: 'cassandra_*,cassandra-tools_*'],
- 'redhat': [script: 'build-redhat.sh rpm', toCopy: '*.rpm'],
- ]
- buildSteps.each() {
- it.value.put('type', 'build')
- it.value.put('size', 'small')
- it.value.put('splits', 1)
- }
-
- def testSteps = [
- // Each splits size need to be high enough to avoid the one hour per split timeout,
- // and low enough so test time is factors more than the setup+build time in each split.
- // Splits can also be poorly balanced: splitting or renaming test classes is the best tactic.
- // On unsaturated systems 10 minutes per split is optimal, higher with saturation
- // (some buffer on the heaviest split under the 1h max is required, ref `timeout(…)` in `test(…)`)
- 'cqlsh-test': [splits: 1],
- 'fqltool-test': [splits: 1, size: 'small'],
- 'test-cdc': [splits: 8],
- 'test': [splits: 16],
- 'test-latest': [splits: 16],
- 'test-compression': [splits: 16],
- 'stress-test': [splits: 1, size: 'small'],
- 'test-burn': [splits: 2],
- 'long-test': [splits: 4],
- 'test-oa': [splits: 16],
- 'test-system-keyspace-directory': [splits: 16],
- 'jvm-dtest': [splits: 12],
- 'jvm-dtest-upgrade': [splits: 6],
- 'simulator-dtest': [splits: 1, size: 'large'],
- 'dtest': [splits: 64, size: 'large'],
- 'dtest-novnode': [splits: 64, size: 'large'],
- 'dtest-latest': [splits: 64, size: 'large'],
- 'dtest-large': [splits: 6, size: 'large'],
- 'dtest-large-novnode': [splits: 6, size: 'large'],
- 'dtest-large-latest': [splits: 6, size: 'large'],
- 'dtest-upgrade': [splits: 128, size: 'large'],
- 'dtest-upgrade-novnode': [splits: 128, size: 'large'],
- 'dtest-upgrade-large': [splits: 32, size: 'large'],
- 'dtest-upgrade-large-novnode': [splits: 32, size: 'large'],
- 'microbench-test': [splits: 1, size: 'large'],
- // performance tests need 'cassandra-*large-dedicated' nodes
- 'microbench': [splits: 1, size: 'large', timeout_hours: 6, benchmark: true],
- ]
- testSteps.each() {
- it.value.put('type', 'test')
- if (!it.value['size']) {
- it.value.put('size', 'medium')
- }
- if (!it.value['timeout_hours']) {
- // default 1 hour
- it.value.put('timeout_hours', 1)
- }
- if (it.key.startsWith('dtest')) {
- it.value.put('python-dtest', true)
- }
- }
-
- def stepsMap = buildSteps + testSteps
-
- // find the default JDK and the supported JDKs defined in the build.xml
- def build_xml = readFile(file: 'build.xml')
- def javaVersionDefaultMatch = (build_xml =~ /property\s*name="java\.default"\s*value="([^"]*)"/)
- assert javaVersionDefaultMatch, "java.default property not found in build.xml"
- def javaVersionDefault = javaVersionDefaultMatch[0][1]
- def javaVersionsSupportedMatch = (build_xml =~ /property\s*name="java\.supported"\s*value="([^"]*)"/)
- assert javaVersionsSupportedMatch, "java.supported property not found in build.xml"
- def javaVersionsSupported = javaVersionsSupportedMatch[0][1].split(',') as List
-
- // define matrix axes
- def Map matrix_axes = [
- arch: archsSupported,
- jdk: javaVersionsSupported,
- python: pythonsSupported,
- cython: ['yes', 'no'],
- step: stepsMap.keySet(),
- split: (1..testSteps.values().splits.max()).toList()
- ]
-
- def List _axes = getMatrixAxes(matrix_axes).findAll { axis ->
- (isArchEnabled(axis['arch'])) && // skip disabled archs
- (isJdkEnabled(axis['jdk'])) && // skip disabled jdks
- (isStageEnabled(axis['step'])) && // skip disabled steps
- !(axis['python'] != pythonDefault && 'cqlsh-test' != axis['step']) && // Use only python 3.8 for all tests but cqlsh-test
- !(axis['cython'] != 'no' && 'cqlsh-test' != axis['step']) && // cython only for cqlsh-test, disable for others
- !(axis['jdk'] != javaVersionDefault && ('cqlsh-test' == axis['step'] || 'simulator-dtest' == axis['step'] || axis['step'].contains('dtest-upgrade'))) && // run cqlsh-test, simulator-dtest, *dtest-upgrade only with jdk11
- // Disable splits for all but proper stages
- !(axis['split'] > 1 && !stepsMap.findAll { entry -> entry.value.splits >= axis['split'] }.keySet().contains(axis['step'])) &&
- // run only the build types on non-amd64
- !(axis['arch'] != 'amd64' && !stepsMap.findAll { entry -> 'build' == entry.value.type }.keySet().contains(axis['step']))
- }
-
- def Map tasks = [
- jars: [failFast: true],
- tests: [failFast: true]
- ]
-
- for (def axis in _axes) {
- def cell = axis
- def name = getStepName(cell, stepsMap[cell.step])
- tasks[cell.step == "jar" ? "jars" : "tests"][name] = { ->
- "${stepsMap[cell.step].type}"(stepsMap[cell.step], cell)
- }
- }
-
- return cachedTasks = tasks
-}
-
-@NonCPS
-def List getMatrixAxes(Map matrix_axes) {
- def List axes = []
- matrix_axes.each { axis, values ->
- List axisList = []
- values.each { value ->
- axisList << [(axis): value]
- }
- axes << axisList
+def getNodeLabel(command, cell) {
+ if (command.label) {
+ return command.label
}
- axes.combinations()*.sum()
-}
-
-def getStepName(cell, command) {
- def arch = "amd64" == cell.arch ? "" : " ${cell.arch}"
- def python = "cqlsh-test" != cell.step ? "" : " python${cell.python}"
- def cython = "no" == cell.cython ? "" : " cython"
- def split = command.splits > 1 ? " ${cell.split}/${command.splits}" : ""
- return "${cell.step}${arch} jdk${cell.jdk}${python}${cython}${split}"
-}
-
-def getJarTasks() {
- Map jars = tasks()['jars']
- assert jars.size() > 1, "Nothing to build. Check parameters: jdk ${params.jdk}, arch ${params.architecture}"
- return jars
-}
-
-def hasNonJarTasks() {
- return tasks()['tests'].size() > 1
-}
-
-/**
- * Is this a post-commit build (or a pre-commit build)
- **/
-def isPostCommit() {
- // any build of a branch found on github.com/apache/cassandra is considered a post-commit (post-merge) CI run
- return params.repository && params.repository.contains("apache/cassandra") // no params exist first build
-}
-
-/**
- * Are we running on ci-cassandra.apache.org ?
- **/
-def isCanonical() {
- return "${JENKINS_URL}".contains("ci-cassandra.apache.org")
-}
-
-def isStageEnabled(stage) {
- return "jar" == stage || pipelineProfiles()[params.profile]?.contains(stage) || ("custom" == params.profile && stage ==~ params.profile_custom_regexp)
+ return cell.arch == 'arm64' ? 'linux && arm64' : 'linux && amd64'
}
-def isArchEnabled(arch) {
- return params.architecture == arch || "all" == params.architecture
-}
-
-def isJdkEnabled(jdk) {
- return !params.jdk?.trim() || params.jdk.trim() == jdk
+def copyToNightlies(artifacts, target) {
+ sh """
+ mkdir -p /var/lib/jenkins/workspace/Nightlies/${JOB_NAME}/${BUILD_NUMBER}/${target}
+ cp -r ${artifacts} /var/lib/jenkins/workspace/Nightlies/${JOB_NAME}/${BUILD_NUMBER}/${target}
+ """
}
-/**
- * Renders build script into pipeline steps
- **/
def build(command, cell) {
- def build_script = ".build/docker/${command.script}"
- def maxAttempts = 2
- def attempt = 0
- def nodeExclusion = ""
- retry(maxAttempts) {
- attempt++
- node(getNodeLabel(command, cell) + nodeExclusion) {
- nodeExclusion = "&&!${NODE_NAME}"
- withEnv(cell.collect { k, v -> "${k}=${v}" }) {
- ws("workspace/${JOB_NAME}/${BUILD_NUMBER}/${cell.step}/${cell.arch}/jdk-${cell.jdk}") {
+ if (command.script) {
+ sh label: "RUNNING ${command.script}...", script: command.script
+ return
+ }
+ node(getNodeLabel(command, cell)) {
+ withEnv(cell.collect { k, v -> "${k}=${v}" }) {
+ ws("workspace/${JOB_NAME}/${BUILD_NUMBER}/${cell.step}/${cell.arch}/jdk-${cell.jdk}/python-${cell.python}") {
+ try {
fetchSource(cell.step, cell.arch, cell.jdk)
- sh """
- test -f .jenkins/Jenkinsfile || { echo "Invalid git fork/branch"; exit 1; }
- grep -q "Jenkins CI declaration" .jenkins/Jenkinsfile || { echo "Only Cassandra 5.0+ supported"; exit 1; }
- """
- fetchDockerImages("redhat" == cell.step ? ['almalinux-build'] : ['bullseye-build'])
- def cell_suffix = "_jdk${cell.jdk}_${cell.arch}"
- def logfile = "stage-logs/${JOB_NAME}_${BUILD_NUMBER}_${cell.step}${cell_suffix}_attempt${attempt}.log.xz"
- def script_vars = "#!/bin/bash \n set -o pipefail ; " // pipe to tee needs pipefail
+ fetchDockerImages(['ubuntu-test'])
+ def cell_suffix = "_jdk${cell.jdk}_python_${cell.python}_${cell.cython}_${cell.arch}"
+ def logfile = "stage-logs/${JOB_NAME}_${BUILD_NUMBER}_${cell.step}${cell_suffix}.log.xz"
+ def script_vars = "#!/bin/bash \n set -o pipefail ; "
+ script_vars = "${script_vars} python_version=\'${cell.python}\'"
script_vars = "${script_vars} m2_dir=\'${WORKSPACE}/build/m2\'"
- timeout(time: 1, unit: 'HOURS') {
- def status = sh label: "RUNNING ${cell.step}...", script: "${script_vars} ${build_script} ${cell.jdk} 2>&1 | tee >( xz -c > build/${logfile} )", returnStatus: true
- dir("build") {
- archiveArtifacts artifacts: "${logfile}", fingerprint: true
- copyToNightlies("${logfile}", "${cell.step}/jdk${cell.jdk}/${cell.arch}/")
- }
- if (0 != status) { error("Stage ${cell.step}${cell_suffix} failed with exit status ${status}") }
- if ("jar" == cell.step) {
- stash name: "${cell.arch}_${cell.jdk}"
+ script_vars = fetchDTestsSource(command, script_vars)
+ timeout(time: command.timeout_hours, unit: 'HOURS') {
+ try {
+ buildJVMDTestJars(cell, script_vars, logfile)
+ sh label: "RUNNING ${cell.step}...", script: "${script_vars} .build/docker/run-tests.sh -a ${cell.step} -j ${cell.jdk} 2>&1 | tee >( xz -c > build/${logfile} )"
+ } finally {
+ dir("build") {
+ archiveArtifacts artifacts: "${logfile}", fingerprint: true
+ stash name: "${cell.arch}_${cell.jdk}", includes: '**/*.tar.gz,**/*.deb,**/*.changes,**/apache-cassandra-*.jar,**/stage-logs/*.xz,**/test/**'
+ if (command.toCopy) {
+ copyToNightlies("${command.toCopy}", "${cell.step}/jdk${cell.jdk}/${cell.arch}/")
+ }
+ }
}
}
- dir("build") {
- copyToNightlies("${command.toCopy}", "${cell.step}/jdk${cell.jdk}/${cell.arch}/")
- }
+ } finally {
cleanAgent(cell.step)
}
}
@@ -389,63 +99,72 @@ def test(command, cell) {
nodeExclusion = "&&!${NODE_NAME}"
withEnv(cell.collect { k, v -> "${k}=${v}" }) {
ws("workspace/${JOB_NAME}/${BUILD_NUMBER}/${cell.step}/${cell.arch}/jdk-${cell.jdk}/python-${cell.python}") {
- fetchSource(cell.step, cell.arch, cell.jdk)
- fetchDockerImages(['ubuntu2004_test'])
- def cell_suffix = "_jdk${cell.jdk}_python_${cell.python}_${cell.cython}_${cell.arch}_${cell.split}_${splits}"
- def logfile = "stage-logs/${JOB_NAME}_${BUILD_NUMBER}_${cell.step}${cell_suffix}_attempt${attempt}.log.xz"
- def script_vars = "#!/bin/bash \n set -o pipefail ; " // pipe to tee needs pipefail
- script_vars = "${script_vars} python_version=\'${cell.python}\'"
- script_vars = "${script_vars} m2_dir=\'${WORKSPACE}/build/m2\'"
- if ("cqlsh-test" == cell.step) {
- script_vars = "${script_vars} cython=\'${cell.cython}\'"
- }
- script_vars = fetchDTestsSource(command, script_vars)
- timeout(time: command.timeout_hours, unit: 'HOURS') { // best throughput with each cell at ~10 minutes
- def timer = System.currentTimeMillis()
- try {
- buildJVMDTestJars(cell, script_vars, logfile)
- script_vars = "${script_vars} docker_timeout_hours=\"${command.timeout_hours}\""
- def status = sh label: "RUNNING TESTS ${cell.step}...", script: "${script_vars} .build/docker/run-tests.sh -a ${cell.step} -c '${cell.split}/${splits}' -j ${cell.jdk} 2>&1 | tee >( xz -c > build/${logfile} )", returnStatus: true
- dir("build") {
- archiveArtifacts artifacts: "${logfile}", fingerprint: true
- }
- if (0 != status) { error("Stage ${cell.step}${cell_suffix} failed with exit status ${status}") }
- } catch (exc) {
- if (exc.getClass().getName() == "org.jenkinsci.plugins.workflow.steps.FlowInterruptedException") {
- for (def causeOfInterruption in exc.getCauses()) {
- echo "CauseOfInterruption: ${causeOfInterruption.getClass().getName()} - ${causeOfInterruption.getShortDescription()}"
+ try {
+ fetchSource(cell.step, cell.arch, cell.jdk)
+ fetchDockerImages(['ubuntu-test'])
+ def cell_suffix = "_jdk${cell.jdk}_python_${cell.python}_${cell.cython}_${cell.arch}_${cell.split}_${splits}"
+ def logfile = "stage-logs/${JOB_NAME}_${BUILD_NUMBER}_${cell.step}${cell_suffix}_attempt${attempt}.log.xz"
+ def script_vars = "#!/bin/bash \n set -o pipefail ; " // pipe to tee needs pipefail
+ script_vars = "${script_vars} python_version=\'${cell.python}\'"
+ script_vars = "${script_vars} m2_dir=\'${WORKSPACE}/build/m2\'"
+ if ("cqlsh-test" == cell.step) {
+ script_vars = "${script_vars} cython=\'${cell.cython}\'"
+ }
+ script_vars = fetchDTestsSource(command, script_vars)
+ timeout(time: command.timeout_hours, unit: 'HOURS') { // best throughput with each cell at ~10 minutes
+ def timer = System.currentTimeMillis()
+ try {
+ buildJVMDTestJars(cell, script_vars, logfile)
+ script_vars = "${script_vars} docker_timeout_hours=\"${command.timeout_hours}\""
+ def status = sh label: "RUNNING TESTS ${cell.step}...", script: "${script_vars} .build/docker/run-tests.sh -a ${cell.step} -c '${cell.split}/${splits}' -j ${cell.jdk} 2>&1 | tee >( xz -c > build/${logfile} )", returnStatus: true
+ dir("build") {
+ archiveArtifacts artifacts: "${logfile}", fingerprint: true
}
+ if (0 != status) { error("Stage ${cell.step}${cell_suffix} failed with exit status ${status}") }
+ } catch (exc) {
+ if ("org.jenkinsci.plugins.workflow.steps.FlowInterruptedException" == exc.getClass().getName()) {
+ def descriptions = []
+ for (def cause in exc.getCauses()) {
+ echo "CauseOfInterruption: ${cause.getClass().getName()} - ${cause.getShortDescription()}"
+ if (cause.getClass().getName().contains('CauseOfInterruption$UserInterruption')) {
+ throw exc // user explicitly aborted — do not retry
+ }
+ descriptions.add(cause.getShortDescription())
+ }
+ error("Retryable interruption: ${descriptions.join(', ')}")
+ }
+ throw exc
+ } finally {
+ def duration = System.currentTimeMillis() - timer
+ def formattedTime = String.format("%tT.%tL", duration, duration)
+ echo "Time ${cell.step}${cell_suffix}: ${formattedTime}"
}
- throw exc
- } finally {
- def duration = System.currentTimeMillis() - timer
- def formattedTime = String.format("%tT.%tL", duration, duration)
- echo "Time ${cell.step}${cell_suffix}: ${formattedTime}"
}
- }
- dir("build") {
- sh """
- mkdir -p test/output/${cell.step}
- find test/output -type f -name "TEST*.xml" -execdir mkdir -p jdk_${cell.jdk}/${cell.arch} ';' -execdir mv {} jdk_${cell.jdk}/${cell.arch}/{} ';'
- find test/output -name cqlshlib.xml -execdir mv cqlshlib.xml ${cell.step}/cqlshlib${cell_suffix}.xml ';'
- find test/output -name nosetests.xml -execdir mv nosetests.xml ${cell.step}/nosetests${cell_suffix}.xml ';'
- """
- if (!cell.step.startsWith("microbench")) {
- junit testResults: "test/**/TEST-*.xml,test/**/cqlshlib*.xml,test/**/nosetests*.xml", testDataPublishers: [[$class: 'StabilityTestDataPublisher']]
+ dir("build") {
+ sh """
+ mkdir -p test/output/${cell.step}
+ find test/output -type f -name "TEST*.xml" -execdir mkdir -p jdk_${cell.jdk}/${cell.arch} ';' -execdir mv {} jdk_${cell.jdk}/${cell.arch}/{} ';'
+ find test/output -name cqlshlib.xml -execdir mv cqlshlib.xml ${cell.step}/cqlshlib${cell_suffix}.xml ';'
+ find test/output -name nosetests.xml -execdir mv nosetests.xml ${cell.step}/nosetests${cell_suffix}.xml ';'
+ """
+ if (!cell.step.startsWith("microbench")) {
+ junit testResults: "test/**/TEST-*.xml,test/**/cqlshlib*.xml,test/**/nosetests*.xml", testDataPublishers: [[$class: 'StabilityTestDataPublisher']]
+ }
+ // check if we had Linux OOM killer active within the test container which could kill forked JUnit JVM processes
+ sh """
+ echo "docker memory/oomkiller debug:"
+ cat /sys/fs/cgroup/docker/memory.events || true
+ """
+ sh """
+ find test/output -type f -name "*.xml" -print0 | xargs -0 -r -n1 -P"\$(nproc)" xz -f
+ echo "test result files compressed"; find test/output -type f -name "*.xml.xz" | wc -l
+ """
+ archiveArtifacts artifacts: "test/logs/**,test/**/TEST-*.xml.xz,test/**/cqlshlib*.xml.xz,test/**/nosetests*.xml.xz,test/**/jmh-result.json", fingerprint: true
+ copyToNightlies("${logfile},test/logs/**,test/**/jmh-result.json", "${cell.step}/${cell.arch}/jdk${cell.jdk}/python${cell.python}/cython_${cell.cython}/" + "split_${cell.split}_${splits}".replace("/", "_"))
}
- // check if we had Linux OOM killer active within the test container which could kill forked JUnit JVM processes
- sh """
- echo "docker memory/oomkiller debug:"
- cat /sys/fs/cgroup/docker/memory.events || true
- """
- sh """
- find test/output -type f -name "*.xml" -print0 | xargs -0 -r -n1 -P"\$(nproc)" xz -f
- echo "test result files compressed"; find test/output -type f -name "*.xml.xz" | wc -l
- """
- archiveArtifacts artifacts: "test/logs/**,test/**/TEST-*.xml.xz,test/**/cqlshlib*.xml.xz,test/**/nosetests*.xml.xz,test/jmh-result.json", fingerprint: true
- copyToNightlies("${logfile}, test/logs/**", "${cell.step}/${cell.arch}/jdk${cell.jdk}/python${cell.python}/cython_${cell.cython}/" + "split_${cell.split}_${splits}".replace("/", "_"))
+ } finally {
+ cleanAgent(cell.step)
}
- cleanAgent(cell.step)
}
}
}
@@ -486,180 +205,110 @@ def fetchDockerImages(dockerfiles) {
// prefetch, from apache jfrog, reduces risking dockerhub pull rate limits
// also prefetch alpine:latest as its used as a utility in the scripts
def dockerfilesVar = dockerfiles.join(' ')
- sh """#!/bin/bash
- for dockerfile in ${dockerfilesVar} ; do
- image_tag="\$(md5sum .build/docker/\${dockerfile}.docker | cut -d' ' -f1)"
- image_name="apache/cassandra-\${dockerfile}:\${image_tag}"
- if ! ( [[ "" != "\$(docker images -q \${image_name} 2>/dev/null)" ]] ) ; then
- docker pull -q apache.jfrog.io/cassan-docker/\${image_name} &
- fi
- done
- docker pull -q apache.jfrog.io/cassan-docker/alpine:3.19.1 &
- wait
- """
+ sh ".build/docker/_docker_pull_base.sh ${dockerfilesVar}"
}
-def getNodeLabel(command, cell) {
- def label = "cassandra-${cell.arch}-${command.size}"
- if (command.containsKey('benchmark') && command.benchmark) {
- // to provide reliable results the "microbench" target
- // expects to be running on baremetal jenkins agents configured with only one executor
- // those jenkins agents need to be manually configured to have the "cassandra-amd64-large-dedicated" label
- label = "${label}-dedicated"
+def cleanAgent(stage) {
+ if ("jar" == stage) {
+ deleteDir()
}
- echo "using node label: ${label}"
- return label
+ sh '''
+ docker ps -aq --no-trunc \
+ | xargs -r docker rm -f -v
+ docker volume ls -q \
+ | xargs -r docker volume rm -f
+ docker system prune --volumes -af || true
+ '''
}
-def copyToNightlies(sourceFiles, remoteDirectory='') {
- if (isCanonical() && sourceFiles?.trim()) {
- def remotePath = remoteDirectory.startsWith("cassandra/") ? "${remoteDirectory}" : "cassandra/${JOB_NAME}/${BUILD_NUMBER}/${remoteDirectory}"
- def attempt = 1
- retry(9) {
- if (attempt > 1) { sleep(60 * attempt) }
- sshPublisher(
- continueOnError: true, failOnError: false,
- publishers: [
- sshPublisherDesc(
- configName: "Nightlies",
- transfers: [ sshTransfer( sourceFiles: sourceFiles, remoteDirectory: remotePath) ]
- )
- ])
+def runCommands(commandType) {
+ def config = getConfig()
+ def commands = config.steps.collectEntries { [(it): [:]] }
+ commands.putAll([
+ 'jar': [target: this.&build, timeout_hours: 1, toCopy: 'apache-cassandra-*.jar,apache-cassandra-*.tar.gz,apache-cassandra-*.deb,apache-cassandra-*.changes'],
+ 'microbench': [target: this.&test, timeout_hours: 2],
+ 'cqlsh-test': [target: this.&test, timeout_hours: 1],
+ 'jvm-dtest': [target: this.&test, timeout_hours: 1, 'python-dtest': true],
+ 'jvm-dtest-upgrade': [target: this.&test, timeout_hours: 2, 'python-dtest': true],
+ 'jvm-dtest-upgrade-40': [target: this.&test, timeout_hours: 2, 'python-dtest': true],
+ 'jvm-dtest-upgrade-41': [target: this.&test, timeout_hours: 2, 'python-dtest': true],
+ 'jvm-upgrade-dtest': [target: this.&test, timeout_hours: 2, 'python-dtest': true],
+ 'jvm-upgrade-dtest-vnode': [target: this.&test, timeout_hours: 2, 'python-dtest': true],
+ 'jvm-upgrade-dtest-no-vnode': [target: this.&test, timeout_hours: 2, 'python-dtest': true],
+ 'jvm-upgrade-dtest-large': [target: this.&test, timeout_hours: 2, 'python-dtest': true],
+ 'jvm-upgrade-dtest-40': [target: this.&test, timeout_hours: 2, 'python-dtest': true],
+ 'jvm-upgrade-dtest-41': [target: this.&test, timeout_hours: 2, 'python-dtest': true],
+ 'jvm-upgrade-dtest-ssl': [target: this.&test, timeout_hours: 2, 'python-dtest': true],
+ 'jvm-upgrade-dtest-no-preview': [target: this.&test, timeout_hours: 2, 'python-dtest': true],
+ 'jvm-upgrade-dtest-storage': [target: this.&test, timeout_hours: 2, 'python-dtest': true],
+ 'jvm-upgrade-dtest-latest-killer': [target: this.&test, timeout_hours: 2, 'python-dtest': true],
+ 'jvm-upgrade-dtest-tls': [target: this.&test, timeout_hours: 2, 'python-dtest': true],
+ 'jvm-upgrade-dtest-auth': [target: this.&test, timeout_hours: 2, 'python-dtest': true],
+ 'jvm-dtest-latest': [target: this.&test, timeout_hours: 2, 'python-dtest': true],
+ 'jvm-dtest-latest-vnode': [target: this.&test, timeout_hours: 2, 'python-dtest': true],
+ 'jvm-dtest-latest-cdc': [target: this.&test, timeout_hours: 2, 'python-dtest': true],
+ 'jvm-dtest-latest-auth': [target: this.&test, timeout_hours: 2, 'python-dtest': true],
+ 'jvm-dtest-latest-compression': [target: this.&test, timeout_hours: 2, 'python-dtest': true],
+ 'jvm-dtest-latest-legacy-sstable': [target: this.&test, timeout_hours: 2, 'python-dtest': true],
+ 'jvm-dtest-latest-oa': [target: this.&test, timeout_hours: 2, 'python-dtest': true],
+ 'jvm-dtest-latest-large': [target: this.&test, timeout_hours: 2, 'python-dtest': true],
+ 'jvm-dtest-latest-no-vnode': [target: this.&test, timeout_hours: 2, 'python-dtest': true],
+ 'jvm-dtest-latest-sai': [target: this.&test, timeout_hours: 2, 'python-dtest': true],
+ 'jvm-dtest-latest-ssl': [target: this.&test, timeout_hours: 2, 'python-dtest': true],
+ 'jvm-dtest-latest-tls': [target: this.&test, timeout_hours: 2, 'python-dtest': true],
+ 'jvm-dtest-latest-upgrade': [target: this.&test, timeout_hours: 2, 'python-dtest': true],
+ 'jvm-dtest-latest-vnode-upgrade': [target: this.&test, timeout_hours: 2, 'python-dtest': true],
+ 'jvm-dtest-latest-trigger': [target: this.&test, timeout_hours: 2, 'python-dtest': true],
+ 'jvm-dtest-latest-materialized-view': [target: this.&test, timeout_hours: 2, 'python-dtest': true],
+ 'jvm-dtest-latest-transient-replication': [target: this.&test, timeout_hours: 2, 'python-dtest': true],
+ 'jvm-dtest-latest-counters': [target: this.&test, timeout_hours: 2, 'python-dtest': true],
+ 'jvm-dtest-latest-repair': [target: this.&test, timeout_hours: 2, 'python-dtest': true],
+ 'jvm-dtest-latest-secondary-index': [target: this.&test, timeout_hours: 2, 'python-dtest': true],
+ 'jvm-dtest-latest-paging': [target: this.&test, timeout_hours: 2, 'python-dtest': true],
+ 'jvm-dtest-latest-topology': [target: this.&test, timeout_hours: 2, 'python-dtest': true],
+ 'jvm-dtest-latest-system-keyspace-directory': [target: this.&test, timeout_hours: 2, 'python-dtest': true],
+ 'jvm-dtest-latest-stress': [target: this.&test, timeout_hours: 2, 'python-dtest': true],
+ 'jvm-dtest-latest-fqltool': [target: this.&test, timeout_hours: 2, 'python-dtest': true],
+ 'jvm-dtest-latest-reverse-query': [target: this.&test, timeout_hours: 2, 'python-dtest': true],
+ 'jvm-dtest-latest-cql': [target: this.&test, timeout_hours: 2, 'python-dtest': true],
+ 'jvm-dtest-latest-jmx': [target: this.&test, timeout_hours: 2, 'python-dtest': true],
+ 'jvm-dtest-latest-sql': [target: this.&test, timeout_hours: 2, 'python-dtest': true],
+ 'jvm-dtest-latest-bti': [target: this.&test, timeout_hours: 2, 'python-dtest': true],
+ 'jvm-dtest-latest-accord': [target: this.&test, timeout_hours: 2, 'python-dtest': true],
+ ])
+
+ def parallelMap = [:]
+ config.cells.findAll { it.step in commands.keySet() }.each { cell ->
+ def command = commands[cell.step]
+ if (commandType == 'build' && command.target != this.&build) {
+ return
}
- echo "archived to https://nightlies.apache.org/${remotePath}"
- }
-}
-
-def cleanAgent(job_name) {
- // get any public IP which is more helpful correlating back to the cloud instance
- sh script: 'hostname; curl -sm 10 ifconfig.me', returnStatus: true
- if (isCanonical()) {
- def agentScriptsUrl = "https://raw.githubusercontent.com/apache/cassandra-builds/trunk/jenkins-dsl/agent_scripts/"
- cleanAgentDocker(job_name, agentScriptsUrl)
- logAgentInfo(job_name, agentScriptsUrl)
- }
- cleanWs()
-}
-
-def cleanAgentDocker(job_name, agentScriptsUrl) {
- // we don't expect any build to have been running for longer than maxBuildHours
- def maxBuildHours = 12
- echo "Pruning docker for '${job_name}' on ${NODE_NAME}…" ;
- sh """#!/bin/bash
- set +e
- wget -q ${agentScriptsUrl}/docker_image_pruner.py
- wget -q ${agentScriptsUrl}/docker_agent_cleaner.sh
- bash docker_agent_cleaner.sh ${maxBuildHours}
- """
-}
-
-def logAgentInfo(job_name, agentScriptsUrl) {
- sh """#!/bin/bash
- set +e -o pipefail
- wget -q ${agentScriptsUrl}/agent_report.sh
- bash -x agent_report.sh | tee -a \$(date +"%Y%m%d%H%M")-disk-usage-stats.txt
- """
- copyToNightlies("*-disk-usage-stats.txt", "cassandra/ci-cassandra.apache.org/agents/${NODE_NAME}/disk-usage/")
-}
-
-/////////////////////////////////////////
-////// scripting support for summary ////
-/////////////////////////////////////////
-
-def generateTestReports() {
- node("cassandra-medium") {
- cleanAgent("generateTestReports")
- checkout changelog: false, scm: scmGit(branches: [[name: params.branch]], extensions: [cloneOption(depth: 1, noTags: true, reference: '', shallow: true)], userRemoteConfigs: [[url: params.repository]])
- def logfile = "stage-logs/${JOB_NAME}_${BUILD_NUMBER}_generateTestReports.log.xz"
- sh "mkdir -p build/stage-logs"
- def teeSuffix = "2>&1 | tee >( xz -c > build/${logfile} )"
- def script_vars = "#!/bin/bash -x \n "
- if (isCanonical()) {
- // copyArtifacts takes >4hrs, hack with manual download
- sh """${script_vars}
- ( mkdir -p build/test
- wget -q ${BUILD_URL}/artifact/test/output/*zip*/output.zip
- unzip -x -d build/test -q output.zip ) ${teeSuffix}
- """
- } else {
- copyArtifacts filter: 'test/**/TEST-*.xml.xz,test/**/cqlshlib*.xml.xz,test/**/nosetests*.xml.xz,test/jmh-result.json', fingerprintArtifacts: true, projectName: env.JOB_NAME, selector: specific(env.BUILD_NUMBER), target: "build/", optional: true
+ if (commandType == 'test' && command.target != this.&test) {
+ return
}
- if (fileExists('build/test/output')) {
- // merge splits for each target's test report, other axes are kept separate
- // TODO parallelised for loop
- // TODO results_details.tar.xz needs to include all logs for failed tests
- sh """${script_vars} (
- echo "test result files to decompress"; find build/test/output -type f -name "*.xml.xz" | wc -l
- find build/test/output -type f -name "*.xml.xz" -print0 | xargs -0 -r -n1 -P"\$(nproc)" xz -f --decompress
-
- for target in \$(ls build/test/output/) ; do
- if test -d build/test/output/\${target} ; then
- mkdir -p build/test/reports/\${target}
- echo "Report for \${target} (\$(find build/test/output/\${target} -name '*.xml' | wc -l) test files)"
- CASSANDRA_DOCKER_ANT_OPTS="-Dbuild.test.output.dir=build/test/output/\${target} -Dbuild.test.report.dir=build/test/reports/\${target}"
- export CASSANDRA_DOCKER_ANT_OPTS
- .build/docker/_docker_run.sh bullseye-build.docker ci/generate-test-report.sh
- fi
- done
-
- .build/docker/_docker_run.sh bullseye-build.docker ci/generate-ci-summary.sh || echo "failed generate-ci-summary.sh"
-
- tar -cf build/results_details.tar -C build/test/ reports
- xz -8f build/results_details.tar ) ${teeSuffix}
- """
-
- dir('build/') {
- archiveArtifacts artifacts: "ci_summary.html,results_details.tar.xz,${logfile}", fingerprint: true
- copyToNightlies('ci_summary.html,results_details.tar.xz,${logfile},test/jmh-result.json')
- }
- }
- if (fileExists('build/test/jmh-result.json')) {
- jmhReport('build/test/jmh-result.json')
+ parallelMap["${cell.step}_${cell.arch}_jdk${cell.jdk}_py${cell.python}_${cell.cython}${cell.split ? "_split${cell.split}" : ''}"] = {
+ command.target(command, cell)
}
}
+ parallel parallelMap
}
-def sendNotifications() {
- if (isPostCommit() && isCanonical()) {
- // the following is expected only to work on ci-cassandra.apache.org
- def changes = '?'
- try {
- script {
- changes = formatChangeLogChanges(currentBuild.changeSets)
- echo "changes: ${changes}"
+pipeline {
+ agent none
+ stages {
+ stage('Build') {
+ steps {
+ script {
+ runCommands('build')
+ }
}
- slackSend channel: '#cassandra-builds', message: ":apache: <${BUILD_URL}|${currentBuild.fullDisplayName}> completed: ${currentBuild.result}. \n${changes}"
- emailext to: 'builds@cassandra.apache.org', subject: "Build complete: ${currentBuild.fullDisplayName} [${currentBuild.result}] ${GIT_COMMIT}", presendScript: 'msg.removeHeader("In-Reply-To"); msg.removeHeader("References")', body: emailContent()
- } catch (Exception ex) {
- echo 'failed to send notifications ' + ex.toString()
}
- }
-}
-
-def formatChangeLogChanges(changeLogSets) {
- def result = ''
- for (int i = 0; i < changeLogSets.size(); i++) {
- def entries = changeLogSets[i].items
- for (int j = 0; j < entries.length; j++) {
- def entry = entries[j]
- result = result + "${entry.commitId} by ${entry.author} on ${new Date(entry.timestamp)}: ${entry.msg}\n"
+ stage('Test') {
+ steps {
+ script {
+ runCommands('test')
+ }
+ }
}
}
- return result
}
-
-def emailContent() {
- return '''
- -------------------------------------------------------------------------------
- Build ${ENV,var="JOB_NAME"} #${BUILD_NUMBER} ${BUILD_STATUS}
- URL: ${BUILD_URL}
- -------------------------------------------------------------------------------
- Changes:
- ${CHANGES}
- -------------------------------------------------------------------------------
- Failed Tests:
- ${FAILED_TESTS,maxTests=500,showMessage=false,showStack=false}
- -------------------------------------------------------------------------------
- For complete test report and logs see https://nightlies.apache.org/cassandra/${JOB_NAME}/${BUILD_NUMBER}/
- '''
-}
\ No newline at end of file
diff --git a/.jenkins/k8s/README.md b/.jenkins/k8s/README.md
index 671b61961a08..844130651331 100644
--- a/.jenkins/k8s/README.md
+++ b/.jenkins/k8s/README.md
@@ -24,7 +24,7 @@ ZONE="us-central1-c"
gcloud container clusters create ${CLUSTER_NAME} --machine-type e2-standard-8 --disk-type=pd-ssd --num-nodes 1 --node-labels=cassandra.jenkins.controller=true --autoscaling-profile optimize-utilization --zone ${ZONE}
# small resource nodes
-gcloud container node-pools create agents-small --cluster ${CLUSTER_NAME} --machine-type n2-highcpu-4 --disk-type=pd-ssd --enable-autoscaling --spot --num-nodes=0 --min-nodes=0 --max-nodes=50 --node-labels=cassandra.jenkins.agent=true,cassandra.jenkins.agent.small=true --zone ${ZONE}
+gcloud container node-pools create agents-small --cluster ${CLUSTER_NAME} --machine-type e2-highcpu-8 --disk-type=pd-ssd --enable-autoscaling --spot --num-nodes=0 --min-nodes=0 --max-nodes=50 --node-labels=cassandra.jenkins.agent=true,cassandra.jenkins.agent.small=true --zone ${ZONE}
# medium resource nodes
# preference (by cost): n2-highcpu-8, c3-highcpu-8, n4-highcpu-8, n1-highcpu-16
@@ -32,6 +32,12 @@ gcloud container node-pools create agents-medium --cluster ${CLUSTER_NAME} --mac
# large resource nodes
gcloud container node-pools create agents-large --cluster ${CLUSTER_NAME} --machine-type n2-standard-8 --disk-type=pd-ssd --enable-autoscaling --spot --num-nodes=0 --min-nodes=0 --max-nodes=160 --node-labels=cassandra.jenkins.agent=true,cassandra.jenkins.agent.large=true --zone ${ZONE}
+
+# For each sized resource nodes, pick any machine type that fits, those listed above should work and be the most cost-effective, but this can change region to region
+# See https://github.com/apache/cassandra/blob/cassandra-6.0/.jenkins/Jenkinsfile#L35-L38
+# and agent.podTemplates.*.resourceLimitCpu and agent.podTemplates.*.resourceLimitMemory (adding gke/eks requirements) in https://github.com/apache/cassandra/blob/cassandra-6.0/.jenkins/k8s/jenkins-deployment.yaml
+# The jenkins resource requirements should fit into the corresponding dind podTemplate limits.
+# Remember to allow a buffer for gke/eks pods deployed on each node.
```
diff --git a/.jenkins/k8s/jenkins-deployment.yaml b/.jenkins/k8s/jenkins-deployment.yaml
index 46cc77fc3c17..98290e601158 100644
--- a/.jenkins/k8s/jenkins-deployment.yaml
+++ b/.jenkins/k8s/jenkins-deployment.yaml
@@ -36,6 +36,7 @@ controller:
customJenkinsLabels:
- controller
resources:
+ # increase cpu/memory as agent pool sizes get bigger (pre-ci.c.a.o uses 8 and 20g)
requests:
cpu: 4
memory: 16G
@@ -97,6 +98,23 @@ controller:
}
}
}
+ - script: >
+ pipelineJob('cassandra-6.0') {
+ definition {
+ cpsScm {
+ scm {
+ git {
+ remote {
+ url('https://github.com/apache/cassandra')
+ }
+ branch('cassandra-6.0')
+ scriptPath('.jenkins/Jenkinsfile')
+ }
+ }
+ lightweight()
+ }
+ }
+ }
- script: >
pipelineJob('cassandra-5.0') {
definition {
@@ -216,6 +234,19 @@ agent:
- emptyDirVolume:
memory: 'false'
mountPath: /certs
+ # limit one agent pod per node for simpler operations (like orphan cleanup)
+ yaml: |
+ spec:
+ affinity:
+ podAntiAffinity:
+ requiredDuringSchedulingIgnoredDuringExecution:
+ - labelSelector:
+ matchExpressions:
+ - key: jenkins/cassius-jenkins-agent
+ operator: In
+ values:
+ - "true"
+ topologyKey: kubernetes.io/hostname
agent-dind-medium: |
- name: agent-dind-medium
label: agent-dind cassandra-medium cassandra-amd64-medium
@@ -293,6 +324,19 @@ agent:
- emptyDirVolume:
memory: 'false'
mountPath: /certs
+ # limit one agent pod per node for simpler operations (like orphan cleanup)
+ yaml: |
+ spec:
+ affinity:
+ podAntiAffinity:
+ requiredDuringSchedulingIgnoredDuringExecution:
+ - labelSelector:
+ matchExpressions:
+ - key: jenkins/cassius-jenkins-agent
+ operator: In
+ values:
+ - "true"
+ topologyKey: kubernetes.io/hostname
agent-dind-large: |
- name: agent-dind-large
label: agent-dind cassandra-large cassandra-amd64-large cassandra-amd64-large-dedicated
@@ -370,5 +414,18 @@ agent:
- emptyDirVolume:
memory: 'false'
mountPath: /certs
+ # limit one agent pod per node for simpler operations (like orphan cleanup)
+ yaml: |
+ spec:
+ affinity:
+ podAntiAffinity:
+ requiredDuringSchedulingIgnoredDuringExecution:
+ - labelSelector:
+ matchExpressions:
+ - key: jenkins/cassius-jenkins-agent
+ operator: In
+ values:
+ - "true"
+ topologyKey: kubernetes.io/hostname
diff --git a/CHANGES.txt b/CHANGES.txt
index ab0e548b0d7e..be2d7094a430 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,49 @@
+5.0.9
+ * Coordinator load-shedding returns OverloadedException without setting streamId, misrouting query responses (CASSANDRA-21508)
+ * SAI Component Checksum Validation Should be Segment-Aware (CASSANDRA-21516)
+ * Support Python 3.12 and 3.13 in cqlsh (CASSANDRA-20997)
+ * Fix AssertionError in hasReplicaWithOngoingRepair when parallel_repair_count > 1 (CASSANDRA-21426)
+ * putShortVolatile is not volatile in InMemoryTrie (CASSANDRA-21353)
+ * Fix RequestFailureReason serializer and nits in a few others (CASSANDRA-21437)
+ * Remove golang dependency in gen-doc and replace with python implementation (CASSANDRA-21432)
+ * Use estimated compressed size for tables to check if there is enough free space for a compaction (CASSANDRA-21245)
+ * Fix failing select on system_views.settings for non-string keys (CASSANDRA-21348)
+ * Ensure SAI sends range tombstones to the coordinator for queries on static columns (CASSANDRA-21332)
+Merged from 4.1:
+ * Add Paxos v2 option and informatin in cassandra.yaml (CASSANDRA-21316)
+Merged from 4.0:
+ * Bound declared value length against readable bytes in CBUtil (CASSANDRA-21521)
+ * Verify extension type before initializing reflectively-loaded classes (CASSANDRA-21525)
+ * Rename conflicting nodetool import --copy-data short option from -p to -cd (CASSANDRA-20214)
+ * Fix PasswordObfuscator failing to obfuscate certain passwords (CASSANDRA-21113)
+ * Fix negative memtable allocator ownership when an update is shadowed by an existing row deletion (CASSANDRA-21469)
+ * Consider first token of SSTable when calculating SSTable intersection in LeveledScanner (CASSANDRA-21369)
+ * Remove inFlightEcho entry on ECHO_REQ failure (CASSANDRA-21428)
+ * Validate snapshot names (CASSANDRA-21389)
+ * BTree.FastBuilder.reset() fails to clear savedBuffer and savedNextKey, causing ClassCastException and SSTable header corruption during schema disagreement (CASSANDRA-21216, CASSANDRA-21260)
+ * Backport CASSANDRA-17810 fix and improve RTBoundValidator error messages (CASSANDRA-18282)
+
+
+5.0.8
+ * Backport Automated Repair Inside Cassandra for CEP-37 (CASSANDRA-21138)
+ * Update cassandra-stress to support TLS 1.3 by default by auto-negotiation (CASSANDRA-21007)
+ * Ensure schema created before 2.1 without tableId in folder name can be loaded in SnapshotLoader (CASSANDRA-21173)
+Merged from 4.1:
+ * Harden data resurrection startup check with atomic heartbeat file write with fallback (CASSANDRA-21290)
+Merged from 4.0:
+Backported from 6.0:
+ * Improved observability in AutoRepair to report both expected vs. actual repair bytes and expected vs. actual keyspaces (CASSANDRA-20581)
+ * Stop repair scheduler if two major versions are detected (CASSANDRA-20048)
+ * AutoRepair: Safeguard Full repair against disk protection (CASSANDRA-20045)
+ * Stop AutoRepair monitoring thread upon Cassandra shutdown (CASSANDRA-20623)
+ * Fix race condition in auto-repair scheduler (CASSANDRA-20265)
+ * Implement minimum repair task duration setting for auto-repair scheduler (CASSANDRA-20160)
+ * Implement preview_repaired auto-repair type (CASSANDRA-20046)
+ * Automated Repair Inside Cassandra for CEP-37 (CASSANDRA-19918)
+
+
5.0.7
+ * Clear BTree.FastBuilder saved overflow state on reset to prevent cross-table column contamination after schema disagreement (CASSANDRA-21216, CASSANDRA-21260)
* Refactor SAI ANN query execution to use score ordered iterators for correctness and speed (CASSANDRA-20086)
* Disallow binding an identity to a superuser when the user is a regular user (CASSANDRA-21219)
* Fix ConcurrentModificationException in compaction garbagecollect (CASSANDRA-21065)
@@ -95,6 +140,10 @@ Merged from 4.0:
* Fix Dropwizard Meter causes timeouts when infrequently used (CASSANDRA-19332)
+Merged from 5.1:
+ * Expose current compaction throughput in nodetool (CASSANDRA-13890)
+
+
5.0.4
* Update netty to 4.1.119.Final and netty-tcnative to 2.0.70.Final (CASSANDRA-20314)
* Serialization can lose complex deletions in a mutation with multiple collections in a row (CASSANDRA-20449)
@@ -349,13 +398,11 @@ Merged from 3.0:
5.0-alpha2
- * Add support for vector search in SAI (CASSANDRA-18715)
* Remove crc_check_chance from CompressionParams (CASSANDRA-18872)
* Fix schema loading of UDTs inside vectors inside UDTs (CASSANDRA-18964)
* Add cqlsh autocompletion for the vector data type (CASSANDRA-18946)
* Fix nodetool tablehistograms output to avoid printing repeated information and ensure at most two arguments (CASSANDRA-18955)
* Change the checksum algorithm SAI-related files use from CRC32 to CRC32C (CASSANDRA-18836)
- * Correctly remove Index.Group from IndexRegistry (CASSANDRA-18905)
* Fix vector type to support DDM's mask_default function (CASSANDRA-18889)
* Remove unnecessary reporter-config3 dependency (CASSANDRA-18907)
* Remove support for empty values on the vector data type (CASSANDRA-18876)
@@ -668,6 +715,8 @@ Merged from 3.0:
* Do not remove SSTables when cause of FSReadError is OutOfMemoryError while using best_effort disk failure policy (CASSANDRA-18336)
* Do not remove truncated_at entry in system.local while dropping an index (CASSANDRA-18105)
+4.0.14
+ * Fix memory leak in BTree.FastBuilder (CASSANDRA-19785)
4.0.9
* Update zstd-jni library to version 1.5.5 (CASSANDRA-18429)
diff --git a/CONTRIBUTING_CC.md b/CONTRIBUTING_CC.md
new file mode 100644
index 000000000000..482d97360c32
--- /dev/null
+++ b/CONTRIBUTING_CC.md
@@ -0,0 +1,26 @@
+/*
+ * Copyright DataStax, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+# Merging
+
+Feel free to make your contributions: Patches, PRs, diffs,... all welcomed.
+- Create a PR against main.
+- Ensure all your commits are squashed into 1 or under a sensible grouping. Rebase to account for any changes.
+- Follow the checklist in the PR and DoD (Definition of Done). This needs to be reviewed and approved before merging.
+- Next you have 2 options: you merge to main and it will eventually be cherrypicked into main-5.0 when it catches up.
+- Or you create a PR for main-5.0 and get approval likewise.
+- When merging your PRs: Squash and merge or Rebase and merge can be used.
+- IMPORTANT: Do not forward merge main into main-5.0.
diff --git a/NEWS.txt b/NEWS.txt
index 1192f4178a5d..a63e0e36d102 100644
--- a/NEWS.txt
+++ b/NEWS.txt
@@ -65,6 +65,45 @@ restore snapshots created with the previous major version using the
'sstableloader' tool. You can upgrade the file format of your snapshots
using the provided 'sstableupgrade' tool.
+5.0.8
+======
+
+New features
+------------
+ - CEP-37 Auto Repair is a fully automated scheduler that provides repair orchestration within Apache Cassandra. This
+ significantly reduces operational overhead by eliminating the need for operators to deploy external tools to submit
+ and manage repairs. See
+ https://cwiki.apache.org/confluence/display/CASSANDRA/CEP-37+Apache+Cassandra+Unified+Repair+Solution for more
+ details on the motivation and design.
+
+Upgrading
+---------
+ - The auto-repair feature requires enabling the JVM property `cassandra.autorepair.enable=true` (add
+ `-Dcassandra.autorepair.enable=true` to JVM options) before starting the node. This property creates the required
+ schema elements for auto-repair, including the auto_repair column in system_schema.tables and system_schema.views,
+ as well as the auto_repair_history and auto_repair_priority tables in system_distributed. After enabling this
+ property, you still need to enable auto-repair scheduling either in cassandra.yaml under the `auto_repair` section
+ or at runtime via JMX.
+
+ Users who do not intend to use auto-repair can leave this property disabled (the default) to maintain schema
+ compatibility with pre-5.0.8 nodes during rolling upgrades. This property must be set consistently across all
+ nodes before startup and cannot be changed at runtime.
+
+ WARNING: This property is non-reversible. Once enabled, it cannot be disabled. Attempting to start a node
+ with `cassandra.autorepair.enable=false` after it was previously enabled will cause the node to fail during
+ initialization due to schema incompatibility (the persisted schema contains auto-repair columns that are not
+ recognized when the property is disabled). To disable auto-repair scheduling after the property has been
+ enabled, use cassandra.yaml or JMX instead of changing the JVM property.
+
+ IMPORTANT: The `cassandra.autorepair.enable` property must be enabled consistently across all nodes in the
+ cluster before any schema changes are made. When some nodes have the property enabled and others do not, the
+ system_distributed keyspace schema generation will differ between nodes (generation 7 with auto-repair vs
+ generation 6 without), causing schema disagreement. This is similar to what happens during a major version
+ upgrade when new system tables are added. Any schema change (e.g. CREATE KEYSPACE) attempted while nodes
+ are in this inconsistent state will time out and schema versions will not converge until all nodes are
+ brought up with the same setting. Once all nodes have the property set consistently, schema will converge
+ automatically.
+
5.0.7
======
@@ -136,7 +175,6 @@ New features
src/java/org/apache/cassandra/db/compaction/UnifiedCompactionStrategy.md
- New `VectorType` (cql `vector`) which adds new fixed-length element arrays. See CASSANDRA-18504
- Added new vector similarity functions `similarity_cosine`, `similarity_euclidean` and `similarity_dot_product`.
- - Added ANN vector similarity search via ORDER BY ANN OF syntax on SAI indexes (using jvector library).
- Removed UDT type migration logic for 3.6+ clusters upgrading to 4.0. If migration has been disabled, it must be
enabled before upgrading to 5.0 if the cluster used UDTs. See CASSANDRA-18504
- Entended max expiration time from 2038-01-19T03:14:06+00:00 to 2106-02-07T06:28:13+00:00
@@ -321,6 +359,11 @@ Deprecation
Cluster hosts running with dual native ports were not correctly identified in the system.peers tables and server-sent EVENTs,
causing clients that encrypt traffic to fail to maintain correct connection pools. For more information, see CASSANDRA-19392.
- Deprecated `use_deterministic_table_id` in cassandra.yaml. Table IDs may still be supplied explicitly on CREATE.
+ - Chronicle Queue has changed the enums used for log rolling (cassandra.yaml -> full_query_logging_options:roll_cycle).
+ Older legacy options will still work for the foreseeable future but you will see warnings in logs and future dependency
+ upgrades may break your log rolling param. The default log rolling param will be changed with the next major release
+ from HOURLY to FAST_HOURLY, primarily different on how frequently indexes are built (256 in FAST_HOURLY vs. 16 in HOURLY).
+ For more info refer to: net.openhft.chronicle.queue.RollCycles
4.1
===
diff --git a/NOTICE.txt b/NOTICE.txt
index fd185210450f..5a2c26ae740d 100644
--- a/NOTICE.txt
+++ b/NOTICE.txt
@@ -8,3 +8,9 @@ Android Code
Copyright 2005-2008 The Android Open Source Project
This product includes software developed as part of
The Android Open Source Project (http://source.android.com).
+
+This project includes software from the Apache Lucene project. Relevant
+portions of its NOTICE are excerpted below:
+=======================================================================
+Apache Lucene
+Copyright 2001-2018 The Apache Software Foundation
diff --git a/README.asc b/README.asc
index 5c6713580d30..684c8f6b4cdc 100644
--- a/README.asc
+++ b/README.asc
@@ -25,8 +25,8 @@ and running, and demonstrate some simple reads and writes. For a more-complete g
First, we'll unpack our archive:
- $ tar -zxvf apache-cassandra-$VERSION.tar.gz
- $ cd apache-cassandra-$VERSION
+ $ tar -zxvf dse-db-$VERSION.tar.gz
+ $ cd dse-db-$VERSION
After that we start the server. Running the startup script with the -f argument will cause
Cassandra to remain in the foreground and log to standard out; it can be stopped with ctrl-C.
diff --git a/bin/cassandra.in.sh b/bin/cassandra.in.sh
index b838c2d4cf9c..5d83b4ed673e 100644
--- a/bin/cassandra.in.sh
+++ b/bin/cassandra.in.sh
@@ -30,7 +30,7 @@ CLASSPATH="$CASSANDRA_CONF"
# compiled classes. NOTE: This isn't needed by the startup script,
# it's just used here in constructing the classpath.
if [ -d $CASSANDRA_HOME/build ] ; then
- jars_cnt="`ls -1 $CASSANDRA_HOME/build/apache-cassandra*.jar | grep -v 'javadoc.jar' | grep -v 'sources.jar' | wc -l | xargs echo`"
+ jars_cnt="`ls -1 $CASSANDRA_HOME/build/dse-db*.jar | grep -v 'javadoc.jar' | grep -v 'sources.jar' | wc -l | xargs echo`"
if [ "$jars_cnt" -gt 1 ]; then
dir="`cd $CASSANDRA_HOME/build; pwd`"
echo "There are JAR artifacts for multiple versions in the $dir directory. Please clean the project with 'ant realclean' and build it again." 1>&2
@@ -38,8 +38,8 @@ if [ -d $CASSANDRA_HOME/build ] ; then
fi
if [ "$jars_cnt" = "1" ]; then
- cassandra_bin="`ls -1 $CASSANDRA_HOME/build/apache-cassandra*.jar | grep -v javadoc | grep -v sources`"
- CLASSPATH="$CLASSPATH:$cassandra_bin"
+ dse_db_bin="`ls -1 $CASSANDRA_HOME/build/dse-db*.jar | grep -v javadoc | grep -v sources`"
+ CLASSPATH="$CLASSPATH:$dse_db_bin"
fi
fi
@@ -122,11 +122,16 @@ jvmver=`echo "$java_ver_output" | grep '[openjdk|java] version' | awk -F'"' 'NR=
JVM_VERSION=${jvmver%_*}
short=$(echo "${jvmver}" | cut -c1-2)
-JAVA_VERSION=17
+JAVA_VERSION=22
if [ "$short" = "11" ] ; then
JAVA_VERSION=11
elif [ "$JVM_VERSION" \< "17" ] ; then
- echo "Cassandra 5.0 requires Java 11 or Java 17."
+ echo "DSE DB 5.0 requires Java 11 or higher."
+ exit 1;
+elif [ "$short" = "17" ] ; then
+ JAVA_VERSION=17
+elif [ "$JVM_VERSION" \< "22" ] ; then
+ echo "DSE DB 5.0 requires Java 11 or higher."
exit 1;
fi
@@ -151,7 +156,9 @@ esac
# Read user-defined JVM options from jvm-server.options file
JVM_OPTS_FILE=$CASSANDRA_CONF/jvm${jvmoptions_variant:--clients}.options
-if [ $JAVA_VERSION -ge 17 ] ; then
+if [ $JAVA_VERSION -ge 22 ] ; then
+ JVM_DEP_OPTS_FILE=$CASSANDRA_CONF/jvm22${jvmoptions_variant:--clients}.options
+elif [ $JAVA_VERSION -ge 17 ] ; then
JVM_DEP_OPTS_FILE=$CASSANDRA_CONF/jvm17${jvmoptions_variant:--clients}.options
elif [ $JAVA_VERSION -ge 11 ] ; then
JVM_DEP_OPTS_FILE=$CASSANDRA_CONF/jvm11${jvmoptions_variant:--clients}.options
diff --git a/bin/cqlsh b/bin/cqlsh
index 2a9651968b4b..25fa029b4f57 100755
--- a/bin/cqlsh
+++ b/bin/cqlsh
@@ -63,7 +63,7 @@ is_supported_version() {
major_version="${version%.*}"
minor_version="${version#*.}"
# python 3.8-3.11 are supported
- if [ "$major_version" = 3 ] && [ "$minor_version" -ge 8 ] && [ "$minor_version" -le 11 ]; then
+ if [ "$major_version" = 3 ] && [ "$minor_version" -ge 8 ] && [ "$minor_version" -le 13 ]; then
echo "supported"
# python 3.6-3.7 are deprecated
elif [ "$major_version" = 3 ] && [ "$minor_version" -ge 6 ] && [ "$minor_version" -le 7 ]; then
@@ -88,7 +88,7 @@ run_if_supported_version() {
exec "$interpreter" "$($interpreter -c "import os; print(os.path.dirname(os.path.realpath('$0')))")/cqlsh.py" "$@"
exit
else
- echo "Warning: unsupported version of Python, required 3.6-3.11 but found" "$version" >&2
+ echo "Warning: unsupported version of Python, required 3.6-3.13 but found" "$version" >&2
fi
fi
}
diff --git a/bin/cqlsh.py b/bin/cqlsh.py
index 738f0aeeb716..d7aa56e41c9a 100755
--- a/bin/cqlsh.py
+++ b/bin/cqlsh.py
@@ -21,8 +21,8 @@
import sys
from glob import glob
-if sys.version_info < (3, 6) or sys.version_info >= (3, 12):
- sys.exit("\ncqlsh requires Python 3.6-3.11\n")
+if sys.version_info < (3, 6) or sys.version_info >= (3, 14):
+ sys.exit("\ncqlsh requires Python 3.6-3.13\n")
# see CASSANDRA-10428
if platform.python_implementation().startswith('Jython'):
@@ -56,7 +56,7 @@ def find_zip(libprefix):
sys.path.insert(0, os.path.join(cql_zip, 'cassandra-driver-' + ver))
# the driver needs dependencies
-third_parties = ('pure_sasl-', 'wcwidth-')
+third_parties = ('pure_sasl-', 'wcwidth-', 'pyasyncore-', 'geomet-', 'datastax_db_*-')
for lib in third_parties:
lib_zip = find_zip(lib)
diff --git a/build.properties.default b/build.properties.default
index 36676f5712d8..380270479620 100644
--- a/build.properties.default
+++ b/build.properties.default
@@ -21,3 +21,4 @@
artifact.remoteRepository.central: https://repo1.maven.org/maven2
artifact.remoteRepository.apache: https://repo.maven.apache.org/maven2
artifact.remoteRepository.apacheSnapshot: https://repository.apache.org/content/repositories/snapshots
+artifact.remoteRepository.datastax: https://repo.datastax.com/dse
diff --git a/build.xml b/build.xml
index 15298ede273b..ba2944f41c1e 100644
--- a/build.xml
+++ b/build.xml
@@ -14,7 +14,7 @@
See the License for the specific language governing permissions and
limitations under the License.
-->
-
@@ -33,19 +33,18 @@
-
-
-
-
+
+
+
+
-
+
@@ -95,11 +94,12 @@
-
-
-
+
+
+
@@ -108,19 +108,20 @@
+
-
+
-
+
-
-
+
+
@@ -133,16 +134,24 @@
-
+
-
+
+
+
+
+
+
+
+
+
-
+
@@ -160,7 +169,6 @@
-
+
+ -XX:G1RSetUpdatingPauseTimePercent=5
+ -XX:MaxGCPauseMillis=100
+
+
+ -XX:-RestrictContended
+ -XX:+UseThreadPriorities
+ -XX:+DebugNonSafepoints
+ -XX:+UseStringDeduplication
+ -XX:StringTableSize=1000003
+ -XX:+PerfDisableSharedMem
+ -XX:+AlwaysPreTouch
+ -XX:+UseTLAB
+ -XX:+ResizeTLAB
+ -XX:+UseNUMA
+
+
+ --add-exports java.base/jdk.internal.misc=ALL-UNNAMED
+ --add-exports java.base/jdk.internal.ref=ALL-UNNAMED
+ --add-exports java.base/jdk.internal.perf=ALL-UNNAMED
+ --add-exports java.base/sun.nio.ch=ALL-UNNAMED
+ --add-exports java.management.rmi/com.sun.jmx.remote.internal.rmi=ALL-UNNAMED
+ --add-exports java.rmi/sun.rmi.registry=ALL-UNNAMED
+ --add-exports java.rmi/sun.rmi.server=ALL-UNNAMED
+ --add-exports jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED
+ --add-exports jdk.naming.dns/com.sun.jndi.dns=java.naming
+ --add-exports jdk.unsupported/sun.misc=ALL-UNNAMED
+
+ --add-opens java.base/java.io=ALL-UNNAMED
+ --add-opens java.base/java.lang=ALL-UNNAMED
+ --add-opens java.base/java.lang.module=ALL-UNNAMED
+ --add-opens java.base/java.lang.ref=ALL-UNNAMED
+ --add-opens java.base/java.lang.reflect=ALL-UNNAMED
+ --add-opens java.base/java.math=ALL-UNNAMED
+ --add-opens java.base/java.net=ALL-UNNAMED
+ --add-opens java.base/java.nio=ALL-UNNAMED
+ --add-opens java.base/java.nio.charset=ALL-UNNAMED
+ --add-opens java.base/java.nio.file.spi=ALL-UNNAMED
+ --add-opens java.base/java.util=ALL-UNNAMED
+ --add-opens java.base/java.util.concurrent.locks=ALL-UNNAMED
+ --add-opens java.base/jdk.internal.loader=ALL-UNNAMED
+ --add-opens java.base/jdk.internal.math=ALL-UNNAMED
+ --add-opens java.base/jdk.internal.module=ALL-UNNAMED
+ --add-opens java.base/jdk.internal.ref=ALL-UNNAMED
+ --add-opens java.base/jdk.internal.reflect=ALL-UNNAMED
+ --add-opens java.base/jdk.internal.vm=ALL-UNNAMED
+ --add-opens java.base/sun.nio.ch=ALL-UNNAMED
+ --add-opens jdk.compiler/com.sun.tools.javac=ALL-UNNAMED
+ --add-opens jdk.management.jfr/jdk.management.jfr=ALL-UNNAMED
+ --add-opens jdk.management/com.sun.management.internal=ALL-UNNAMED
+ --add-opens jdk.naming.dns/com.sun.jndi.dns=ALL-UNNAMED
+
+ --add-opens java.base/java.nio.file.attribute=ALL-UNNAMED
+
+
+ --add-opens java.base/java.util.concurrent=ALL-UNNAMED
+ --add-opens java.base/java.util.concurrent.atomic=ALL-UNNAMED
+ --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED
+
+
+
+
+
+
+ --add-modules jdk.incubator.vector
+
+
+
+
+
+
+
+
-
+
+ failonerror="true"
+ fork="true"
+ outputproperty="antlr.output"
+ errorproperty="antlr.error">
@@ -479,15 +592,6 @@
-
-
-
-
-
-
@@ -508,6 +612,7 @@
+
@@ -536,7 +641,7 @@
-
@@ -716,7 +821,6 @@
-
@@ -728,6 +832,15 @@
+
+
+
+
+ dse-db-all]]>
+ com.datastax.db
+ db-all]]>
+
+ description="Assemble DSE DB JAR files">
@@ -749,9 +862,9 @@
-
+
-
+
@@ -760,7 +873,7 @@
+ description="Assemble DSE DB JAR files">
@@ -775,7 +888,7 @@
+ description="Assemble DSE DB JAR files">
@@ -791,7 +904,7 @@
+ description="Assemble DSE DB JAR files">
@@ -826,13 +939,13 @@
+ description="Assemble DSE DB JAR files">
-
+
@@ -842,7 +955,7 @@
-
+
@@ -869,7 +982,6 @@
-
@@ -910,8 +1022,8 @@
-
+
@@ -980,7 +1092,7 @@
-
+
@@ -996,6 +1108,118 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
-
+
-
+
+
+
+
+
+
+
@@ -1276,6 +1515,8 @@
+
+
@@ -1296,6 +1537,8 @@
+
+
@@ -1318,6 +1561,7 @@
+
@@ -1330,7 +1574,7 @@
-
@@ -1339,10 +1583,32 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -1359,10 +1625,46 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -1388,6 +1690,7 @@
+
@@ -1401,6 +1704,7 @@
+
@@ -1413,6 +1717,7 @@
+
@@ -1449,6 +1754,22 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -1457,22 +1778,41 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
@@ -1480,6 +1820,7 @@
+
@@ -1494,6 +1835,7 @@
timeout="${test.long.timeout}">
+
@@ -1501,6 +1843,7 @@
+
@@ -1519,6 +1862,7 @@
+
@@ -1565,6 +1909,7 @@
+
@@ -1708,6 +2053,13 @@
+
+
+
+
+
+
+
@@ -1715,6 +2067,13 @@
+
+
+
+
+
+
+
@@ -1777,6 +2136,7 @@
+
@@ -1816,6 +2176,7 @@
+
@@ -1833,6 +2194,7 @@
+
@@ -1921,7 +2283,7 @@
-
+
@@ -1943,10 +2305,10 @@
]]>
- IDE configuration in .idea/ updated for use with JDK${ant.java.version}.
+ IDE configuration in .idea/ updated for use with JDK${ant.java.version}.
- In IntelliJ verify that the SDK is ${ant.java.version}, and its path is valid.
- This can be verified in 'Project Structure/Project Setting/Project' and 'Project Structure/Platform Setting/SDKs'.
+ In IntelliJ verify that the SDK is ${ant.java.version}, and its path is valid.
+ This can be verified in 'Project Structure/Project Setting/Project' and 'Project Structure/Platform Setting/SDKs'.
@@ -1996,6 +2358,7 @@
+
@@ -2003,8 +2366,8 @@
+
-
${eclipse-libs-list}
]]>
@@ -2046,7 +2409,7 @@
file="${build.dir}/${final.name}-parent.pom"
packaging="pom"/>
-
+
+
+
+
+
+
@@ -2067,7 +2440,7 @@
file="${build.dir}/${final.name}-parent.pom"
packaging="pom"/>
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/conf/cassandra-env.sh b/conf/cassandra-env.sh
index 2d83763b8994..14caee2366d8 100644
--- a/conf/cassandra-env.sh
+++ b/conf/cassandra-env.sh
@@ -80,7 +80,7 @@ echo "$JVM_OPTS" | grep -qe "-[X]log:gc"
if [ "$?" = "1" ] ; then # [X] to prevent ccm from replacing this line
# only add -Xlog:gc if it's not mentioned in jvm-server.options file
mkdir -p ${CASSANDRA_LOG_DIR}
- JVM_OPTS="$JVM_OPTS -Xlog:gc=info,heap*=trace,age*=debug,safepoint=info,promotion*=trace:file=${CASSANDRA_LOG_DIR}/gc.log:time,uptime,pid,tid,level:filecount=10,filesize=10485760"
+ JVM_OPTS="$JVM_OPTS -Xlog:gc=info,heap*=debug,age*=debug,safepoint=info,promotion*=debug:file=${CASSANDRA_LOG_DIR}/gc.log:time,uptime,pid,tid,level:filecount=10,filesize=10485760"
fi
# Check what parameters were defined on jvm-server.options file to avoid conflicts
@@ -221,9 +221,9 @@ JVM_ON_OUT_OF_MEMORY_ERROR_OPT="-XX:OnOutOfMemoryError=kill -9 %p"
# for more on configuring JMX through firewalls, etc. (Short version:
# get it working with no firewall first.)
#
-# Cassandra ships with JMX accessible *only* from localhost.
+# Cassandra ships with JMX accessible *only* from localhost.
# To enable remote JMX connections, uncomment lines below
-# with authentication and/or ssl enabled. See https://wiki.apache.org/cassandra/JmxSecurity
+# with authentication and/or ssl enabled. See https://wiki.apache.org/cassandra/JmxSecurity
#
if [ "x$LOCAL_JMX" = "x" ]; then
LOCAL_JMX=yes
diff --git a/conf/cassandra.yaml b/conf/cassandra.yaml
index e09518188a60..f620f68d3cc9 100644
--- a/conf/cassandra.yaml
+++ b/conf/cassandra.yaml
@@ -395,6 +395,11 @@ partitioner: org.apache.cassandra.dht.Murmur3Partitioner
# data_file_directories:
# - /var/lib/cassandra/data
+# Metadata directory that holds information about the cluster, local node and its peers.
+# Currently, only a single subdirectory called 'nodes' will be used.
+# If not set, the default directory is $CASSANDRA_HOME/data/metadata.
+# metadata_directory: /var/lib/cassandra/metadata
+
# Directory were Cassandra should store the data of the local system keyspaces.
# By default Cassandra will store the data of the local system keyspaces in the first of the data directories specified
# by data_file_directories.
@@ -687,6 +692,8 @@ commitlog_disk_access_mode: legacy
# none : Flush without compressing blocks but while still doing checksums.
# fast : Flush with a fast compressor. If the table is already using a
# fast compressor that compressor is used.
+# adaptive : Flush with a fast adaptive compressor. If the table is already using a
+# fast compressor that compressor is used.
# table: Always flush with the same compressor that the table uses. This
# was the pre 4.0 behavior.
#
@@ -822,7 +829,7 @@ memtable:
#
# offheap_objects
# off heap objects
-memtable_allocation_type: heap_buffers
+memtable_allocation_type: offheap_objects
# Limit memory usage for Merkle tree calculations during repairs of a certain
# table and common token range. Repair commands targetting multiple tables or
@@ -848,7 +855,7 @@ memtable_allocation_type: heap_buffers
# There isn't a limit by default for backwards compatibility, but this can
# produce OOM for commands repairing multiple tables or multiple virtual nodes.
# A limit of just 1 simultaneous Merkle tree request is generally recommended
-# with no virtual nodes so repair_session_space, and thereof the Merkle tree
+# with no virtual nodes so repair_session_space, and therefore the Merkle tree
# resolution, can be high. For virtual nodes a value of 1 with the default
# repair_session_space value will produce higher resolution Merkle trees
# at the expense of speed. Alternatively, when working with virtual nodes it
@@ -951,7 +958,7 @@ index_summary_resize_interval: 60m
# buffers. Enable this to avoid sudden dirty buffer flushing from
# impacting read latencies. Almost always a good idea on SSDs; not
# necessarily on platters.
-trickle_fsync: false
+trickle_fsync: true
# Min unit: KiB
trickle_fsync_interval: 10240KiB
@@ -1264,7 +1271,8 @@ sstable_preemptive_open_interval: 50MiB
# set to true, each newly created sstable will have a UUID based generation identifier and such files are
# not readable by previous Cassandra versions. At some point, this option will become true by default
# and eventually get removed from the configuration.
-uuid_sstable_identifiers_enabled: false
+# In Converged Cassandra, we enable this option by default
+uuid_sstable_identifiers_enabled: true
# When enabled, permits Cassandra to zero-copy stream entire eligible
# SSTables between nodes, including every component.
@@ -1316,14 +1324,21 @@ uuid_sstable_identifiers_enabled: false
# low is equally ill-advised since clients could get timeouts even for successful
# operations just because the timeout setting is too tight.
-# How long the coordinator should wait for read operations to complete.
+# How long the coordinator should wait for read operations to complete. This
+# timeout does not apply to aggregated queries such as SELECT COUNT(*), MIN(x), etc.
# Lowest acceptable value is 10 ms.
# Min unit: ms
read_request_timeout: 5000ms
-# How long the coordinator should wait for seq or index scans to complete.
+# How long the coordinator should wait for seq or index scans to complete. This
+# timeout does not apply to aggregated queries such as SELECT COUNT(*), MIN(x), etc.
# Lowest acceptable value is 10 ms.
# Min unit: ms
range_request_timeout: 10000ms
+# How long the coordinator should wait for aggregation read operations to complete,
+# such as SELECT COUNT(*), MIN(x), etc.
+# Lowest acceptable value is 10 ms.
+# Min unit: ms
+aggregation_request_timeout: 120000ms
# How long the coordinator should wait for writes to complete.
# Lowest acceptable value is 10 ms.
# Min unit: ms
@@ -1347,6 +1362,16 @@ truncate_request_timeout: 60000ms
# Lowest acceptable value is 10 ms.
# Min unit: ms
request_timeout: 10000ms
+# Upper bound for how long any request received via native transport
+# should be considered live and serviceable by the system. This is
+# currently considered at two points: when the message is dequeued and
+# executed by the NATIVE_TRANSPORT_REQUESTS stage, and when the message
+# is dequeued and executed by an async stage if NATIVE_TRANSPORT_ASYNC_READ_WRITE_ENABLED
+# is set to true. If the request is not completed within this time, an
+# OverloadedException is thrown.
+# Min unit: ms
+native_transport_timeout: 12000ms
+
# Defensive settings for protecting Cassandra from true network partitions.
# See (CASSANDRA-14358) for details.
@@ -1861,6 +1886,11 @@ unlogged_batch_across_partitions_warn_threshold: 10
# Audit logging - Logs every incoming CQL command request, authentication to a node. See the docs
# on audit_logging for full details about the various configuration options and production tips.
+# NOTE: Chronicle Queue has changed the enums used for log rolling roll_cycle).
+# Older legacy options will still work for the foreseeable future, but you will see warnings in logs and future dependency
+# upgrades may break your log rolling param. The default log rolling param will be changed with the next major release
+# from HOURLY to FAST_HOURLY, primarily different on how frequently indexes are built. For more info refer to:
+# net.openhft.chronicle.queue.RollCycles
audit_logging_options:
enabled: false
logger:
@@ -1943,6 +1973,10 @@ report_unconfirmed_repaired_data_mismatches: false
# Defaults to false to disable dynamic data masking.
# dynamic_data_masking_enabled: false
+# This is the page size used internally by aggregation queries. It aims to limit the memory used by aggregation
+# queries when there is a lot of data to aggregate.
+# aggregation_subpage_size_in_kb: 2048
+
#########################
# EXPERIMENTAL FEATURES #
#########################
@@ -1951,10 +1985,6 @@ report_unconfirmed_repaired_data_mismatches: false
# Materialized views are considered experimental and are not recommended for production use.
materialized_views_enabled: false
-# Enables SASI index creation on this node.
-# SASI indexes are considered experimental and are not recommended for production use.
-sasi_indexes_enabled: false
-
# Enables creation of transiently replicated keyspaces on this node.
# Transient replication is experimental and is not recommended for production use.
transient_replication_enabled: false
@@ -2011,7 +2041,7 @@ drop_compact_storage_enabled: false
# columns_per_table_warn_threshold: -1
# columns_per_table_fail_threshold: -1
#
-# Guardrail to warn or fail when creating more secondary indexes per table than threshold.
+# Guardrail to warn or fail when creating more secondary indexes per table than threshold (does not apply to CUSTOM INDEX StorageAttachedIndex).
# The two thresholds default to -1 to disable.
# secondary_indexes_per_table_warn_threshold: -1
# secondary_indexes_per_table_fail_threshold: -1
@@ -2019,6 +2049,22 @@ drop_compact_storage_enabled: false
# Guardrail to enable or disable the creation of secondary indexes
# secondary_indexes_enabled: true
#
+# Failure threshold for number of StorageAttachedIndex per table (only applies to CUSTOM INDEX StorageAttachedIndex)
+# Default is 10 (same when apply_dbaas_defaults is enabled)
+# sai_indexes_per_table_warn_threshold: -1
+# sai_indexes_per_table_fail_threshold: 10
+#
+# Failure threshold for total number of StorageAttachedIndex across all keyspaces (only applies to CUSTOM INDEX StorageAttachedIndex)
+# Default is 10 (same when apply_dbaas_defaults is enabled)
+# sai_indexes_total_warn_threshold: -1
+# sai_indexes_total_fail_threshold: 100
+#
+# Guardrail to warn or fail when creating more trusted custom indexes (cassandra.trusted_index_implementations)
+# per table than threshold, counted per implementation class.
+# The two thresholds default to -1 to disable.
+# trusted_indexes_per_table_warn_threshold: -1
+# trusted_indexes_per_table_fail_threshold: -1
+#
# Guardrail to warn or fail when creating more materialized views per table than threshold.
# The two thresholds default to -1 to disable.
# materialized_views_per_table_warn_threshold: -1
@@ -2147,7 +2193,7 @@ drop_compact_storage_enabled: false
# Guardrail to warn or fail when creating a vector column with more dimensions than threshold.
# Default -1 to disable.
# vector_dimensions_warn_threshold: -1
-# vector_dimensions_fail_threshold: -1
+# vector_dimensions_fail_threshold: 8192
#
# Guardrail to indicate whether or not users are allowed to use ALTER TABLE commands to make column changes to tables
# alter_table_enabled: true
@@ -2218,6 +2264,20 @@ drop_compact_storage_enabled: false
# sai_vector_term_size_warn_threshold: 16KiB
# sai_vector_term_size_fail_threshold: 32KiB
+# Guardrail to warn or fail when using LIMIT/OFFSET paging skipping more rows than threshold.
+# Default offset_rows_warn_threshold is 10000, may differ if emulate_dbaas_defaults is enabled
+# Default offset_rows_failure_threshold is 20000, may differ if emulate_dbaas_defaults is enabled
+# offset_rows_warn_threshold: 10000
+# offset_rows_failure_threshold: 20000
+
+# Guardrail to warn or fail when a SELECT query has more column value filters than threshold.
+# Note that restrictions on indexed columns can be expanded to multiple column filters if the indexes have an analyzer.
+# In that case, there will be a filter for every token produced by the analyzer for the queried column value. This can
+# prevent that productive analyzers such as n-gram explode the query to a large number of filtering operations.
+# Default -1 to disable, may differ if emulate_dbaas_defaults is enabled
+# query_filters_warn_threshold: -1
+# query_filters_fail_threshold: -1
+
# The default secondary index implementation when CREATE INDEX does not specify one via USING.
# ex. "legacy_local_table" - (default) legacy secondary index, implemented as a hidden table
# ex. "sai" - "storage-attched" index, implemented via optimized SSTable/Memtable-attached indexes
@@ -2256,23 +2316,21 @@ drop_compact_storage_enabled: false
# This property indicates with what Cassandra major version the storage format will be compatible with.
#
# The chosen storage compatibility mode will determine the versions of the written sstables, commitlogs, hints, etc.
-# For example, if we're going to remain compatible with Cassandra 4.x, the value of this property should be 4, which
-# will make us use sstables in the latest N version of the BIG format.
#
# This will also determine if certain features that depend on newer formats are available. For example, extended TTL
# (up to 2106) depends on the sstable, commit-log, hints, and messaging versions introduced by Cassandra 5.0, so that
-# feature won't be available if this property is set to CASSANDRA_4. See the upgrade guide for more details.
+# feature won't be available if this property is set to HCD_1.
#
# Possible values are:
#
-# ** CASSANDRA_4: Stays compatible with the 4.x line in features, formats and component versions.
+# ** HCD_1: Stays compatible with the 4.x line in features, formats and component versions.
# ** UPGRADING: The cluster monitors the version of each node during this interim stage. This has a cost but ensures
# all new features, formats, versions, etc. are enabled safely.
# ** NONE: Start with all the new features and formats enabled.
#
# A typical upgrade would be:
#
-# . Do a rolling upgrade, starting all nodes in CASSANDRA_X compatibility mode.
+# . Do a rolling upgrade, starting all nodes in HCD_1 compatibility mode.
# . Once the new binary is rendered stable, do a rolling restart with the UPGRADING mode. The cluster will keep new
# features disabled until all nodes are started in the UPGRADING mode; when that happens, new features controlled by
# the storage compatibility mode are enabled.
@@ -2280,4 +2338,7 @@ drop_compact_storage_enabled: false
# and ensures stability. If Cassandra was started at the previous version by accident, a node with disabled
# compatibility mode would no longer toggle behaviors as when it was running in the UPGRADING mode.
#
-storage_compatibility_mode: CASSANDRA_4
+storage_compatibility_mode: HCD_1
+
+# Changes defaults considered production safest for HCD users
+# hcd_guardrail_defaults: false
diff --git a/conf/cassandra_latest.yaml b/conf/cassandra_latest.yaml
index 0c7a792c8f40..ccedb80640af 100644
--- a/conf/cassandra_latest.yaml
+++ b/conf/cassandra_latest.yaml
@@ -1331,6 +1331,15 @@ truncate_request_timeout: 60000ms
# Lowest acceptable value is 10 ms.
# Min unit: ms
request_timeout: 10000ms
+# Upper bound for how long any request received via native transport
+# should be considered live and serviceable by the system. This is
+# currently considered at two points: when the message is dequeued and
+# executed by the NATIVE_TRANSPORT_REQUESTS stage, and when the message
+# is dequeued and executed by an async stage if NATIVE_TRANSPORT_ASYNC_READ_WRITE_ENABLED
+# is set to true. If the request is not completed within this time, an
+# OverloadedException is thrown.
+# Min unit: ms
+native_transport_timeout: 12000ms
# Defensive settings for protecting Cassandra from true network partitions.
# See (CASSANDRA-14358) for details.
@@ -1556,6 +1565,35 @@ dynamic_snitch_reset_interval: 600000ms
# until the pinned host was 20% worse than the fastest.
dynamic_snitch_badness_threshold: 1.0
+# Paxos variant for lightweight transactions (LWTs)
+# Options:
+# v1
+# - Legacy Paxos. Expect 4RTs for a write and 3RTs for a read. (default)
+#
+# v1_without_linearizable_reads_or_rejected_writes
+# - Legacy Paxos. Expect 4RTs for a write and 2RTs for a read.
+# With legacy semantics for read/read and rejected write linearizability, i.e. not guaranteed.
+#
+# v2
+# - Optimized Paxos. Expect 2RTs for a write, and either 1RT or 2RT for a read. (recommended)
+#
+# v2_without_linearizable_reads
+# - Optimized Paxos. Expect 2RTs for a write and 1RT for a read.
+#
+# v2_without_linearizable_reads_or_rejected_writes
+# - Optimized Paxos. Expect 2RTs for a write and 1RT for a read.
+# With legacy semantics for read/read and rejected write linearizability, i.e. not guaranteed.
+#
+# To upgrade from v1:
+# 1. Ensure all nodes are on same Cassandra version 4.1+.
+# 2. Run `nodetool repair --full -pr` on each node.
+# 3. Set paxos_variant: v2 on each node and rolling restart.
+# Rollback is safe: revert to v1 and rolling restart. No data migration needed.
+#
+# With any v2 variant and `paxos_state_purging: repaired` it is safe to use ANY Commit consistency.
+#
+paxos_variant: v2
+
# Configures Java crypto provider. By default, it will use DefaultCryptoProvider
# which will install Amazon Correto Crypto Provider.
#
@@ -2248,3 +2286,167 @@ default_secondary_index_enabled: true
# compatibility mode would no longer toggle behaviors as when it was running in the UPGRADING mode.
#
storage_compatibility_mode: NONE
+
+# Prevents preparing a repair session or beginning a repair streaming session if pending compactions is over
+# the given value. Defaults to disabled.
+# reject_repair_compaction_threshold: 1024
+
+# Ratio of disk that must be unused to run repair. It is useful to avoid disks filling up during
+# repair as anti-compaction during repair may contribute to additional space temporarily.
+# For example, setting this to 0.2 means at least 20% of disk must be unused.
+# Set to 0.0 to disable this check. Defaults to 0.0 (disabled) on 5.0 for backward-compatibility.
+# repair_disk_headroom_reject_ratio: 0.0
+
+# Configuration for Auto Repair Scheduler.
+#
+# This feature is disabled by default.
+#
+# NOTE: The auto-repair feature requires enabling the JVM property `cassandra.autorepair.enable=true`.
+#
+# See: https://cassandra.apache.org/doc/latest/cassandra/managing/operating/auto_repair.html for an overview of this
+# feature.
+#
+# auto_repair:
+# # Enable/Disable the auto-repair scheduler.
+# # If set to false, the scheduler thread will not be started.
+# # If set to true, the repair scheduler thread will be created. The thread will
+# # check for secondary configuration available for each repair type (full, incremental,
+# # and preview_repaired), and based on that, it will schedule repairs.
+# enabled: true
+# repair_type_overrides:
+# full:
+# # Enable/Disable full auto-repair
+# enabled: true
+# # Minimum duration between repairing the same node again. This is useful for tiny clusters,
+# # such as clusters with 5 nodes that finish repairs quickly. This means that if the scheduler completes one
+# # round on all nodes in less than this duration, it will not start a new repair round on a given node until
+# # this much time has passed since the last repair completed. Consider increasing to a larger value to reduce
+# # the impact of repairs, however note that one should attempt to run repairs at a smaller interval than
+# # gc_grace_seconds to avoid potential data resurrection.
+# min_repair_interval: 24h
+# token_range_splitter:
+# # Implementation of IAutoRepairTokenRangeSplitter; responsible for splitting token ranges
+# # for repair assignments.
+# #
+# # Out of the box, Cassandra provides org.apache.cassandra.repair.autorepair.{RepairTokenRangeSplitter,
+# # FixedTokenRangeSplitter}.
+# #
+# # - RepairTokenRangeSplitter (default) attempts to intelligently split ranges based on data size and partition
+# # count.
+# # - FixedTokenRangeSplitter splits into fixed ranges based on the 'number_of_subranges' option.
+# # class_name: org.apache.cassandra.repair.autorepair.RepairTokenRangeSplitter
+#
+# # Optional parameters can be specified in the form of:
+# # parameters:
+# # param_key1: param_value1
+# parameters:
+# # The target and maximum amount of compressed bytes that should be included in a repair assignment.
+# # This scopes the amount of work involved in a repair and includes the data covering the range being
+# # repaired.
+# bytes_per_assignment: 50GiB
+# # The maximum number of bytes to cover in an individual schedule. This serves as
+# # a mechanism to throttle the work done in each repair cycle. You may reduce this
+# # value if the impact of repairs is causing too much load on the cluster or increase it
+# # if writes outpace the amount of data being repaired. Alternatively, adjust the
+# # min_repair_interval.
+# # This is set to a large value for full repair to attempt to repair all data per repair schedule.
+# max_bytes_per_schedule: 100000GiB
+# incremental:
+# enabled: false
+# # Incremental repairs operate over unrepaired data and should finish quickly. Running incremental repair
+# # frequently keeps the unrepaired set smaller and thus causes repairs to operate over a smaller set of data,
+# # so a more frequent schedule such as 1h is recommended.
+# # NOTE: Please consult
+# # https://cassandra.apache.org/doc/latest/cassandra/managing/operating/auto_repair.html#enabling-ir
+# # for guidance on enabling incremental repair on ane exiting cluster.
+# min_repair_interval: 24h
+# token_range_splitter:
+# parameters:
+# # Configured to attempt repairing 50GiB of compressed data per repair.
+# # This throttles the amount of incremental repair and anticompaction done per schedule after incremental
+# # repairs are turned on.
+# bytes_per_assignment: 50GiB
+# # Restricts the maximum number of bytes to cover in an individual schedule to the configured
+# # max_bytes_per_schedule value (defaults to 100GiB for incremental).
+# # Consider increasing this value if more data is written than this limit within the min_repair_interval.
+# max_bytes_per_schedule: 100GiB
+# preview_repaired:
+# # Performs preview repair over repaired SSTables, useful to detect possible inconsistencies in the repaired
+# # data set.
+# enabled: false
+# min_repair_interval: 24h
+# token_range_splitter:
+# parameters:
+# bytes_per_assignment: 50GiB
+# max_bytes_per_schedule: 100000GiB
+# # Time interval between successive checks to see if ongoing repairs are complete or if it is time to schedule
+# # repairs.
+# repair_check_interval: 5m
+# # Minimum duration for the execution of a single repair task. This prevents the scheduler from overwhelming
+# # the node by scheduling too many repair tasks in a short period of time.
+# repair_task_min_duration: 5s
+# # The scheduler needs to adjust its order when nodes leave the ring. Deleted hosts are tracked in metadata
+# # for a specified duration to ensure they are indeed removed before adjustments are made to the schedule.
+# history_clear_delete_hosts_buffer_interval: 2h
+# # By default repair is disabled if there are mixed major versions detected - which would happen
+# # if a major version upgrade is being performed on the cluster, but a user can enable it using this flag
+# mixed_major_version_repair_enabled: false
+# # NOTE: Each of the below settings can be overridden per repair type under repair_type_overrides
+# global_settings:
+# # If true, attempts to group tables in the same keyspace into one repair; otherwise, each table is repaired
+# # individually.
+# repair_by_keyspace: true
+# # Number of threads to use for each repair job scheduled by the scheduler. Similar to the -j option in nodetool
+# # repair.
+# number_of_repair_threads: 1
+# # Number of nodes running repair in parallel. If parallel_repair_percentage is set, the larger value is used.
+# parallel_repair_count: 3
+# # Percentage of nodes in the cluster running repair in parallel. If parallel_repair_count is set, the larger value
+# # is used.
+# parallel_repair_percentage: 3
+# # Whether to allow a node to take its turn running repair while one or more of its replicas are running repair.
+# # Defaults to false, as running repairs concurrently on replicas can increase load and also cause anticompaction
+# # conflicts while running incremental repair.
+# allow_parallel_replica_repair: false
+# # An addition to allow_parallel_replica_repair that also blocks repairs when replicas (including this node itself)
+# # are repairing in any schedule. For example, if a replica is executing full repairs, a value of false will
+# # prevent starting incremental repairs for this node. Defaults to true and is only evaluated when
+# # allow_parallel_replica_repair is false.
+# allow_parallel_replica_repair_across_schedules: true
+# # Repairs materialized views if true.
+# materialized_view_repair_enabled: false
+# # Delay before starting repairs after a node restarts to avoid repairs starting immediately after a restart.
+# initial_scheduler_delay: 5m
+# # Timeout for retrying stuck repair sessions.
+# repair_session_timeout: 3h
+# # Force immediate repair on new nodes after they join the ring.
+# force_repair_new_node: false
+# # Threshold to skip repairing tables with too many SSTables. Defaults to 10,000 SSTables to avoid penalizing good
+# # tables.
+# sstable_upper_threshold: 50000
+# # Maximum time allowed for repairing one table on a given node. If exceeded, the repair proceeds to the
+# # next table.
+# table_max_repair_time: 6h
+# # Avoid running repairs in specific data centers. By default, repairs run in all data centers. Specify data
+# # centers to exclude in this list. Note that repair sessions will still consider all replicas from excluded
+# # data centers. Useful if you have keyspaces that are not replicated in certain data centers, and you want to
+# # not run repair schedule in certain data centers.
+# ignore_dcs: []
+# # Repair only the primary ranges owned by a node. Equivalent to the -pr option in nodetool repair. Defaults
+# # to true. General advice is to keep this true.
+# repair_primary_token_range_only: true
+# # Maximum number of retries for a repair session.
+# repair_max_retries: 3
+# # Backoff time before retrying a repair session.
+# repair_retry_backoff: 30s
+# token_range_splitter:
+# # Splitter implementation to generate repair assignments. Defaults to RepairTokenRangeSplitter.
+# class_name: org.apache.cassandra.repair.autorepair.RepairTokenRangeSplitter
+# parameters:
+# # Maximum number of partitions to include in a repair assignment. Used to reduce number of partitions
+# # present in merkle tree leaf nodes to avoid overstreaming.
+# partitions_per_assignment: 1048576
+# # Maximum number of tables to include in a repair assignment. This reduces the number of repairs,
+# # especially in keyspaces with many tables. The splitter avoids batching tables together if they
+# # exceed other configuration parameters like bytes_per_assignment or partitions_per_assignment.
+# max_tables_per_assignment: 64
diff --git a/conf/cqlshrc.sample b/conf/cqlshrc.sample
index 79d719460e29..abc11b6084fe 100644
--- a/conf/cqlshrc.sample
+++ b/conf/cqlshrc.sample
@@ -32,6 +32,15 @@
; classname = PlainTextAuthProvider
; username = user1
+[auth_provider]
+;; you can specify any auth provider found in your python environment
+;; module and class will be used to dynamically load the class
+;; all other properties found here and in the credentials file under the class name
+;; will be passed to the constructor
+; module = cassandra.auth
+; classname = PlainTextAuthProvider
+; username = user1
+
[protocol]
;; Specify a specific protcol version otherwise the client will default and downgrade as necessary
; version = None
diff --git a/conf/cqlshrc.sample.cloud b/conf/cqlshrc.sample.cloud
new file mode 100644
index 000000000000..62528670c48b
--- /dev/null
+++ b/conf/cqlshrc.sample.cloud
@@ -0,0 +1,17 @@
+; Copyright DataStax, Inc.
+;
+; Licensed under the Apache License, Version 2.0 (the "License");
+; you may not use this file except in compliance with the License.
+; You may obtain a copy of the License at
+;
+; http://www.apache.org/licenses/LICENSE-2.0
+;
+; Unless required by applicable law or agreed to in writing, software
+; distributed under the License is distributed on an "AS IS" BASIS,
+; WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+; See the License for the specific language governing permissions and
+; limitations under the License.
+;
+; Sample ~/.cqlshrc file with cloud configuration.
+[connection]
+secure_connect_bundle = /path/to/creds.zip
diff --git a/conf/jvm11-clients.options b/conf/jvm11-clients.options
index 3d59816c045f..08ce8f2a30f6 100644
--- a/conf/jvm11-clients.options
+++ b/conf/jvm11-clients.options
@@ -29,18 +29,28 @@
-Djdk.attach.allowAttachSelf=true
--add-exports java.base/jdk.internal.misc=ALL-UNNAMED
--add-exports java.base/jdk.internal.ref=ALL-UNNAMED
+--add-exports java.base/jdk.internal.util=ALL-UNNAMED
--add-exports java.base/sun.nio.ch=ALL-UNNAMED
--add-exports java.management.rmi/com.sun.jmx.remote.internal.rmi=ALL-UNNAMED
--add-exports java.rmi/sun.rmi.registry=ALL-UNNAMED
--add-exports java.rmi/sun.rmi.server=ALL-UNNAMED
--add-exports java.sql/java.sql=ALL-UNNAMED
+--add-exports jdk.compiler/com.sun.tools.javac=ALL-UNNAMED
+--add-exports jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED
+--add-exports jdk.unsupported/sun.misc=ALL-UNNAMED
+--add-opens java.base/java.io=ALL-UNNAMED
+--add-opens java.base/java.lang=ALL-UNNAMED
--add-opens java.base/java.lang.module=ALL-UNNAMED
+--add-opens java.base/java.lang.reflect=ALL-UNNAMED
+--add-opens java.base/java.util=ALL-UNNAMED
--add-opens java.base/jdk.internal.loader=ALL-UNNAMED
--add-opens java.base/jdk.internal.ref=ALL-UNNAMED
--add-opens java.base/jdk.internal.reflect=ALL-UNNAMED
--add-opens java.base/jdk.internal.math=ALL-UNNAMED
--add-opens java.base/jdk.internal.module=ALL-UNNAMED
+--add-opens java.base/java.util=ALL-UNNAMED
+--add-opens java.base/jdk.internal.util=ALL-UNNAMED
--add-opens java.base/jdk.internal.util.jar=ALL-UNNAMED
--add-opens jdk.management/com.sun.management.internal=ALL-UNNAMED
--add-opens java.base/java.lang.reflect=ALL-UNNAMED
diff --git a/conf/jvm11-server.options b/conf/jvm11-server.options
index 857e07857a05..05e180a4d995 100644
--- a/conf/jvm11-server.options
+++ b/conf/jvm11-server.options
@@ -30,6 +30,8 @@
# Disable biased locking as it does not benefit Cassandra.
-XX:-UseBiasedLocking
+-XX:ThreadPriorityPolicy=1
+-XX:+UseThreadPriorities
#################
# GC SETTINGS #
@@ -104,14 +106,21 @@
--add-exports java.rmi/sun.rmi.registry=ALL-UNNAMED
--add-exports java.rmi/sun.rmi.server=ALL-UNNAMED
--add-exports java.sql/java.sql=ALL-UNNAMED
+--add-exports jdk.unsupported/sun.misc=ALL-UNNAMED
+--add-exports jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED
+--add-opens java.base/java.io=ALL-UNNAMED
+--add-opens java.base/java.lang=ALL-UNNAMED
--add-opens java.base/java.lang.module=ALL-UNNAMED
+--add-opens java.base/java.lang.reflect=ALL-UNNAMED
+--add-opens=java.base/java.util=ALL-UNNAMED
--add-opens java.base/jdk.internal.loader=ALL-UNNAMED
--add-opens java.base/jdk.internal.ref=ALL-UNNAMED
--add-opens java.base/jdk.internal.reflect=ALL-UNNAMED
--add-opens java.base/jdk.internal.math=ALL-UNNAMED
--add-opens java.base/jdk.internal.module=ALL-UNNAMED
--add-opens java.base/jdk.internal.util.jar=ALL-UNNAMED
+--add-opens jdk.compiler/com.sun.tools.javac=ALL-UNNAMED
--add-opens jdk.management/com.sun.management.internal=ALL-UNNAMED
@@ -120,7 +129,7 @@
# Java 11 (and newer) GC logging options:
# See description of https://bugs.openjdk.java.net/browse/JDK-8046148 for details about the syntax
# The following is the equivalent to -XX:+PrintGCDetails -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10 -XX:GCLogFileSize=10M
-#-Xlog:gc=info,heap*=trace,age*=debug,safepoint=info,promotion*=trace:file=/var/log/cassandra/gc.log:time,uptime,pid,tid,level:filecount=10,filesize=10485760
+#-Xlog:gc=info,heap*=debug,age*=debug,safepoint=info,promotion*=debug:file=/var/log/cassandra/gc.log:time,uptime,pid,tid,level:filecount=10,filesize=10485760
# Notes for Java 8 migration:
#
diff --git a/conf/jvm17-clients.options b/conf/jvm17-clients.options
index 671d91b21f95..36e15c838fce 100644
--- a/conf/jvm17-clients.options
+++ b/conf/jvm17-clients.options
@@ -28,6 +28,8 @@
-Djdk.attach.allowAttachSelf=true
--add-exports java.base/jdk.internal.misc=ALL-UNNAMED
+--add-exports java.base/jdk.internal.ref=ALL-UNNAMED
+--add-exports java.base/sun.nio.ch=ALL-UNNAMED
--add-exports java.management.rmi/com.sun.jmx.remote.internal.rmi=ALL-UNNAMED
--add-exports java.rmi/sun.rmi.registry=ALL-UNNAMED
--add-exports java.rmi/sun.rmi.server=ALL-UNNAMED
diff --git a/conf/jvm17-server.options b/conf/jvm17-server.options
index 9a695aa351d7..ee121a0c86da 100644
--- a/conf/jvm17-server.options
+++ b/conf/jvm17-server.options
@@ -22,6 +22,9 @@
# See jvm-server.options. This file is specific for Java 17 and newer. #
###########################################################################
+-XX:ThreadPriorityPolicy=1
+-XX:+UseThreadPriorities
+
#################
# GC SETTINGS #
#################
@@ -46,7 +49,7 @@
# Main G1GC tunable: lowering the pause target will lower throughput and vise versa.
# 200ms is the JVM default and lowest viable setting
# 1000ms increases throughput. Keep it smaller than the timeouts in cassandra.yaml.
--XX:MaxGCPauseMillis=300
+-XX:MaxGCPauseMillis=500
## Optional G1 Settings
# Save CPU time on large (>= 16GB) heaps by delaying region scanning
@@ -67,6 +70,8 @@
-Djdk.attach.allowAttachSelf=true
--add-exports java.base/jdk.internal.misc=ALL-UNNAMED
+--add-exports java.base/jdk.internal.ref=ALL-UNNAMED
+--add-exports java.base/sun.nio.ch=ALL-UNNAMED
--add-exports java.management.rmi/com.sun.jmx.remote.internal.rmi=ALL-UNNAMED
--add-exports java.management/com.sun.jmx.remote.security=ALL-UNNAMED
--add-exports java.rmi/sun.rmi.registry=ALL-UNNAMED
@@ -74,8 +79,8 @@
--add-exports java.sql/java.sql=ALL-UNNAMED
--add-exports java.base/java.lang.ref=ALL-UNNAMED
--add-exports jdk.unsupported/sun.misc=ALL-UNNAMED
+--add-exports jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED
---add-opens java.base/java.lang.module=ALL-UNNAMED
--add-opens java.base/jdk.internal.loader=ALL-UNNAMED
--add-opens java.base/jdk.internal.ref=ALL-UNNAMED
--add-opens java.base/jdk.internal.reflect=ALL-UNNAMED
@@ -87,15 +92,20 @@
--add-opens java.base/java.io=ALL-UNNAMED
--add-opens java.base/java.lang.reflect=ALL-UNNAMED
--add-opens java.base/java.lang=ALL-UNNAMED
+--add-opens=java.base/java.nio.charset=ALL-UNNAMED
--add-opens java.base/java.util=ALL-UNNAMED
--add-opens java.base/java.nio=ALL-UNNAMED
+--add-opens jdk.compiler/com.sun.tools.javac=ALL-UNNAMED
+
+# required for org.apache.cassandra.Util.getSupportedMTimeGranularity
+--add-opens java.base/java.nio.file.attribute=ALL-UNNAMED
### GC logging options -- uncomment to enable
# Java 11 (and newer) GC logging options:
# See description of https://bugs.openjdk.java.net/browse/JDK-8046148 for details about the syntax
# The following is the equivalent to -XX:+PrintGCDetails -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10 -XX:GCLogFileSize=10M
-#-Xlog:gc=info,heap*=trace,age*=debug,safepoint=info,promotion*=trace:file=/var/log/cassandra/gc.log:time,uptime,pid,tid,level:filecount=10,filesize=10485760
+# -Xlog:gc=info,heap*=debug,age*=debug,safepoint=info,promotion*=debug:file=/var/log/cassandra/gc.log:time,uptime,pid,tid,level:filecount=10,filesize=10485760
# Notes for Java 8 migration:
#
@@ -118,5 +128,12 @@
# Revert changes in defaults introduced in https://netty.io/news/2022/03/10/4-1-75-Final.html
-Dio.netty.allocator.useCacheForAllThreads=true
-Dio.netty.allocator.maxOrder=11
+### Enable vector incubator feature (simd support)
+
+--add-modules jdk.incubator.vector
+
+### Compatibility Options
+--add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens java.base/java.io=ALL-UNNAMED --add-opens java.base/sun.nio.ch=ALL-UNNAMED --add-opens java.base/java.lang=ALL-UNNAMED --add-opens java.base/java.util=ALL-UNNAMED
+-Djava.security.manager=allow
# The newline in the end of file is intentional
diff --git a/conf/jvm22-clients.options b/conf/jvm22-clients.options
new file mode 100644
index 000000000000..81af895ed216
--- /dev/null
+++ b/conf/jvm22-clients.options
@@ -0,0 +1,50 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+###########################################################################
+# jvm22-clients.options #
+# #
+# See jvm-clients.options. This file is specific for Java 22 and newer. #
+###########################################################################
+
+###################
+# JPMS SETTINGS #
+###################
+
+-Djdk.attach.allowAttachSelf=true
+--add-exports java.base/jdk.internal.misc=ALL-UNNAMED
+--add-exports java.base/jdk.internal.ref=ALL-UNNAMED
+--add-exports java.base/sun.nio.ch=ALL-UNNAMED
+--add-exports java.management.rmi/com.sun.jmx.remote.internal.rmi=ALL-UNNAMED
+--add-exports java.rmi/sun.rmi.registry=ALL-UNNAMED
+--add-exports java.rmi/sun.rmi.server=ALL-UNNAMED
+--add-exports java.sql/java.sql=ALL-UNNAMED
+--add-exports jdk.attach/sun.tools.attach=ALL-UNNAMED
+
+--add-opens java.base/java.io=ALL-UNNAMED
+--add-opens java.base/java.lang.module=ALL-UNNAMED
+--add-opens java.base/java.lang.reflect=ALL-UNNAMED
+--add-opens java.base/jdk.internal.loader=ALL-UNNAMED
+--add-opens java.base/jdk.internal.math=ALL-UNNAMED
+--add-opens java.base/jdk.internal.module=ALL-UNNAMED
+--add-opens java.base/jdk.internal.ref=ALL-UNNAMED
+--add-opens java.base/jdk.internal.reflect=ALL-UNNAMED
+--add-opens java.base/sun.nio.ch=ALL-UNNAMED
+--add-opens jdk.management/com.sun.management.internal=ALL-UNNAMED
+
+# The newline in the end of file is intentional
diff --git a/conf/jvm22-server.options b/conf/jvm22-server.options
new file mode 100644
index 000000000000..b836204660bb
--- /dev/null
+++ b/conf/jvm22-server.options
@@ -0,0 +1,128 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+###########################################################################
+# jvm22-server.options #
+# #
+# See jvm-server.options. This file is specific for Java 22 and newer. #
+###########################################################################
+
+#################
+# GC SETTINGS #
+#################
+
+### G1 Settings
+## Use the Hotspot garbage-first collector.
+-XX:+UseG1GC
+-XX:+ParallelRefProcEnabled
+
+#
+## Have the JVM do less remembered set work during STW, instead
+## preferring concurrent GC. Reduces p99.9 latency.
+-XX:G1RSetUpdatingPauseTimePercent=5
+#
+## Main G1GC tunable: lowering the pause target will lower throughput and vise versa.
+## 200ms is the JVM default and lowest viable setting
+## 1000ms increases throughput. Keep it smaller than the timeouts in cassandra.yaml.
+-XX:MaxGCPauseMillis=500
+
+## Optional G1 Settings
+# Save CPU time on large (>= 16GB) heaps by delaying region scanning
+# until the heap is 70% full. The default in Hotspot 8u40 is 40%.
+#-XX:InitiatingHeapOccupancyPercent=70
+
+# For systems with > 8 cores, the default ParallelGCThreads is 5/8 the number of logical cores.
+# Otherwise equal to the number of cores when 8 or less.
+# Machines with > 10 cores should try setting these to <= full cores.
+#-XX:ParallelGCThreads=16
+# By default, ConcGCThreads is 1/4 of ParallelGCThreads.
+# Setting both to the same value can reduce STW durations.
+#-XX:ConcGCThreads=16
+
+
+### JPMS
+
+-Djdk.attach.allowAttachSelf=true
+-Djava.security.manager=allow
+--add-exports java.base/jdk.internal.misc=ALL-UNNAMED
+--add-exports java.base/jdk.internal.ref=ALL-UNNAMED
+--add-exports java.base/jdk.internal.perf=ALL-UNNAMED
+--add-exports java.base/sun.nio.ch=ALL-UNNAMED
+--add-exports java.management.rmi/com.sun.jmx.remote.internal.rmi=ALL-UNNAMED
+--add-exports java.rmi/sun.rmi.registry=ALL-UNNAMED
+--add-exports java.rmi/sun.rmi.server=ALL-UNNAMED
+--add-exports jdk.compiler/com.sun.tools.javac.file=ALL-UNNAMED
+--add-exports jdk.naming.dns/com.sun.jndi.dns=java.naming
+--add-exports jdk.unsupported/sun.misc=ALL-UNNAMED
+
+--add-opens java.base/java.io=ALL-UNNAMED
+--add-opens java.base/java.lang.module=ALL-UNNAMED
+--add-opens java.base/java.lang=ALL-UNNAMED
+--add-opens java.base/java.lang.reflect=ALL-UNNAMED
+--add-opens java.base/java.nio.charset=ALL-UNNAMED
+--add-opens java.base/java.nio.file.spi=ALL-UNNAMED
+--add-opens java.base/java.nio=ALL-UNNAMED
+--add-opens java.base/java.net=ALL-UNNAMED
+--add-opens java.base/java.util=ALL-UNNAMED
+--add-opens java.base/java.util.concurrent.atomic=ALL-UNNAMED
+--add-opens java.base/java.util.concurrent.locks=ALL-UNNAMED
+--add-opens java.base/jdk.internal.loader=ALL-UNNAMED
+--add-opens java.base/jdk.internal.math=ALL-UNNAMED
+--add-opens java.base/jdk.internal.module=ALL-UNNAMED
+--add-opens java.base/jdk.internal.ref=ALL-UNNAMED
+--add-opens java.base/jdk.internal.reflect=ALL-UNNAMED
+--add-opens java.base/jdk.internal.vm=ALL-UNNAMED
+--add-opens java.base/sun.nio.ch=ALL-UNNAMED
+--add-opens jdk.compiler/com.sun.tools.javac=ALL-UNNAMED
+--add-opens jdk.management.jfr/jdk.management.jfr=ALL-UNNAMED
+--add-opens jdk.management/com.sun.management.internal=ALL-UNNAMED
+--add-opens jdk.naming.dns/com.sun.jndi.dns=ALL-UNNAMED
+
+# required for org.apache.cassandra.Util.getSupportedMTimeGranularity
+--add-opens java.base/java.nio.file.attribute=ALL-UNNAMED
+
+### GC logging options -- uncomment to enable
+
+# Java 11 (and newer) GC logging options:
+# See description of https://bugs.openjdk.java.net/browse/JDK-8046148 for details about the syntax
+# The following is the equivalent to -XX:+PrintGCDetails -XX:+UseGCLogFileRotation -XX:NumberOfGCLogFiles=10 -XX:GCLogFileSize=10M
+# -Xlog:gc=info,heap*=debug,age*=debug,safepoint=info,promotion*=debug:file=/var/log/cassandra/gc.log:time,uptime,pid,tid,level:filecount=10,filesize=10485760
+
+# Notes for Java 8 migration:
+#
+# -XX:+PrintGCDetails maps to -Xlog:gc*:... - i.e. add a '*' after "gc"
+# -XX:+PrintGCDateStamps maps to decorator 'time'
+#
+# -XX:+PrintHeapAtGC maps to 'heap' with level 'trace'
+# -XX:+PrintTenuringDistribution maps to 'age' with level 'debug'
+# -XX:+PrintGCApplicationStoppedTime maps to 'safepoint' with level 'info'
+# -XX:+PrintPromotionFailure maps to 'promotion' with level 'trace'
+# -XX:PrintFLSStatistics=1 maps to 'freelist' with level 'trace'
+
+### Netty Options
+
+# On Java >= 9 Netty requires the io.netty.tryReflectionSetAccessible system property to be set to true to enable
+# creation of direct buffers using Unsafe. Without it, this falls back to ByteBuffer.allocateDirect which has
+# inferior performance and risks exceeding MaxDirectMemory
+-Dio.netty.tryReflectionSetAccessible=true
+
+### Enable vector incubator feature (simd support)
+
+--add-modules jdk.incubator.vector
+
+# The newline in the end of file is intentional
diff --git a/debian/changelog b/debian/changelog
index 432841232af9..1f04f7a05b7a 100644
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,15 @@
+cassandra (5.0.9) unstable; urgency=medium
+
+ * New release
+
+ -- Stefan Miklosovic Tue, 28 Jul 2026 12:57:11 +0200
+
+cassandra (5.0.8) unstable; urgency=medium
+
+ * New release
+
+ -- Stefan Miklosovic Fri, 10 Apr 2026 11:33:31 +0200
+
cassandra (5.0.7) unstable; urgency=medium
* New release
diff --git a/doc/Makefile b/doc/Makefile
index dac8e11fb06f..b1722037f5f6 100644
--- a/doc/Makefile
+++ b/doc/Makefile
@@ -11,9 +11,10 @@
# See the License for the specific language governing permissions and
# limitations under the License.
+GENERATE_ANTORA_YML = ./scripts/gen-antora-yml.py
GENERATE_NODETOOL_DOCS = ./scripts/gen-nodetool-docs.py
MAKE_CASSANDRA_YAML = ./scripts/convert_yaml_to_adoc.py ../conf/cassandra.yaml ./modules/cassandra/pages/managing/configuration/cass_yaml_file.adoc
-PROCESS_NATIVE_PROC_SPECS = ./scripts/process-native-protocol-specs-in-docker.sh
+GEN_NATIVE_PROTOCOL_DOCS = ./scripts/gen-native-protocol-docs.sh
.PHONY: html
html:
@@ -21,8 +22,9 @@ html:
.PHONY: gen-asciidoc
gen-asciidoc:
+ python3 $(GENERATE_ANTORA_YML)
@mkdir -p modules/cassandra/pages/managing/tools/nodetool
@mkdir -p modules/cassandra/examples/TEXT/NODETOOL
python3 $(GENERATE_NODETOOL_DOCS)
python3 $(MAKE_CASSANDRA_YAML)
- $(PROCESS_NATIVE_PROC_SPECS)
+ $(GEN_NATIVE_PROTOCOL_DOCS)
diff --git a/doc/README.md b/doc/README.md
index 608d236cb75b..0d534a52b97a 100644
--- a/doc/README.md
+++ b/doc/README.md
@@ -36,7 +36,9 @@ The source for the official documentation for Apache Cassandra can be found in
the `modules/cassandra/pages` subdirectory. The documentation uses [antora](http://www.antora.org/)
and is thus written in [asciidoc](http://asciidoc.org).
-To generate the asciidoc files for cassandra.yaml and the nodetool commands, run (from project root):
+The `antora.yml` file is auto-generated and should not be manually edited. It is generated from the version in `build.xml` using `scripts/gen-antora-yml.py` and automatically detects whether building from a release tag or branch HEAD to set the appropriate version.
+
+To generate the asciidoc files (including antora.yml) for cassandra.yaml and the nodetool commands, run (from project root):
```bash
ant gen-asciidoc
```
diff --git a/doc/SASI.md b/doc/SASI.md
deleted file mode 100644
index fc38845ce2cd..000000000000
--- a/doc/SASI.md
+++ /dev/null
@@ -1,818 +0,0 @@
-
-
-# SASIIndex
-
-[`SASIIndex`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/SASIIndex.java),
-or "SASI" for short, is an implementation of Cassandra's
-`Index` interface that can be used as an alternative to the
-existing implementations. SASI's indexing and querying improves on
-existing implementations by tailoring it specifically to Cassandra's
-needs. SASI has superior performance in cases where queries would
-previously require filtering. In achieving this performance, SASI aims
-to be significantly less resource intensive than existing
-implementations, in memory, disk, and CPU usage. In addition, SASI
-supports prefix and contains queries on strings (similar to SQL's
-`LIKE = "foo*"` or `LIKE = "*foo*"'`).
-
-The following goes on describe how to get up and running with SASI,
-demonstrates usage with examples, and provides some details on its
-implementation.
-
-## Using SASI
-
-The examples below walk through creating a table and indexes on its
-columns, and performing queries on some inserted data.
-
-The examples below assume the `demo` keyspace has been created and is
-in use.
-
-```
-cqlsh> CREATE KEYSPACE demo WITH replication = {
- ... 'class': 'SimpleStrategy',
- ... 'replication_factor': '1'
- ... };
-cqlsh> USE demo;
-```
-
-All examples are performed on the `sasi` table:
-
-```
-cqlsh:demo> CREATE TABLE sasi (id uuid, first_name text, last_name text,
- ... age int, height int, created_at bigint, primary key (id));
-```
-
-#### Creating Indexes
-
-To create SASI indexes use CQLs `CREATE CUSTOM INDEX` statement:
-
-```
-cqlsh:demo> CREATE CUSTOM INDEX ON sasi (first_name) USING 'org.apache.cassandra.index.sasi.SASIIndex'
- ... WITH OPTIONS = {
- ... 'analyzer_class':
- ... 'org.apache.cassandra.index.sasi.analyzer.NonTokenizingAnalyzer',
- ... 'case_sensitive': 'false'
- ... };
-
-cqlsh:demo> CREATE CUSTOM INDEX ON sasi (last_name) USING 'org.apache.cassandra.index.sasi.SASIIndex'
- ... WITH OPTIONS = {'mode': 'CONTAINS'};
-
-cqlsh:demo> CREATE CUSTOM INDEX ON sasi (age) USING 'org.apache.cassandra.index.sasi.SASIIndex';
-
-cqlsh:demo> CREATE CUSTOM INDEX ON sasi (created_at) USING 'org.apache.cassandra.index.sasi.SASIIndex'
- ... WITH OPTIONS = {'mode': 'SPARSE'};
-```
-
-The indexes created have some options specified that customize their
-behaviour and potentially performance. The index on `first_name` is
-case-insensitive. The analyzers are discussed more in a subsequent
-example. The `NonTokenizingAnalyzer` performs no analysis on the
-text. Each index has a mode: `PREFIX`, `CONTAINS`, or `SPARSE`, the
-first being the default. The `last_name` index is created with the
-mode `CONTAINS` which matches terms on suffixes instead of prefix
-only. Examples of this are available below and more detail can be
-found in the section on
-[OnDiskIndex](#ondiskindexbuilder).The
-`created_at` column is created with its mode set to `SPARSE`, which is
-meant to improve performance of querying large, dense number ranges
-like timestamps for data inserted every millisecond. Details of the
-`SPARSE` implementation can also be found in the section on the
-[OnDiskIndex](#ondiskindexbuilder). The `age`
-index is created with the default `PREFIX` mode and no
-case-sensitivity or text analysis options are specified since the
-field is numeric.
-
-After inserting the following data and performing a `nodetool flush`,
-SASI performing index flushes to disk can be seen in Cassandra's logs
--- although the direct call to flush is not required (see
-[IndexMemtable](#indexmemtable) for more details).
-
-```
-cqlsh:demo> INSERT INTO sasi (id, first_name, last_name, age, height, created_at)
- ... VALUES (556ebd54-cbe5-4b75-9aae-bf2a31a24500, 'Pavel', 'Yaskevich', 27, 181, 1442959315018);
-
-cqlsh:demo> INSERT INTO sasi (id, first_name, last_name, age, height, created_at)
- ... VALUES (5770382a-c56f-4f3f-b755-450e24d55217, 'Jordan', 'West', 26, 173, 1442959315019);
-
-cqlsh:demo> INSERT INTO sasi (id, first_name, last_name, age, height, created_at)
- ... VALUES (96053844-45c3-4f15-b1b7-b02c441d3ee1, 'Mikhail', 'Stepura', 36, 173, 1442959315020);
-
-cqlsh:demo> INSERT INTO sasi (id, first_name, last_name, age, height, created_at)
- ... VALUES (f5dfcabe-de96-4148-9b80-a1c41ed276b4, 'Michael', 'Kjellman', 26, 180, 1442959315021);
-
-cqlsh:demo> INSERT INTO sasi (id, first_name, last_name, age, height, created_at)
- ... VALUES (2970da43-e070-41a8-8bcb-35df7a0e608a, 'Johnny', 'Zhang', 32, 175, 1442959315022);
-
-cqlsh:demo> INSERT INTO sasi (id, first_name, last_name, age, height, created_at)
- ... VALUES (6b757016-631d-4fdb-ac62-40b127ccfbc7, 'Jason', 'Brown', 40, 182, 1442959315023);
-
-cqlsh:demo> INSERT INTO sasi (id, first_name, last_name, age, height, created_at)
- ... VALUES (8f909e8a-008e-49dd-8d43-1b0df348ed44, 'Vijay', 'Parthasarathy', 34, 183, 1442959315024);
-
-cqlsh:demo> SELECT first_name, last_name, age, height, created_at FROM sasi;
-
- first_name | last_name | age | height | created_at
-------------+---------------+-----+--------+---------------
- Michael | Kjellman | 26 | 180 | 1442959315021
- Mikhail | Stepura | 36 | 173 | 1442959315020
- Jason | Brown | 40 | 182 | 1442959315023
- Pavel | Yaskevich | 27 | 181 | 1442959315018
- Vijay | Parthasarathy | 34 | 183 | 1442959315024
- Jordan | West | 26 | 173 | 1442959315019
- Johnny | Zhang | 32 | 175 | 1442959315022
-
-(7 rows)
-```
-
-#### Equality & Prefix Queries
-
-SASI supports all queries already supported by CQL, including LIKE statement
-for PREFIX, CONTAINS and SUFFIX searches.
-
-```
-cqlsh:demo> SELECT first_name, last_name, age, height, created_at FROM sasi
- ... WHERE first_name = 'Pavel';
-
- first_name | last_name | age | height | created_at
--------------+-----------+-----+--------+---------------
- Pavel | Yaskevich | 27 | 181 | 1442959315018
-
-(1 rows)
-```
-
-```
-cqlsh:demo> SELECT first_name, last_name, age, height, created_at FROM sasi
- ... WHERE first_name = 'pavel';
-
- first_name | last_name | age | height | created_at
--------------+-----------+-----+--------+---------------
- Pavel | Yaskevich | 27 | 181 | 1442959315018
-
-(1 rows)
-```
-
-```
-cqlsh:demo> SELECT first_name, last_name, age, height, created_at FROM sasi
- ... WHERE first_name LIKE 'M%';
-
- first_name | last_name | age | height | created_at
-------------+-----------+-----+--------+---------------
- Michael | Kjellman | 26 | 180 | 1442959315021
- Mikhail | Stepura | 36 | 173 | 1442959315020
-
-(2 rows)
-```
-
-Of course, the case of the query does not matter for the `first_name`
-column because of the options provided at index creation time.
-
-```
-cqlsh:demo> SELECT first_name, last_name, age, height, created_at FROM sasi
- ... WHERE first_name LIKE 'm%';
-
- first_name | last_name | age | height | created_at
-------------+-----------+-----+--------+---------------
- Michael | Kjellman | 26 | 180 | 1442959315021
- Mikhail | Stepura | 36 | 173 | 1442959315020
-
-(2 rows)
-```
-
-#### Compound Queries
-
-SASI supports queries with multiple predicates, however, due to the
-nature of the default indexing implementation, CQL requires the user
-to specify `ALLOW FILTERING` to opt-in to the potential performance
-pitfalls of such a query. With SASI, while the requirement to include
-`ALLOW FILTERING` remains, to reduce modifications to the grammar, the
-performance pitfalls do not exist because filtering is not
-performed. Details on how SASI joins data from multiple predicates is
-available below in the
-[Implementation Details](#implementation-details)
-section.
-
-```
-cqlsh:demo> SELECT first_name, last_name, age, height, created_at FROM sasi
- ... WHERE first_name LIKE 'M%' and age < 30 ALLOW FILTERING;
-
- first_name | last_name | age | height | created_at
-------------+-----------+-----+--------+---------------
- Michael | Kjellman | 26 | 180 | 1442959315021
-
-(1 rows)
-```
-
-#### Suffix Queries
-
-The next example demonstrates `CONTAINS` mode on the `last_name`
-column. By using this mode, predicates can search for any strings
-containing the search string as a sub-string. In this case the strings
-containing "a" or "an".
-
-```
-cqlsh:demo> SELECT * FROM sasi WHERE last_name LIKE '%a%';
-
- id | age | created_at | first_name | height | last_name
---------------------------------------+-----+---------------+------------+--------+---------------
- f5dfcabe-de96-4148-9b80-a1c41ed276b4 | 26 | 1442959315021 | Michael | 180 | Kjellman
- 96053844-45c3-4f15-b1b7-b02c441d3ee1 | 36 | 1442959315020 | Mikhail | 173 | Stepura
- 556ebd54-cbe5-4b75-9aae-bf2a31a24500 | 27 | 1442959315018 | Pavel | 181 | Yaskevich
- 8f909e8a-008e-49dd-8d43-1b0df348ed44 | 34 | 1442959315024 | Vijay | 183 | Parthasarathy
- 2970da43-e070-41a8-8bcb-35df7a0e608a | 32 | 1442959315022 | Johnny | 175 | Zhang
-
-(5 rows)
-
-cqlsh:demo> SELECT * FROM sasi WHERE last_name LIKE '%an%';
-
- id | age | created_at | first_name | height | last_name
---------------------------------------+-----+---------------+------------+--------+-----------
- f5dfcabe-de96-4148-9b80-a1c41ed276b4 | 26 | 1442959315021 | Michael | 180 | Kjellman
- 2970da43-e070-41a8-8bcb-35df7a0e608a | 32 | 1442959315022 | Johnny | 175 | Zhang
-
-(2 rows)
-```
-
-#### Expressions on Non-Indexed Columns
-
-SASI also supports filtering on non-indexed columns like `height`. The
-expression can only narrow down an existing query using `AND`.
-
-```
-cqlsh:demo> SELECT * FROM sasi WHERE last_name LIKE '%a%' AND height >= 175 ALLOW FILTERING;
-
- id | age | created_at | first_name | height | last_name
---------------------------------------+-----+---------------+------------+--------+---------------
- f5dfcabe-de96-4148-9b80-a1c41ed276b4 | 26 | 1442959315021 | Michael | 180 | Kjellman
- 556ebd54-cbe5-4b75-9aae-bf2a31a24500 | 27 | 1442959315018 | Pavel | 181 | Yaskevich
- 8f909e8a-008e-49dd-8d43-1b0df348ed44 | 34 | 1442959315024 | Vijay | 183 | Parthasarathy
- 2970da43-e070-41a8-8bcb-35df7a0e608a | 32 | 1442959315022 | Johnny | 175 | Zhang
-
-(4 rows)
-```
-
-#### Delimiter based Tokenization Analysis
-
-A simple text analysis provided is delimiter based tokenization. This provides an alternative to indexing collections,
-as delimiter separated text can be indexed without the overhead of `CONTAINS` mode nor using `PREFIX` or `SUFFIX` queries.
-
-```
-cqlsh:demo> ALTER TABLE sasi ADD aliases text;
-cqlsh:demo> CREATE CUSTOM INDEX on sasi (aliases) USING 'org.apache.cassandra.index.sasi.SASIIndex'
- ... WITH OPTIONS = {
- ... 'analyzer_class': 'org.apache.cassandra.index.sasi.analyzer.DelimiterAnalyzer',
- ... 'delimiter': ',',
- ... 'mode': 'prefix',
- ... 'analyzed': 'true'};
-cqlsh:demo> UPDATE sasi SET aliases = 'Mike,Mick,Mikey,Mickey' WHERE id = f5dfcabe-de96-4148-9b80-a1c41ed276b4;
-cqlsh:demo> SELECT * FROM sasi WHERE aliases LIKE 'Mikey' ALLOW FILTERING;
-
- id | age | aliases | created_at | first_name | height | last_name
---------------------------------------+-----+------------------------+---------------+------------+--------+-----------
- f5dfcabe-de96-4148-9b80-a1c41ed276b4 | 26 | Mike,Mick,Mikey,Mickey | 1442959315021 | Michael | 180 | Kjellman
-```
-
-#### Text Analysis (Tokenization and Stemming)
-
-Lastly, to demonstrate text analysis an additional column is needed on
-the table. Its definition, index, and statements to update rows are shown below.
-
-```
-cqlsh:demo> ALTER TABLE sasi ADD bio text;
-cqlsh:demo> CREATE CUSTOM INDEX ON sasi (bio) USING 'org.apache.cassandra.index.sasi.SASIIndex'
- ... WITH OPTIONS = {
- ... 'analyzer_class': 'org.apache.cassandra.index.sasi.analyzer.StandardAnalyzer',
- ... 'tokenization_enable_stemming': 'true',
- ... 'analyzed': 'true',
- ... 'tokenization_normalize_lowercase': 'true',
- ... 'tokenization_locale': 'en'
- ... };
-cqlsh:demo> UPDATE sasi SET bio = 'Software Engineer, who likes distributed systems, doesnt like to argue.' WHERE id = 5770382a-c56f-4f3f-b755-450e24d55217;
-cqlsh:demo> UPDATE sasi SET bio = 'Software Engineer, works on the freight distribution at nights and likes arguing' WHERE id = 556ebd54-cbe5-4b75-9aae-bf2a31a24500;
-cqlsh:demo> SELECT * FROM sasi;
-
- id | age | bio | created_at | first_name | height | last_name
---------------------------------------+-----+----------------------------------------------------------------------------------+---------------+------------+--------+---------------
- f5dfcabe-de96-4148-9b80-a1c41ed276b4 | 26 | null | 1442959315021 | Michael | 180 | Kjellman
- 96053844-45c3-4f15-b1b7-b02c441d3ee1 | 36 | null | 1442959315020 | Mikhail | 173 | Stepura
- 6b757016-631d-4fdb-ac62-40b127ccfbc7 | 40 | null | 1442959315023 | Jason | 182 | Brown
- 556ebd54-cbe5-4b75-9aae-bf2a31a24500 | 27 | Software Engineer, works on the freight distribution at nights and likes arguing | 1442959315018 | Pavel | 181 | Yaskevich
- 8f909e8a-008e-49dd-8d43-1b0df348ed44 | 34 | null | 1442959315024 | Vijay | 183 | Parthasarathy
- 5770382a-c56f-4f3f-b755-450e24d55217 | 26 | Software Engineer, who likes distributed systems, doesnt like to argue. | 1442959315019 | Jordan | 173 | West
- 2970da43-e070-41a8-8bcb-35df7a0e608a | 32 | null | 1442959315022 | Johnny | 175 | Zhang
-
-(7 rows)
-```
-
-Index terms and query search strings are stemmed for the `bio` column
-because it was configured to use the
-[`StandardAnalyzer`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/analyzer/StandardAnalyzer.java)
-and `analyzed` is set to `true`. The
-`tokenization_normalize_lowercase` is similar to the `case_sensitive`
-property but for the
-[`StandardAnalyzer`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/analyzer/StandardAnalyzer.java). These
-query demonstrates the stemming applied by [`StandardAnalyzer`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/analyzer/StandardAnalyzer.java).
-
-```
-cqlsh:demo> SELECT * FROM sasi WHERE bio LIKE 'distributing';
-
- id | age | bio | created_at | first_name | height | last_name
---------------------------------------+-----+----------------------------------------------------------------------------------+---------------+------------+--------+-----------
- 556ebd54-cbe5-4b75-9aae-bf2a31a24500 | 27 | Software Engineer, works on the freight distribution at nights and likes arguing | 1442959315018 | Pavel | 181 | Yaskevich
- 5770382a-c56f-4f3f-b755-450e24d55217 | 26 | Software Engineer, who likes distributed systems, doesnt like to argue. | 1442959315019 | Jordan | 173 | West
-
-(2 rows)
-
-cqlsh:demo> SELECT * FROM sasi WHERE bio LIKE 'they argued';
-
- id | age | bio | created_at | first_name | height | last_name
---------------------------------------+-----+----------------------------------------------------------------------------------+---------------+------------+--------+-----------
- 556ebd54-cbe5-4b75-9aae-bf2a31a24500 | 27 | Software Engineer, works on the freight distribution at nights and likes arguing | 1442959315018 | Pavel | 181 | Yaskevich
- 5770382a-c56f-4f3f-b755-450e24d55217 | 26 | Software Engineer, who likes distributed systems, doesnt like to argue. | 1442959315019 | Jordan | 173 | West
-
-(2 rows)
-
-cqlsh:demo> SELECT * FROM sasi WHERE bio LIKE 'working at the company';
-
- id | age | bio | created_at | first_name | height | last_name
---------------------------------------+-----+----------------------------------------------------------------------------------+---------------+------------+--------+-----------
- 556ebd54-cbe5-4b75-9aae-bf2a31a24500 | 27 | Software Engineer, works on the freight distribution at nights and likes arguing | 1442959315018 | Pavel | 181 | Yaskevich
-
-(1 rows)
-
-cqlsh:demo> SELECT * FROM sasi WHERE bio LIKE 'soft eng';
-
- id | age | bio | created_at | first_name | height | last_name
---------------------------------------+-----+----------------------------------------------------------------------------------+---------------+------------+--------+-----------
- 556ebd54-cbe5-4b75-9aae-bf2a31a24500 | 27 | Software Engineer, works on the freight distribution at nights and likes arguing | 1442959315018 | Pavel | 181 | Yaskevich
- 5770382a-c56f-4f3f-b755-450e24d55217 | 26 | Software Engineer, who likes distributed systems, doesnt like to argue. | 1442959315019 | Jordan | 173 | West
-
-(2 rows)
-```
-
-## Implementation Details
-
-While SASI, at the surface, is simply an implementation of the
-`Index` interface, at its core there are several data
-structures and algorithms used to satisfy it. These are described
-here. Additionally, the changes internal to Cassandra to support SASI's
-integration are described.
-
-The `Index` interface divides responsibility of the
-implementer into two parts: Indexing and Querying. Further, Cassandra
-makes it possible to divide those responsibilities into the memory and
-disk components. SASI takes advantage of Cassandra's write-once,
-immutable, ordered data model to build indexes along with the flushing
-of the memtable to disk -- this is the origin of the name "SSTable
-Attached Secondary Index".
-
-The SASI index data structures are built in memory as the SSTable is
-being written and they are flushed to disk before the writing of the
-SSTable completes. The writing of each index file only requires
-sequential writes to disk. In some cases, partial flushes are
-performed, and later stitched back together, to reduce memory
-usage. These data structures are optimized for this use case.
-
-Taking advantage of Cassandra's ordered data model, at query time,
-candidate indexes are narrowed down for searching, minimizing the amount
-of work done. Searching is then performed using an efficient method
-that streams data off disk as needed.
-
-### Indexing
-
-Per SSTable, SASI writes an index file for each indexed column. The
-data for these files is built in memory using the
-[`OnDiskIndexBuilder`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/disk/OnDiskIndexBuilder.java). Once
-flushed to disk, the data is read using the
-[`OnDiskIndex`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/disk/OnDiskIndex.java)
-class. These are composed of bytes representing indexed terms,
-organized for efficient writing or searching respectively. The keys
-and values they hold represent tokens and positions in an SSTable and
-these are stored per-indexed term in
-[`TokenTreeBuilder`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/disk/TokenTreeBuilder.java)s
-for writing, and
-[`TokenTree`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/disk/TokenTree.java)s
-for querying. These index files are memory mapped after being written
-to disk, for quicker access. For indexing data in the memtable, SASI
-uses its
-[`IndexMemtable`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/memory/IndexMemtable.java)
-class.
-
-#### OnDiskIndex(Builder)
-
-Each
-[`OnDiskIndex`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/disk/OnDiskIndex.java)
-is an instance of a modified
-[Suffix Array](https://en.wikipedia.org/wiki/Suffix_array) data
-structure. The
-[`OnDiskIndex`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/disk/OnDiskIndex.java)
-is comprised of page-size blocks of sorted terms and pointers to the
-terms' associated data, as well as the data itself, stored also in one
-or more page-sized blocks. The
-[`OnDiskIndex`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/disk/OnDiskIndex.java)
-is structured as a tree of arrays, where each level describes the
-terms in the level below, the final level being the terms
-themselves. The `PointerLevel`s and their `PointerBlock`s contain
-terms and pointers to other blocks that *end* with those terms. The
-`DataLevel`, the final level, and its `DataBlock`s contain terms and
-point to the data itself, contained in [`TokenTree`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/disk/TokenTree.java)s.
-
-The terms written to the
-[`OnDiskIndex`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/disk/OnDiskIndex.java)
-vary depending on its "mode": either `PREFIX`, `CONTAINS`, or
-`SPARSE`. In the `PREFIX` and `SPARSE` cases, terms' exact values are
-written exactly once per `OnDiskIndex`. For example, when using a `PREFIX` index
-with terms `Jason`, `Jordan`, `Pavel`, all three will be included in
-the index. A `CONTAINS` index writes additional terms for each suffix of
-each term recursively. Continuing with the example, a `CONTAINS` index
-storing the previous terms would also store `ason`, `ordan`, `avel`,
-`son`, `rdan`, `vel`, etc. This allows for queries on the suffix of
-strings. The `SPARSE` mode differs from `PREFIX` in that for every 64
-blocks of terms a
-[`TokenTree`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/disk/TokenTree.java)
-is built merging all the
-[`TokenTree`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/disk/TokenTree.java)s
-for each term into a single one. This copy of the data is used for
-efficient iteration of large ranges of e.g. timestamps. The index
-"mode" is configurable per column at index creation time.
-
-#### TokenTree(Builder)
-
-The
-[`TokenTree`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/disk/TokenTree.java)
-is an implementation of the well-known
-[B+-tree](https://en.wikipedia.org/wiki/B%2B_tree) that has been
-modified to optimize for its use-case. In particular, it has been
-optimized to associate tokens, longs, with a set of positions in an
-SSTable, also longs. Allowing the set of long values accommodates
-the possibility of a hash collision in the token, but the data
-structure is optimized for the unlikely possibility of such a
-collision.
-
-To optimize for its write-once environment the
-[`TokenTreeBuilder`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/disk/TokenTreeBuilder.java)
-completely loads its interior nodes as the tree is built and it uses
-the well-known algorithm optimized for bulk-loading the data
-structure.
-
-[`TokenTree`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/disk/TokenTree.java)s provide the means to iterate over tokens, and file
-positions, that match a given term, and to skip forward in that
-iteration, an operation used heavily at query time.
-
-#### IndexMemtable
-
-The
-[`IndexMemtable`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/memory/IndexMemtable.java)
-handles indexing the in-memory data held in the memtable. The
-[`IndexMemtable`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/memory/IndexMemtable.java)
-in turn manages either a
-[`TrieMemIndex`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/memory/TrieMemIndex.java)
-or a
-[`SkipListMemIndex`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/memory/SkipListMemIndex.java)
-per-column. The choice of which index type is used is data
-dependent. The
-[`TrieMemIndex`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/memory/TrieMemIndex.java)
-is used for literal types. `AsciiType` and `UTF8Type` are literal
-types by default but any column can be configured as a literal type
-using the `is_literal` option at index creation time. For non-literal
-types the
-[`SkipListMemIndex`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/memory/SkipListMemIndex.java)
-is used. The
-[`TrieMemIndex`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/memory/TrieMemIndex.java)
-is an implementation that can efficiently support prefix queries on
-character-like data. The
-[`SkipListMemIndex`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/memory/SkipListMemIndex.java),
-conversely, is better suited for other Cassandra data types like
-numbers.
-
-The
-[`TrieMemIndex`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/memory/TrieMemIndex.java)
-is built using either the `ConcurrentRadixTree` or
-`ConcurrentSuffixTree` from the `com.goooglecode.concurrenttrees`
-package. The choice between the two is made based on the indexing
-mode, `PREFIX` or other modes, and `CONTAINS` mode, respectively.
-
-The
-[`SkipListMemIndex`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/memory/SkipListMemIndex.java)
-is built on top of `java.util.concurrent.ConcurrentSkipListSet`.
-
-### Querying
-
-Responsible for converting the internal `IndexExpression`
-representation into SASI's
-[`Operation`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/plan/Operation.java)
-and
-[`Expression`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/plan/Expression.java)
-trees, optimizing the trees to reduce the amount of work done, and
-driving the query itself, the
-[`QueryPlan`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/plan/QueryPlan.java)
-is the work horse of SASI's querying implementation. To efficiently
-perform union and intersection operations, SASI provides several
-iterators similar to Cassandra's `MergeIterator`, but tailored
-specifically for SASI's use while including more features. The
-[`RangeUnionIterator`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/utils/RangeUnionIterator.java),
-like its name suggests, performs set unions over sets of tokens/keys
-matching the query, only reading as much data as it needs from each
-set to satisfy the query. The
-[`RangeIntersectionIterator`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/utils/RangeIntersectionIterator.java),
-similar to its counterpart, performs set intersections over its data.
-
-#### QueryPlan
-
-The
-[`QueryPlan`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/plan/QueryPlan.java)
-instantiated per search query is at the core of SASI's querying
-implementation. Its work can be divided in two stages: analysis and
-execution.
-
-During the analysis phase,
-[`QueryPlan`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/plan/QueryPlan.java)
-converts from Cassandra's internal representation of
-`IndexExpression`s, which has also been modified to support encoding
-queries that contain ORs and groupings of expressions using
-parentheses (see the
-[Cassandra Internal Changes](#cassandra-internal-changes)
-section below for more details). This process produces a tree of
-[`Operation`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/plan/Operation.java)s, which in turn may contain [`Expression`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/plan/Expression.java)s, all of which
-provide an alternative, more efficient, representation of the query.
-
-During execution, the
-[`QueryPlan`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/plan/QueryPlan.java)
-uses the `DecoratedKey`-generating iterator created from the
-[`Operation`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/plan/Operation.java) tree. These keys are read from disk and a final check to
-ensure they satisfy the query is made, once again using the
-[`Operation`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/plan/Operation.java) tree. At the point the desired amount of matching data has
-been found, or there is no more matching data, the result set is
-returned to the coordinator through the existing internal components.
-
-The number of queries (total/failed/timed-out), and their latencies,
-are maintined per-table/column family.
-
-SASI also supports concurrently iterating terms for the same index
-across SSTables. The concurrency factor is controlled by the
-`cassandra.search_concurrency_factor` system property. The default is
-`1`.
-
-##### QueryController
-
-Each
-[`QueryPlan`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/plan/QueryPlan.java)
-references a
-[`QueryController`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/plan/QueryController.java)
-used throughout the execution phase. The
-[`QueryController`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/plan/QueryController.java)
-has two responsibilities: to manage and ensure the proper cleanup of
-resources (indexes), and to strictly enforce the time bound per query,
-specified by the user via the range slice timeout. All indexes are
-accessed via the
-[`QueryController`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/plan/QueryController.java)
-so that they can be safely released by it later. The
-[`QueryController`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/plan/QueryController.java)'s
-`checkpoint` function is called in specific places in the execution
-path to ensure the time-bound is enforced.
-
-##### QueryPlan Optimizations
-
-While in the analysis phase, the
-[`QueryPlan`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/plan/QueryPlan.java)
-performs several potential optimizations to the query. The goal of
-these optimizations is to reduce the amount of work performed during
-the execution phase.
-
-The simplest optimization performed is compacting multiple expressions
-joined by logical intersections (`AND`) into a single [`Operation`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/plan/Operation.java) with
-three or more [`Expression`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/plan/Expression.java)s. For example, the query `WHERE age < 100 AND
-fname = 'p*' AND first_name != 'pa*' AND age > 21` would,
-without modification, have the following tree:
-
- ┌───────┐
- ┌────────│ AND │──────┐
- │ └───────┘ │
- ▼ ▼
- ┌───────┐ ┌──────────┐
- ┌─────│ AND │─────┐ │age < 100 │
- │ └───────┘ │ └──────────┘
- ▼ ▼
- ┌──────────┐ ┌───────┐
- │ fname=p* │ ┌─│ AND │───┐
- └──────────┘ │ └───────┘ │
- ▼ ▼
- ┌──────────┐ ┌──────────┐
- │fname!=pa*│ │ age > 21 │
- └──────────┘ └──────────┘
-
-[`QueryPlan`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/plan/QueryPlan.java)
-will remove the redundant right branch whose root is the final `AND`
-and has leaves `fname != pa*` and `age > 21`. These [`Expression`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/plan/Expression.java)s will
-be compacted into the parent `AND`, a safe operation due to `AND`
-being associative and commutative. The resulting tree looks like the
-following:
-
- ┌───────┐
- ┌────────│ AND │──────┐
- │ └───────┘ │
- ▼ ▼
- ┌───────┐ ┌──────────┐
- ┌───────────│ AND │────────┐ │age < 100 │
- │ └───────┘ │ └──────────┘
- ▼ │ ▼
- ┌──────────┐ │ ┌──────────┐
- │ fname=p* │ ▼ │ age > 21 │
- └──────────┘ ┌──────────┐ └──────────┘
- │fname!=pa*│
- └──────────┘
-
-When excluding results from the result set, using `!=`, the
-[`QueryPlan`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/plan/QueryPlan.java)
-determines the best method for handling it. For range queries, for
-example, it may be optimal to divide the range into multiple parts
-with a hole for the exclusion. For string queries, such as this one,
-it is more optimal, however, to simply note which data to skip, or
-exclude, while scanning the index. Following this optimization the
-tree looks like this:
-
- ┌───────┐
- ┌────────│ AND │──────┐
- │ └───────┘ │
- ▼ ▼
- ┌───────┐ ┌──────────┐
- ┌───────│ AND │────────┐ │age < 100 │
- │ └───────┘ │ └──────────┘
- ▼ ▼
- ┌──────────────────┐ ┌──────────┐
- │ fname=p* │ │ age > 21 │
- │ exclusions=[pa*] │ └──────────┘
- └──────────────────┘
-
-The last type of optimization applied, for this query, is to merge
-range expressions across branches of the tree -- without modifying the
-meaning of the query, of course. In this case, because the query
-contains all `AND`s the `age` expressions can be collapsed. Along with
-this optimization, the initial collapsing of unneeded `AND`s can also
-be applied once more to result in this final tree using to execute the
-query:
-
- ┌───────┐
- ┌──────│ AND │───────┐
- │ └───────┘ │
- ▼ ▼
- ┌──────────────────┐ ┌────────────────┐
- │ fname=p* │ │ 21 < age < 100 │
- │ exclusions=[pa*] │ └────────────────┘
- └──────────────────┘
-
-#### Operations and Expressions
-
-As discussed, the
-[`QueryPlan`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/plan/QueryPlan.java)
-optimizes a tree represented by
-[`Operation`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/plan/Operation.java)s
-as interior nodes, and
-[`Expression`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/plan/Expression.java)s
-as leaves. The
-[`Operation`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/plan/Operation.java)
-class, more specifically, can have zero, one, or two
-[`Operation`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/plan/Operation.java)s
-as children and an unlimited number of expressions. The iterators used
-to perform the queries, discussed below in the
-"Range(Union|Intersection)Iterator" section, implement the necessary
-logic to merge results transparently regardless of the
-[`Operation`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/plan/Operation.java)s
-children.
-
-Besides participating in the optimizations performed by the
-[`QueryPlan`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/plan/QueryPlan.java),
-[`Operation`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/plan/Operation.java)
-is also responsible for taking a row that has been returned by the
-query and performing a final validation that it in fact does match. This
-`satisfiesBy` operation is performed recursively from the root of the
-[`Operation`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/plan/Operation.java)
-tree for a given query. These checks are performed directly on the
-data in a given row. For more details on how `satisfiesBy` works, see
-the documentation
-[in the code](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/plan/Operation.java#L87-L123).
-
-#### Range(Union|Intersection)Iterator
-
-The abstract `RangeIterator` class provides a unified interface over
-the two main operations performed by SASI at various layers in the
-execution path: set intersection and union. These operations are
-performed in a iterated, or "streaming", fashion to prevent unneeded
-reads of elements from either set. In both the intersection and union
-cases the algorithms take advantage of the data being pre-sorted using
-the same sort order, e.g. term or token order.
-
-The
-[`RangeUnionIterator`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/utils/RangeUnionIterator.java)
-performs the "Merge-Join" portion of the
-[Sort-Merge-Join](https://en.wikipedia.org/wiki/Sort-merge_join)
-algorithm, with the properties of an outer-join, or union. It is
-implemented with several optimizations to improve its performance over
-a large number of iterators -- sets to union. Specifically, the
-iterator exploits the likely case of the data having many sub-groups
-of overlapping ranges and the unlikely case that all ranges will
-overlap each other. For more details see the
-[javadoc](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/utils/RangeUnionIterator.java#L9-L21).
-
-The
-[`RangeIntersectionIterator`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/utils/RangeIntersectionIterator.java)
-itself is not a subclass of `RangeIterator`. It is a container for
-several classes, one of which, `AbstractIntersectionIterator`,
-sub-classes `RangeIterator`. SASI supports two methods of performing
-the intersection operation, and the ability to be adaptive in choosing
-between them based on some properties of the data.
-
-`BounceIntersectionIterator`, and the `BOUNCE` strategy, works like
-the
-[`RangeUnionIterator`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/utils/RangeUnionIterator.java)
-in that it performs a "Merge-Join", however, its nature is similar to
-a inner-join, where like values are merged by a data-specific merge
-function (e.g. merging two tokens in a list to lookup in a SSTable
-later). See the
-[javadoc](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/utils/RangeIntersectionIterator.java#L88-L101)
-for more details on its implementation.
-
-`LookupIntersectionIterator`, and the `LOOKUP` strategy, performs a
-different operation, more similar to a lookup in an associative data
-structure, or "hash lookup" in database terminology. Once again,
-details on the implementation can be found in the
-[javadoc](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/utils/RangeIntersectionIterator.java#L199-L208).
-
-The choice between the two iterators, or the `ADAPTIVE` strategy, is
-based upon the ratio of data set sizes of the minimum and maximum
-range of the sets being intersected. If the number of the elements in
-minimum range divided by the number of elements is the maximum range
-is less than or equal to `0.01`, then the `ADAPTIVE` strategy chooses
-the `LookupIntersectionIterator`, otherwise the
-`BounceIntersectionIterator` is chosen.
-
-### The SASIIndex Class
-
-The above components are glued together by the
-[`SASIIndex`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/SASIIndex.java)
-class which implements `Index`, and is instantiated
-per-table containing SASI indexes. It manages all indexes for a table
-via the
-[`sasi.conf.DataTracker`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/conf/DataTracker.java)
-and
-[`sasi.conf.view.View`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/conf/view/View.java)
-components, controls writing of all indexes for an SSTable via its
-[`PerSSTableIndexWriter`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/disk/PerSSTableIndexWriter.java), and initiates searches with
-`Searcher`. These classes glue the previously
-mentioned indexing components together with Cassandra's SSTable
-life-cycle ensuring indexes are not only written when Memtable's flush,
-but also as SSTable's are compacted. For querying, the
-`Searcher` does little but defer to
-[`QueryPlan`](https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/plan/QueryPlan.java)
-and update e.g. latency metrics exposed by SASI.
-
-### Cassandra Internal Changes
-
-To support the above changes and integrate them into Cassandra a few
-minor internal changes were made to Cassandra itself. These are
-described here.
-
-#### SSTable Write Life-cycle Notifications
-
-The `SSTableFlushObserver` is an observer pattern-like interface,
-whose sub-classes can register to be notified about events in the
-life-cycle of writing out a SSTable. Sub-classes can be notified when a
-flush begins and ends, as well as when each next row is about to be
-written, and each next column. SASI's `PerSSTableIndexWriter`,
-discussed above, is the only current subclass.
-
-### Limitations and Caveats
-
-The following are items that can be addressed in future updates but are not
-available in this repository or are not currently implemented.
-
-* The cluster must be configured to use a partitioner that produces
- `LongToken`s, e.g. `Murmur3Partitioner`. Other existing partitioners which
- don't produce LongToken e.g. `ByteOrderedPartitioner` and `RandomPartitioner`
- will not work with SASI.
-* Not Equals and OR support have been removed in this release while
- changes are made to Cassandra itself to support them.
-
-### Contributors
-
-* [Pavel Yaskevich](https://github.com/xedin)
-* [Jordan West](https://github.com/jrwest)
-* [Michael Kjellman](https://github.com/mkjellman)
-* [Jason Brown](https://github.com/jasobrown)
-* [Mikhail Stepura](https://github.com/mishail)
diff --git a/doc/antora.yml b/doc/antora.yml
deleted file mode 100644
index d01016f59340..000000000000
--- a/doc/antora.yml
+++ /dev/null
@@ -1,13 +0,0 @@
-name: Cassandra
-version: '5.0'
-display_version: '5.0'
-asciidoc:
- attributes:
- cass_url: 'http://cassandra.apache.org/'
- cass-50: 'Cassandra 5.0'
- cassandra: 'Cassandra'
- product: 'Apache Cassandra'
-
-nav:
-- modules/ROOT/nav.adoc
-- modules/cassandra/nav.adoc
diff --git a/doc/cql3/CQL.textile b/doc/cql3/CQL.textile
index 959533f77186..7bc18722f951 100644
--- a/doc/cql3/CQL.textile
+++ b/doc/cql3/CQL.textile
@@ -501,7 +501,7 @@ h3(#createIndexStmt). CREATE INDEX
__Syntax:__
bc(syntax)..
- ::= CREATE ( CUSTOM )? INDEX ( IF NOT EXISTS )? ( )?
+ ::= CREATE ( CUSTOM )? INDEX ( IF NOT EXISTS )? ( )?
ON '(' ')'
( USING ( WITH OPTIONS = )? )?
@@ -721,6 +721,8 @@ bc(syntax)..
'(' ( ',' )* ')'
( CALLED | RETURNS NULL ) ON NULL INPUT
RETURNS
+ ( DETERMINISTIC )?
+ ( MONOTONIC ( ON )? )?
LANGUAGE
AS
p.
@@ -766,6 +768,10 @@ If the optional @IF NOT EXISTS@ keywords are used, the function will only be cre
@OR REPLACE@ and @IF NOT EXIST@ cannot be used together.
+The optional @DETERMINISTIC@ keyword specifies that the function is deterministic. This means that given a particular input, the function will always produce the same output.
+
+The optional @MONOTONIC@ keyword specifies that the function is monotonic. This means that it is either entirely nonincreasing or nondecreasing. Even if the function is not monotonic on all its arguments, it is possible to specify that it is monotonic @ON@ one of its arguments, meaning that partial applications of the function over that argument will be monotonic. Monotonicity is required to use the function in a @GROUP BY@ clause.
+
Functions belong to a keyspace. If no keyspace is specified in @@, the current keyspace is used (i.e. the keyspace specified using the "@USE@":#useStmt statement). It is not possible to create a user-defined function in one of the system keyspaces.
See the section on "user-defined functions":#udfs for more information.
@@ -806,6 +812,7 @@ bc(syntax)..
STYPE
( FINALFUNC )?
( INITCOND )?
+ ( DETERMINISTIC )?
p.
__Sample:__
@@ -826,6 +833,8 @@ See the section on "user-defined aggregates":#udas for a complete example.
@OR REPLACE@ and @IF NOT EXIST@ cannot be used together.
+The optional @DETERMINISTIC@ keyword specifies that the aggregate function is deterministic. This means that given a particular input, the function will always produce the same output.
+
Aggregates belong to a keyspace. If no keyspace is specified in @@, the current keyspace is used (i.e. the keyspace specified using the "@USE@":#useStmt statement). It is not possible to create a user-defined aggregate in one of the system keyspaces.
Signatures for user-defined aggregates follow the "same rules":#functionSignature as for user-defined functions.
@@ -1092,8 +1101,9 @@ bc(syntax)..
( GROUP BY )?
( ORDER BY )?
( PER PARTITION LIMIT )?
- ( LIMIT )?
+ ( LIMIT ( OFFSET )? )?
( ALLOW FILTERING )?
+ ( WITH )?
::= DISTINCT?
@@ -1125,6 +1135,12 @@ bc(syntax)..
::= ( ',' )*
::= ( ASC | DESC )?
::= '(' (',' )* ')'
+
+ ::= ( AND )*
+ ::= ann_options '='
+ | included_indexes '='
+ | excluded_indexes '='
+ ::= '{' ( ',' )* '}'
p.
__Sample:__
@@ -1228,9 +1244,9 @@ Aggregate functions will produce a separate value for each group. If no @GROUP B
If a column is selected without an aggregate function, in a statement with a @GROUP BY@, the first value encounter in each group will be returned.
-h4(#selectLimit). @LIMIT@ and @PER PARTITION LIMIT@
+h4(#selectLimit). @LIMIT@, @OFFSET@ and @PER PARTITION LIMIT@
-The @LIMIT@ option to a @SELECT@ statement limits the number of rows returned by a query, while the @PER PARTITION LIMIT@ option limits the number of rows returned for a given partition by the query. Note that both type of limit can used in the same statement.
+The @LIMIT@ option in a @SELECT@ statement limits the number of rows returned by a query. The @LIMIT@ option can include an @OFFSET@ option to skip the first rows of the query result. The @PER PARTITION LIMIT@ option limits the number of rows returned for a given partition by the query. Note that both type of limit can used in the same statement.
h4(#selectAllowFiltering). @ALLOW FILTERING@
diff --git a/doc/modules/ROOT/nav.adoc b/doc/modules/ROOT/nav.adoc
index 3d367ad242a1..a484e704d21d 100644
--- a/doc/modules/ROOT/nav.adoc
+++ b/doc/modules/ROOT/nav.adoc
@@ -1,7 +1,7 @@
* xref:index.adoc[Main]
** xref:master@_:ROOT:glossary.adoc[Glossary]
** xref:master@_:ROOT:bugs.adoc[How to report bugs]
-** xref:master@_:ROOT:contactus.adoc[Contact us]
+** xref:master@_:ROOT:community.adoc[Contact us]
** xref:master@_:ROOT:development/index.adoc[Development]
*** xref:master@_:ROOT:development/gettingstarted.adoc[Getting started]
*** xref:master@_:ROOT:development/ide.adoc[Building and IDE integration]
diff --git a/doc/modules/ROOT/pages/index.adoc b/doc/modules/ROOT/pages/index.adoc
index beeb121f220e..f90812a58a66 100644
--- a/doc/modules/ROOT/pages/index.adoc
+++ b/doc/modules/ROOT/pages/index.adoc
@@ -47,7 +47,7 @@ If you would like to contribute to this documentation, you are welcome to do so
== Meta information
* xref:master@_:ROOT:bugs.adoc[Reporting bugs]
-* xref:master@_:ROOT:contactus.adoc[Contact us]
+* xref:master@_:ROOT:community.adoc[Contact us]
* xref:master@_:ROOT:development/index.adoc[Contributing code]
* xref:master@_:ROOT:docdev/index.adoc[Contributing to the docs]
* xref:master@_:ROOT:community.adoc[Community]
diff --git a/doc/modules/cassandra/assets/license_compliance.rst b/doc/modules/cassandra/assets/license_compliance.rst
index e2eba2ab8256..eb3ee826c70a 100644
--- a/doc/modules/cassandra/assets/license_compliance.rst
+++ b/doc/modules/cassandra/assets/license_compliance.rst
@@ -30,7 +30,7 @@ The Apache Cassandra project enforces and verifies ASF License header conformanc
With a few exceptions, source files consisting of works submitted directly to the ASF by the copyright owner or owner's
agent must contain the appropriate ASF license header. Files without any degree of creativity don't require a license header.
-Currently, RAT checks all .bat, .btm, .cql, .css, .g, .hmtl, .iml, .java, .jflex, .jks, .md, .mod, .name, .pom, .py, .sh, .spec, .textile, .yml, .yaml, .xml files for a LICENSE header.
+Currently, RAT checks all .bat, .btm, .cql, .css, .g, .hmtl, .iml, .java, .jks, .md, .mod, .name, .pom, .py, .sh, .spec, .textile, .yml, .yaml, .xml files for a LICENSE header.
If there is an incompliance, the build will fail with the following warning:
diff --git a/doc/modules/cassandra/examples/BNF/create_aggregate_statement.bnf b/doc/modules/cassandra/examples/BNF/create_aggregate_statement.bnf
index c0126a23ffd8..1207ec06328c 100644
--- a/doc/modules/cassandra/examples/BNF/create_aggregate_statement.bnf
+++ b/doc/modules/cassandra/examples/BNF/create_aggregate_statement.bnf
@@ -4,3 +4,4 @@ create_aggregate_statement ::= CREATE [ OR REPLACE ] AGGREGATE [ IF NOT EXISTS ]
STYPE cql_type:
[ FINALFUNC function_name]
[ INITCOND term ]
+ [ DETERMINISTIC ]
diff --git a/doc/modules/cassandra/examples/BNF/create_function_statement.bnf b/doc/modules/cassandra/examples/BNF/create_function_statement.bnf
index 0da769a11fb0..82be39d42911 100644
--- a/doc/modules/cassandra/examples/BNF/create_function_statement.bnf
+++ b/doc/modules/cassandra/examples/BNF/create_function_statement.bnf
@@ -1,6 +1,8 @@
create_function_statement::= CREATE [ OR REPLACE ] FUNCTION [ IF NOT EXISTS]
function_name '(' arguments_declaration ')'
[ CALLED | RETURNS NULL ] ON NULL INPUT
- RETURNS cql_type
+ RETURNS cql_type
+ [ DETERMINISTIC ]
+ [ MONOTONIC [ ON arg_name ] ]
LANGUAGE identifier
AS string arguments_declaration: identifier cql_type ( ',' identifier cql_type )*
diff --git a/doc/modules/cassandra/examples/BNF/select_statement.bnf b/doc/modules/cassandra/examples/BNF/select_statement.bnf
index f53da41da57c..d9906eea12f3 100644
--- a/doc/modules/cassandra/examples/BNF/select_statement.bnf
+++ b/doc/modules/cassandra/examples/BNF/select_statement.bnf
@@ -4,8 +4,9 @@ select_statement::= SELECT [ JSON | DISTINCT ] ( select_clause | '*' )
[ GROUP BY `group_by_clause` ]
[ ORDER BY `ordering_clause` ]
[ PER PARTITION LIMIT (`integer` | `bind_marker`) ]
- [ LIMIT (`integer` | `bind_marker`) ]
+ [ LIMIT (`integer` | `bind_marker`) [ OFFSET (`integer` | `bind_marker`) ] ]
[ ALLOW FILTERING ]
+ [ WITH `select_options` ]
select_clause::= `selector` [ AS `identifier` ] ( ',' `selector` [ AS `identifier` ] )
selector::== `column_name`
| `term`
@@ -17,5 +18,10 @@ relation::= column_name operator term
'(' column_name ( ',' column_name )* ')' operator tuple_literal
TOKEN '(' column_name# ( ',' column_name )* ')' operator term
operator::= '=' | '<' | '>' | '<=' | '>=' | '!=' | IN | CONTAINS | CONTAINS KEY
-group_by_clause::= column_name ( ',' column_name )*
+group_by_clause::= column_name ( ',' column_name )*
ordering_clause::= column_name [ ASC | DESC ] ( ',' column_name [ ASC | DESC ] )*
+select_options::= `select_option` ( AND `select_option` )*
+select_option::= ann_options '='
+ | included_indexes '=' `index_names`
+ | excluded_indexes '=' `index_names`
+index_names::= '{' index_name ( ',' index_name )* '}'
diff --git a/doc/modules/cassandra/examples/CQL/comments-table.cql b/doc/modules/cassandra/examples/CQL/comments-table.cql
index 7ce38f5218a7..ac4dd0361ed2 100644
--- a/doc/modules/cassandra/examples/CQL/comments-table.cql
+++ b/doc/modules/cassandra/examples/CQL/comments-table.cql
@@ -25,7 +25,7 @@ DROP INDEX IF EXISTS cycling.fn_sparse;
// tag::fn_sparse[]
CREATE CUSTOM INDEX IF NOT EXISTS fn_sparse
ON cycling.comments (created_at)
-USING 'org.apache.cassandra.index.sasi.SASIIndex'
+USING 'org.apache.cassandra.index.sai.StorageAttachedIndex'
WITH OPTIONS = { 'mode': 'SPARSE' };
// end::fn_sparse[]
diff --git a/doc/modules/cassandra/examples/CQL/query_with_ann_options.cql b/doc/modules/cassandra/examples/CQL/query_with_ann_options.cql
new file mode 100644
index 000000000000..14ddcd47b938
--- /dev/null
+++ b/doc/modules/cassandra/examples/CQL/query_with_ann_options.cql
@@ -0,0 +1 @@
+SELECT * FROM embeddings ORDER BY vector ANN OF [1.2, 3.4] LIMIT 100 WITH ann_options = { 'rerank_k': 1000 }
diff --git a/doc/modules/cassandra/examples/CQL/query_with_index_hints.cql b/doc/modules/cassandra/examples/CQL/query_with_index_hints.cql
new file mode 100644
index 000000000000..99cdbc915943
--- /dev/null
+++ b/doc/modules/cassandra/examples/CQL/query_with_index_hints.cql
@@ -0,0 +1,7 @@
+CREATE INDEX birth_year_idx ON users (birth_year);
+CREATE INDEX country_idx ON users (country);
+
+SELECT * FROM users
+ WHERE birth_year = 1981 AND country = 'FR' ALLOW FILTERING
+ WITH included_indexes = {birth_year_idx}
+ AND excluded_indexes = {country_idx};
diff --git a/doc/modules/cassandra/nav.adoc b/doc/modules/cassandra/nav.adoc
index 7c1a02cfaa94..a528f499f5ac 100644
--- a/doc/modules/cassandra/nav.adoc
+++ b/doc/modules/cassandra/nav.adoc
@@ -84,6 +84,7 @@
**** xref:cassandra:managing/configuration/cass_jvm_options_file.adoc[jvm-* files]
**** xref:cassandra:managing/configuration/configuration.adoc[Liberating cassandra.yaml Parameters' Names from Their Units]
*** xref:cassandra:managing/operating/index.adoc[]
+**** xref:cassandra:managing/operating/auto_repair.adoc[Auto Repair]
**** xref:cassandra:managing/operating/backups.adoc[Backups]
**** xref:cassandra:managing/operating/bloom_filters.adoc[Bloom filters]
**** xref:cassandra:managing/operating/bulk_loading.adoc[Bulk loading]
@@ -108,7 +109,7 @@
**** xref:cassandra:managing/tools/cqlsh.adoc[cqlsh: the CQL shell]
**** xref:cassandra:managing/tools/nodetool/nodetool.adoc[nodetool]
**** xref:cassandra:managing/tools/sstable/index.adoc[SSTable tools]
-**** xref:cassandra:managing/tools/cassandra_stress.adoc[cassandra-stress]
+**** xref:cassandra:tooling/cassandra-stress.adoc[cassandra-stress]
** xref:cassandra:troubleshooting/index.adoc[Troubleshooting]
*** xref:cassandra:troubleshooting/finding_nodes.adoc[Finding misbehaving nodes]
@@ -118,11 +119,11 @@
** xref:reference/index.adoc[]
*** xref:reference/cql-commands/commands-toc.adoc[CQL commands]
-*** xref:developing/cql/cql_singlefile.html[CQL specification]
+*** xref:cassandra:developing/cql/cql_singlefile.adoc[CQL specification]
*** xref:reference/java17.adoc[Java 17]
*** xref:reference/native-protocol.adoc[Native Protocol specification]
*** xref:reference/sai-virtual-table-indexes.adoc[SAI virtual table]
*** xref:reference/static.adoc[Static columns]
*** xref:reference/vector-data-type.adoc[Vector data type]
-** xref:integrating/plugins/index.adoc[]
\ No newline at end of file
+** xref:integrating/plugins/index.adoc[]
diff --git a/doc/modules/cassandra/pages/architecture/index.adoc b/doc/modules/cassandra/pages/architecture/index.adoc
index 9e674d95a2bb..5b69a8c9f94b 100644
--- a/doc/modules/cassandra/pages/architecture/index.adoc
+++ b/doc/modules/cassandra/pages/architecture/index.adoc
@@ -6,4 +6,4 @@ This section describes the general architecture of Apache Cassandra.
* xref:architecture/dynamo.adoc[Dynamo]
* xref:architecture/storage-engine.adoc[Storage Engine]
* xref:architecture/guarantees.adoc[Guarantees]
-* xref:architecture/snitch.adoc[Snitches]
+* xref:cassandra:managing/operating/snitch.adoc[Snitches]
diff --git a/doc/modules/cassandra/pages/architecture/storage-engine.adoc b/doc/modules/cassandra/pages/architecture/storage-engine.adoc
index 5580120a83ff..8a654ca791f7 100644
--- a/doc/modules/cassandra/pages/architecture/storage-engine.adoc
+++ b/doc/modules/cassandra/pages/architecture/storage-engine.adoc
@@ -30,7 +30,7 @@ Once the defined size is reached, a new commit log segment is created.
Commit log segments can be archived, deleted, or recycled once all the data is flushed to
https://cassandra.apache.org/_/glossary.html#sstable[SSTables].
Commit log segments are truncated when Cassandra has written data older than a certain point to the SSTables.
-Running xref:managing:tools/nodetool/drain.adoc[`nodetool drain`] before stopping Cassandra will write everything in the memtables
+Running xref:cassandra:managing/tools/nodetool/drain.adoc[`nodetool drain`] before stopping Cassandra will write everything in the memtables
to SSTables and remove the need to sync with the commit logs on startup.
* xref:cassandra:managing/configuration/cass_yaml_file.adoc#commitlog_segment_size [`commitlog_segment_size`]: The default size is 32MiB, which is almost always fine, but if you are archiving commitlog segments (see commitlog_archiving.properties), then you probably want a finer granularity of archiving; 8 or 16 MiB is reasonable.
@@ -105,7 +105,7 @@ A partition index is also created on the disk that maps the tokens to a location
The queue can be configured with either the xref:cassandra:managing/configuration/cass_yaml_file.adoc#memtable_heap_space[`memtable_heap_space`] or xref:cassandra:managing/configuration/cass_yaml_file.adoc#memtable_offheap_space[`memtable_offheap_space`] setting in the `cassandra.yaml` file.
If the data to be flushed exceeds the `memtable_cleanup_threshold`, Cassandra blocks writes until the next flush succeeds.
-You can manually flush a table using xref:managing:tools/nodetool/flush.adoc[`nodetool flush`] or `nodetool drain` (flushes memtables without listening for connections to other nodes).
+You can manually flush a table using xref:cassandra:managing/tools/nodetool/flush.adoc[`nodetool flush`] or `nodetool drain` (flushes memtables without listening for connections to other nodes).
To reduce the commit log replay time, the recommended best practice is to flush the memtable before you restart the nodes.
If a node stops working, replaying the commit log restores writes to the memtable that were there before it stopped.
diff --git a/doc/modules/cassandra/pages/developing/cql/SASI.adoc b/doc/modules/cassandra/pages/developing/cql/SASI.adoc
deleted file mode 100644
index 93d87f8ff385..000000000000
--- a/doc/modules/cassandra/pages/developing/cql/SASI.adoc
+++ /dev/null
@@ -1,809 +0,0 @@
-= SASI Index
-
-https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/SASIIndex.java[`SASIIndex`],
-or ``SASI`` for short, is an implementation of Cassandra's `Index`
-interface that can be used as an alternative to the existing
-implementations. SASI's indexing and querying improves on existing
-implementations by tailoring it specifically to Cassandra's needs. SASI
-has superior performance in cases where queries would previously require
-filtering. In achieving this performance, SASI aims to be significantly
-less resource intensive than existing implementations, in memory, disk,
-and CPU usage. In addition, SASI supports prefix and contains queries on
-strings (similar to SQL's ``LIKE = "foo\*"`` or ``LIKE = "*foo*"`` ).
-
-The following goes on describe how to get up and running with SASI,
-demonstrates usage with examples, and provides some details on its
-implementation.
-
-=== Using SASI
-
-The examples below walk through creating a table and indexes on its
-columns, and performing queries on some inserted data.
-
-The examples below assume the `demo` keyspace has been created and is in
-use.
-
-....
-cqlsh> CREATE KEYSPACE demo WITH replication = {
- ... 'class': 'SimpleStrategy',
- ... 'replication_factor': '1'
- ... };
-cqlsh> USE demo;
-....
-
-All examples are performed on the `sasi` table:
-
-....
-cqlsh:demo> CREATE TABLE sasi (id uuid, first_name text, last_name text,
- ... age int, height int, created_at bigint, primary key (id));
-....
-
-==== Creating Indexes
-
-To create SASI indexes use CQLs `CREATE CUSTOM INDEX` statement:
-
-....
-cqlsh:demo> CREATE CUSTOM INDEX ON sasi (first_name) USING 'org.apache.cassandra.index.sasi.SASIIndex'
- ... WITH OPTIONS = {
- ... 'analyzer_class':
- ... 'org.apache.cassandra.index.sasi.analyzer.NonTokenizingAnalyzer',
- ... 'case_sensitive': 'false'
- ... };
-
-cqlsh:demo> CREATE CUSTOM INDEX ON sasi (last_name) USING 'org.apache.cassandra.index.sasi.SASIIndex'
- ... WITH OPTIONS = {'mode': 'CONTAINS'};
-
-cqlsh:demo> CREATE CUSTOM INDEX ON sasi (age) USING 'org.apache.cassandra.index.sasi.SASIIndex';
-
-cqlsh:demo> CREATE CUSTOM INDEX ON sasi (created_at) USING 'org.apache.cassandra.index.sasi.SASIIndex'
- ... WITH OPTIONS = {'mode': 'SPARSE'};
-....
-
-The indexes created have some options specified that customize their
-behaviour and potentially performance. The index on `first_name` is
-case-insensitive. The analyzers are discussed more in a subsequent
-example. The `NonTokenizingAnalyzer` performs no analysis on the text.
-Each index has a mode: `PREFIX`, `CONTAINS`, or `SPARSE`, the first
-being the default. The `last_name` index is created with the mode
-`CONTAINS` which matches terms on suffixes instead of prefix only.
-Examples of this are available below and more detail can be found in the
-section on link:#ondiskindexbuilder[OnDiskIndex].The `created_at` column
-is created with its mode set to `SPARSE`, which is meant to improve
-performance of querying large, dense number ranges like timestamps for
-data inserted every millisecond. Details of the `SPARSE` implementation
-can also be found in the section on the
-link:#ondiskindexbuilder[OnDiskIndex]. The `age` index is created with
-the default `PREFIX` mode and no case-sensitivity or text analysis
-options are specified since the field is numeric.
-
-After inserting the following data and performing a `nodetool flush`,
-SASI performing index flushes to disk can be seen in Cassandra's logs –
-although the direct call to flush is not required (see
-link:#indexmemtable[IndexMemtable] for more details).
-
-....
-cqlsh:demo> INSERT INTO sasi (id, first_name, last_name, age, height, created_at)
- ... VALUES (556ebd54-cbe5-4b75-9aae-bf2a31a24500, 'Pavel', 'Yaskevich', 27, 181, 1442959315018);
-
-cqlsh:demo> INSERT INTO sasi (id, first_name, last_name, age, height, created_at)
- ... VALUES (5770382a-c56f-4f3f-b755-450e24d55217, 'Jordan', 'West', 26, 173, 1442959315019);
-
-cqlsh:demo> INSERT INTO sasi (id, first_name, last_name, age, height, created_at)
- ... VALUES (96053844-45c3-4f15-b1b7-b02c441d3ee1, 'Mikhail', 'Stepura', 36, 173, 1442959315020);
-
-cqlsh:demo> INSERT INTO sasi (id, first_name, last_name, age, height, created_at)
- ... VALUES (f5dfcabe-de96-4148-9b80-a1c41ed276b4, 'Michael', 'Kjellman', 26, 180, 1442959315021);
-
-cqlsh:demo> INSERT INTO sasi (id, first_name, last_name, age, height, created_at)
- ... VALUES (2970da43-e070-41a8-8bcb-35df7a0e608a, 'Johnny', 'Zhang', 32, 175, 1442959315022);
-
-cqlsh:demo> INSERT INTO sasi (id, first_name, last_name, age, height, created_at)
- ... VALUES (6b757016-631d-4fdb-ac62-40b127ccfbc7, 'Jason', 'Brown', 40, 182, 1442959315023);
-
-cqlsh:demo> INSERT INTO sasi (id, first_name, last_name, age, height, created_at)
- ... VALUES (8f909e8a-008e-49dd-8d43-1b0df348ed44, 'Vijay', 'Parthasarathy', 34, 183, 1442959315024);
-
-cqlsh:demo> SELECT first_name, last_name, age, height, created_at FROM sasi;
-
- first_name | last_name | age | height | created_at
-------------+---------------+-----+--------+---------------
- Michael | Kjellman | 26 | 180 | 1442959315021
- Mikhail | Stepura | 36 | 173 | 1442959315020
- Jason | Brown | 40 | 182 | 1442959315023
- Pavel | Yaskevich | 27 | 181 | 1442959315018
- Vijay | Parthasarathy | 34 | 183 | 1442959315024
- Jordan | West | 26 | 173 | 1442959315019
- Johnny | Zhang | 32 | 175 | 1442959315022
-
-(7 rows)
-....
-
-==== Equality & Prefix Queries
-
-SASI supports all queries already supported by CQL, including LIKE
-statement for PREFIX, CONTAINS and SUFFIX searches.
-
-....
-cqlsh:demo> SELECT first_name, last_name, age, height, created_at FROM sasi
- ... WHERE first_name = 'Pavel';
-
- first_name | last_name | age | height | created_at
--------------+-----------+-----+--------+---------------
- Pavel | Yaskevich | 27 | 181 | 1442959315018
-
-(1 rows)
-....
-
-....
-cqlsh:demo> SELECT first_name, last_name, age, height, created_at FROM sasi
- ... WHERE first_name = 'pavel';
-
- first_name | last_name | age | height | created_at
--------------+-----------+-----+--------+---------------
- Pavel | Yaskevich | 27 | 181 | 1442959315018
-
-(1 rows)
-....
-
-....
-cqlsh:demo> SELECT first_name, last_name, age, height, created_at FROM sasi
- ... WHERE first_name LIKE 'M%';
-
- first_name | last_name | age | height | created_at
-------------+-----------+-----+--------+---------------
- Michael | Kjellman | 26 | 180 | 1442959315021
- Mikhail | Stepura | 36 | 173 | 1442959315020
-
-(2 rows)
-....
-
-Of course, the case of the query does not matter for the `first_name`
-column because of the options provided at index creation time.
-
-....
-cqlsh:demo> SELECT first_name, last_name, age, height, created_at FROM sasi
- ... WHERE first_name LIKE 'm%';
-
- first_name | last_name | age | height | created_at
-------------+-----------+-----+--------+---------------
- Michael | Kjellman | 26 | 180 | 1442959315021
- Mikhail | Stepura | 36 | 173 | 1442959315020
-
-(2 rows)
-....
-
-==== Compound Queries
-
-SASI supports queries with multiple predicates, however, due to the
-nature of the default indexing implementation, CQL requires the user to
-specify `ALLOW FILTERING` to opt-in to the potential performance
-pitfalls of such a query. With SASI, while the requirement to include
-`ALLOW FILTERING` remains, to reduce modifications to the grammar, the
-performance pitfalls do not exist because filtering is not performed.
-Details on how SASI joins data from multiple predicates is available
-below in the link:#implementation-details[Implementation Details]
-section.
-
-....
-cqlsh:demo> SELECT first_name, last_name, age, height, created_at FROM sasi
- ... WHERE first_name LIKE 'M%' and age < 30 ALLOW FILTERING;
-
- first_name | last_name | age | height | created_at
-------------+-----------+-----+--------+---------------
- Michael | Kjellman | 26 | 180 | 1442959315021
-
-(1 rows)
-....
-
-==== Suffix Queries
-
-The next example demonstrates `CONTAINS` mode on the `last_name` column.
-By using this mode, predicates can search for any strings containing the
-search string as a sub-string. In this case the strings containing ``a''
-or ``an''.
-
-....
-cqlsh:demo> SELECT * FROM sasi WHERE last_name LIKE '%a%';
-
- id | age | created_at | first_name | height | last_name
---------------------------------------+-----+---------------+------------+--------+---------------
- f5dfcabe-de96-4148-9b80-a1c41ed276b4 | 26 | 1442959315021 | Michael | 180 | Kjellman
- 96053844-45c3-4f15-b1b7-b02c441d3ee1 | 36 | 1442959315020 | Mikhail | 173 | Stepura
- 556ebd54-cbe5-4b75-9aae-bf2a31a24500 | 27 | 1442959315018 | Pavel | 181 | Yaskevich
- 8f909e8a-008e-49dd-8d43-1b0df348ed44 | 34 | 1442959315024 | Vijay | 183 | Parthasarathy
- 2970da43-e070-41a8-8bcb-35df7a0e608a | 32 | 1442959315022 | Johnny | 175 | Zhang
-
-(5 rows)
-
-cqlsh:demo> SELECT * FROM sasi WHERE last_name LIKE '%an%';
-
- id | age | created_at | first_name | height | last_name
---------------------------------------+-----+---------------+------------+--------+-----------
- f5dfcabe-de96-4148-9b80-a1c41ed276b4 | 26 | 1442959315021 | Michael | 180 | Kjellman
- 2970da43-e070-41a8-8bcb-35df7a0e608a | 32 | 1442959315022 | Johnny | 175 | Zhang
-
-(2 rows)
-....
-
-==== Expressions on Non-Indexed Columns
-
-SASI also supports filtering on non-indexed columns like `height`. The
-expression can only narrow down an existing query using `AND`.
-
-....
-cqlsh:demo> SELECT * FROM sasi WHERE last_name LIKE '%a%' AND height >= 175 ALLOW FILTERING;
-
- id | age | created_at | first_name | height | last_name
---------------------------------------+-----+---------------+------------+--------+---------------
- f5dfcabe-de96-4148-9b80-a1c41ed276b4 | 26 | 1442959315021 | Michael | 180 | Kjellman
- 556ebd54-cbe5-4b75-9aae-bf2a31a24500 | 27 | 1442959315018 | Pavel | 181 | Yaskevich
- 8f909e8a-008e-49dd-8d43-1b0df348ed44 | 34 | 1442959315024 | Vijay | 183 | Parthasarathy
- 2970da43-e070-41a8-8bcb-35df7a0e608a | 32 | 1442959315022 | Johnny | 175 | Zhang
-
-(4 rows)
-....
-
-==== Delimiter based Tokenization Analysis
-
-A simple text analysis provided is delimiter based tokenization. This
-provides an alternative to indexing collections, as delimiter separated
-text can be indexed without the overhead of `CONTAINS` mode nor using
-`PREFIX` or `SUFFIX` queries.
-
-....
-cqlsh:demo> ALTER TABLE sasi ADD aliases text;
-cqlsh:demo> CREATE CUSTOM INDEX on sasi (aliases) USING 'org.apache.cassandra.index.sasi.SASIIndex'
- ... WITH OPTIONS = {
- ... 'analyzer_class': 'org.apache.cassandra.index.sasi.analyzer.DelimiterAnalyzer',
- ... 'delimiter': ',',
- ... 'mode': 'prefix',
- ... 'analyzed': 'true'};
-cqlsh:demo> UPDATE sasi SET aliases = 'Mike,Mick,Mikey,Mickey' WHERE id = f5dfcabe-de96-4148-9b80-a1c41ed276b4;
-cqlsh:demo> SELECT * FROM sasi WHERE aliases LIKE 'Mikey' ALLOW FILTERING;
-
- id | age | aliases | created_at | first_name | height | last_name
---------------------------------------+-----+------------------------+---------------+------------+--------+-----------
- f5dfcabe-de96-4148-9b80-a1c41ed276b4 | 26 | Mike,Mick,Mikey,Mickey | 1442959315021 | Michael | 180 | Kjellman
-....
-
-==== Text Analysis (Tokenization and Stemming)
-
-Lastly, to demonstrate text analysis an additional column is needed on
-the table. Its definition, index, and statements to update rows are
-shown below.
-
-....
-cqlsh:demo> ALTER TABLE sasi ADD bio text;
-cqlsh:demo> CREATE CUSTOM INDEX ON sasi (bio) USING 'org.apache.cassandra.index.sasi.SASIIndex'
- ... WITH OPTIONS = {
- ... 'analyzer_class': 'org.apache.cassandra.index.sasi.analyzer.StandardAnalyzer',
- ... 'tokenization_enable_stemming': 'true',
- ... 'analyzed': 'true',
- ... 'tokenization_normalize_lowercase': 'true',
- ... 'tokenization_locale': 'en'
- ... };
-cqlsh:demo> UPDATE sasi SET bio = 'Software Engineer, who likes distributed systems, doesnt like to argue.' WHERE id = 5770382a-c56f-4f3f-b755-450e24d55217;
-cqlsh:demo> UPDATE sasi SET bio = 'Software Engineer, works on the freight distribution at nights and likes arguing' WHERE id = 556ebd54-cbe5-4b75-9aae-bf2a31a24500;
-cqlsh:demo> SELECT * FROM sasi;
-
- id | age | bio | created_at | first_name | height | last_name
---------------------------------------+-----+----------------------------------------------------------------------------------+---------------+------------+--------+---------------
- f5dfcabe-de96-4148-9b80-a1c41ed276b4 | 26 | null | 1442959315021 | Michael | 180 | Kjellman
- 96053844-45c3-4f15-b1b7-b02c441d3ee1 | 36 | null | 1442959315020 | Mikhail | 173 | Stepura
- 6b757016-631d-4fdb-ac62-40b127ccfbc7 | 40 | null | 1442959315023 | Jason | 182 | Brown
- 556ebd54-cbe5-4b75-9aae-bf2a31a24500 | 27 | Software Engineer, works on the freight distribution at nights and likes arguing | 1442959315018 | Pavel | 181 | Yaskevich
- 8f909e8a-008e-49dd-8d43-1b0df348ed44 | 34 | null | 1442959315024 | Vijay | 183 | Parthasarathy
- 5770382a-c56f-4f3f-b755-450e24d55217 | 26 | Software Engineer, who likes distributed systems, doesnt like to argue. | 1442959315019 | Jordan | 173 | West
- 2970da43-e070-41a8-8bcb-35df7a0e608a | 32 | null | 1442959315022 | Johnny | 175 | Zhang
-
-(7 rows)
-....
-
-Index terms and query search strings are stemmed for the `bio` column
-because it was configured to use the
-https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/analyzer/StandardAnalyzer.java[`StandardAnalyzer`]
-and `analyzed` is set to `true`. The `tokenization_normalize_lowercase`
-is similar to the `case_sensitive` property but for the
-https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/analyzer/StandardAnalyzer.java[`StandardAnalyzer`].
-These query demonstrates the stemming applied by
-https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/analyzer/StandardAnalyzer.java[`StandardAnalyzer`].
-
-....
-cqlsh:demo> SELECT * FROM sasi WHERE bio LIKE 'distributing';
-
- id | age | bio | created_at | first_name | height | last_name
---------------------------------------+-----+----------------------------------------------------------------------------------+---------------+------------+--------+-----------
- 556ebd54-cbe5-4b75-9aae-bf2a31a24500 | 27 | Software Engineer, works on the freight distribution at nights and likes arguing | 1442959315018 | Pavel | 181 | Yaskevich
- 5770382a-c56f-4f3f-b755-450e24d55217 | 26 | Software Engineer, who likes distributed systems, doesnt like to argue. | 1442959315019 | Jordan | 173 | West
-
-(2 rows)
-
-cqlsh:demo> SELECT * FROM sasi WHERE bio LIKE 'they argued';
-
- id | age | bio | created_at | first_name | height | last_name
---------------------------------------+-----+----------------------------------------------------------------------------------+---------------+------------+--------+-----------
- 556ebd54-cbe5-4b75-9aae-bf2a31a24500 | 27 | Software Engineer, works on the freight distribution at nights and likes arguing | 1442959315018 | Pavel | 181 | Yaskevich
- 5770382a-c56f-4f3f-b755-450e24d55217 | 26 | Software Engineer, who likes distributed systems, doesnt like to argue. | 1442959315019 | Jordan | 173 | West
-
-(2 rows)
-
-cqlsh:demo> SELECT * FROM sasi WHERE bio LIKE 'working at the company';
-
- id | age | bio | created_at | first_name | height | last_name
---------------------------------------+-----+----------------------------------------------------------------------------------+---------------+------------+--------+-----------
- 556ebd54-cbe5-4b75-9aae-bf2a31a24500 | 27 | Software Engineer, works on the freight distribution at nights and likes arguing | 1442959315018 | Pavel | 181 | Yaskevich
-
-(1 rows)
-
-cqlsh:demo> SELECT * FROM sasi WHERE bio LIKE 'soft eng';
-
- id | age | bio | created_at | first_name | height | last_name
---------------------------------------+-----+----------------------------------------------------------------------------------+---------------+------------+--------+-----------
- 556ebd54-cbe5-4b75-9aae-bf2a31a24500 | 27 | Software Engineer, works on the freight distribution at nights and likes arguing | 1442959315018 | Pavel | 181 | Yaskevich
- 5770382a-c56f-4f3f-b755-450e24d55217 | 26 | Software Engineer, who likes distributed systems, doesnt like to argue. | 1442959315019 | Jordan | 173 | West
-
-(2 rows)
-....
-
-=== Implementation Details
-
-While SASI, at the surface, is simply an implementation of the `Index`
-interface, at its core there are several data structures and algorithms
-used to satisfy it. These are described here. Additionally, the changes
-internal to Cassandra to support SASI's integration are described.
-
-The `Index` interface divides responsibility of the implementer into two
-parts: Indexing and Querying. Further, Cassandra makes it possible to
-divide those responsibilities into the memory and disk components. SASI
-takes advantage of Cassandra's write-once, immutable, ordered data model
-to build indexes along with the flushing of the memtable to disk – this
-is the origin of the name `SSTable Attached Secondary Index`.
-
-The SASI index data structures are built in memory as the SSTable is
-being written and they are flushed to disk before the writing of the
-SSTable completes. The writing of each index file only requires
-sequential writes to disk. In some cases, partial flushes are performed,
-and later stitched back together, to reduce memory usage. These data
-structures are optimized for this use case.
-
-Taking advantage of Cassandra's ordered data model, at query time,
-candidate indexes are narrowed down for searching, minimizing the amount
-of work done. Searching is then performed using an efficient method that
-streams data off disk as needed.
-
-==== Indexing
-
-Per SSTable, SASI writes an index file for each indexed column. The data
-for these files is built in memory using the
-https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/disk/OnDiskIndexBuilder.java[`OnDiskIndexBuilder`].
-Once flushed to disk, the data is read using the
-https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/disk/OnDiskIndex.java[`OnDiskIndex`]
-class. These are composed of bytes representing indexed terms, organized
-for efficient writing or searching respectively. The keys and values
-they hold represent tokens and positions in an SSTable and these are
-stored per-indexed term in
-https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/disk/TokenTreeBuilder.java[`TokenTreeBuilder`]s
-for writing, and
-https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/disk/TokenTree.java[`TokenTree`]s
-for querying. These index files are memory mapped after being written to
-disk, for quicker access. For indexing data in the memtable, SASI uses
-its
-https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/memory/IndexMemtable.java[`IndexMemtable`]
-class.
-
-===== OnDiskIndex(Builder)
-
-Each
-https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/disk/OnDiskIndex.java[`OnDiskIndex`]
-is an instance of a modified
-https://en.wikipedia.org/wiki/Suffix_array[Suffix Array] data structure.
-The
-https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/disk/OnDiskIndex.java[`OnDiskIndex`]
-is comprised of page-size blocks of sorted terms and pointers to the
-terms' associated data, as well as the data itself, stored also in one
-or more page-sized blocks. The
-https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/disk/OnDiskIndex.java[`OnDiskIndex`]
-is structured as a tree of arrays, where each level describes the terms
-in the level below, the final level being the terms themselves. The
-``PointerLevel``s and their ``PointerBlock``s contain terms and pointers to
-other blocks that _end_ with those terms. The `DataLevel`, the final
-level, and its ``DataBlock``s contain terms and point to the data itself,
-contained in
-https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/disk/TokenTree.java[`TokenTree`]s.
-
-The terms written to the
-https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/disk/OnDiskIndex.java[`OnDiskIndex`]
-vary depending on its `mode` : either `PREFIX`, `CONTAINS`, or
-`SPARSE`. In the `PREFIX` and `SPARSE` cases, terms' exact values are
-written exactly once per `OnDiskIndex`. For example, when using a
-`PREFIX` index with terms `Jason`, `Jordan`, `Pavel`, all three will be
-included in the index. A `CONTAINS` index writes additional terms for
-each suffix of each term recursively. Continuing with the example, a
-`CONTAINS` index storing the previous terms would also store `ason`,
-`ordan`, `avel`, `son`, `rdan`, `vel`, etc. This allows for queries on
-the suffix of strings. The `SPARSE` mode differs from `PREFIX` in that
-for every 64 blocks of terms a
-https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/disk/TokenTree.java[`TokenTree`]
-is built merging all the
-https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/disk/TokenTree.java[`TokenTree`]s
-for each term into a single one. This copy of the data is used for
-efficient iteration of large ranges of e.g. timestamps. The index
-`mode` is configurable per column at index creation time.
-
-===== TokenTree(Builder)
-
-The
-https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/disk/TokenTree.java[`TokenTree`]
-is an implementation of the well-known
-https://en.wikipedia.org/wiki/B%2B_tree[B+ tree] that has been modified
-to optimize for its use-case. In particular, it has been optimized to
-associate tokens, longs, with a set of positions in an SSTable, also
-longs. Allowing the set of long values accommodates the possibility of a
-hash collision in the token, but the data structure is optimized for the
-unlikely possibility of such a collision.
-
-To optimize for its write-once environment the
-https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/disk/TokenTreeBuilder.java[`TokenTreeBuilder`]
-completely loads its interior nodes as the tree is built and it uses the
-well-known algorithm optimized for bulk-loading the data structure.
-
-https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/disk/TokenTree.java[`TokenTree`]s
-provide the means to iterate over tokens, and file positions, that match
-a given term, and to skip forward in that iteration, an operation used
-heavily at query time.
-
-===== IndexMemtable
-
-The
-https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/memory/IndexMemtable.java[`IndexMemtable`]
-handles indexing the in-memory data held in the memtable. The
-https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/memory/IndexMemtable.java[`IndexMemtable`]
-in turn manages either a
-https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/memory/TrieMemIndex.java[`TrieMemIndex`]
-or a
-https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/memory/SkipListMemIndex.java[`SkipListMemIndex`]
-per-column. The choice of which index type is used is data dependent.
-The
-https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/memory/TrieMemIndex.java[`TrieMemIndex`]
-is used for literal types. `AsciiType` and `UTF8Type` are literal types
-by default but any column can be configured as a literal type using the
-`is_literal` option at index creation time. For non-literal types the
-https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/memory/SkipListMemIndex.java[`SkipListMemIndex`]
-is used. The
-https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/memory/TrieMemIndex.java[`TrieMemIndex`]
-is an implementation that can efficiently support prefix queries on
-character-like data. The
-https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/memory/SkipListMemIndex.java[`SkipListMemIndex`],
-conversely, is better suited for other Cassandra data types like
-numbers.
-
-The
-https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/memory/TrieMemIndex.java[`TrieMemIndex`]
-is built using either the `ConcurrentRadixTree` or
-`ConcurrentSuffixTree` from the `com.goooglecode.concurrenttrees`
-package. The choice between the two is made based on the indexing mode,
-`PREFIX` or other modes, and `CONTAINS` mode, respectively.
-
-The
-https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/memory/SkipListMemIndex.java[`SkipListMemIndex`]
-is built on top of `java.util.concurrent.ConcurrentSkipListSet`.
-
-==== Querying
-
-Responsible for converting the internal `IndexExpression` representation
-into SASI's
-https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/plan/Operation.java[`Operation`]
-and
-https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/plan/Expression.java[`Expression`]
-trees, optimizing the trees to reduce the amount of work done, and
-driving the query itself, the
-https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/plan/QueryPlan.java[`QueryPlan`]
-is the work horse of SASI's querying implementation. To efficiently
-perform union and intersection operations, SASI provides several
-iterators similar to Cassandra's `MergeIterator`, but tailored
-specifically for SASI's use while including more features. The
-https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/utils/RangeUnionIterator.java[`RangeUnionIterator`],
-like its name suggests, performs set unions over sets of tokens/keys
-matching the query, only reading as much data as it needs from each set
-to satisfy the query. The
-https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/utils/RangeIntersectionIterator.java[`RangeIntersectionIterator`],
-similar to its counterpart, performs set intersections over its data.
-
-===== QueryPlan
-
-The
-https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/plan/QueryPlan.java[`QueryPlan`]
-instantiated per search query is at the core of SASI's querying
-implementation. Its work can be divided in two stages: analysis and
-execution.
-
-During the analysis phase,
-https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/plan/QueryPlan.java[`QueryPlan`]
-converts from Cassandra's internal representation of ``IndexExpression``s,
-which has also been modified to support encoding queries that contain
-ORs and groupings of expressions using parentheses (see the
-link:#cassandra-internal-changes[Cassandra Internal Changes] section
-below for more details). This process produces a tree of
-https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/plan/Operation.java[`Operation`]s,
-which in turn may contain
-https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/plan/Expression.java[`Expression`]s,
-all of which provide an alternative, more efficient, representation of
-the query.
-
-During execution, the
-https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/plan/QueryPlan.java[`QueryPlan`]
-uses the `DecoratedKey`-generating iterator created from the
-https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/plan/Operation.java[`Operation`]
-tree. These keys are read from disk and a final check to ensure they
-satisfy the query is made, once again using the
-https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/plan/Operation.java[`Operation`]
-tree. At the point the desired amount of matching data has been found,
-or there is no more matching data, the result set is returned to the
-coordinator through the existing internal components.
-
-The number of queries (total/failed/timed-out), and their latencies, are
-maintined per-table/column family.
-
-SASI also supports concurrently iterating terms for the same index
-across SSTables. The concurrency factor is controlled by the
-`cassandra.search_concurrency_factor` system property. The default is
-`1`.
-
-====== QueryController
-
-Each
-https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/plan/QueryPlan.java[`QueryPlan`]
-references a
-https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/plan/QueryController.java[`QueryController`]
-used throughout the execution phase. The
-https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/plan/QueryController.java[`QueryController`]
-has two responsibilities: to manage and ensure the proper cleanup of
-resources (indexes), and to strictly enforce the time bound per query,
-specified by the user via the range slice timeout. All indexes are
-accessed via the
-https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/plan/QueryController.java[`QueryController`]
-so that they can be safely released by it later. The
-https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/plan/QueryController.java[`QueryController`]'s
-`checkpoint` function is called in specific places in the execution path
-to ensure the time-bound is enforced.
-
-====== QueryPlan Optimizations
-
-While in the analysis phase, the
-https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/plan/QueryPlan.java[`QueryPlan`]
-performs several potential optimizations to the query. The goal of these
-optimizations is to reduce the amount of work performed during the
-execution phase.
-
-The simplest optimization performed is compacting multiple expressions
-joined by logical intersections (`AND`) into a single
-https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/plan/Operation.java[`Operation`]
-with three or more
-https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/plan/Expression.java[`Expression`]s.
-For example, the query
-`WHERE age < 100 AND fname = 'p*' AND first_name != 'pa*' AND age > 21`
-would, without modification, have the following tree:
-
-....
- ┌───────┐
- ┌────────│ AND │──────┐
- │ └───────┘ │
- ▼ ▼
- ┌───────┐ ┌──────────┐
- ┌─────│ AND │─────┐ │age < 100 │
- │ └───────┘ │ └──────────┘
- ▼ ▼
-┌──────────┐ ┌───────┐
-│ fname=p* │ ┌─│ AND │───┐
-└──────────┘ │ └───────┘ │
- ▼ ▼
- ┌──────────┐ ┌──────────┐
- │fname!=pa*│ │ age > 21 │
- └──────────┘ └──────────┘
-....
-
-https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/plan/QueryPlan.java[`QueryPlan`]
-will remove the redundant right branch whose root is the final `AND` and
-has leaves `fname != pa*` and `age > 21`. These
-https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/plan/Expression.java[`Expression`]s
-will be compacted into the parent `AND`, a safe operation due to `AND`
-being associative and commutative. The resulting tree looks like the
-following:
-
-....
- ┌───────┐
- ┌────────│ AND │──────┐
- │ └───────┘ │
- ▼ ▼
- ┌───────┐ ┌──────────┐
- ┌───────────│ AND │────────┐ │age < 100 │
- │ └───────┘ │ └──────────┘
- ▼ │ ▼
-┌──────────┐ │ ┌──────────┐
-│ fname=p* │ ▼ │ age > 21 │
-└──────────┘ ┌──────────┐ └──────────┘
- │fname!=pa*│
- └──────────┘
-....
-
-When excluding results from the result set, using `!=`, the
-https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/plan/QueryPlan.java[`QueryPlan`]
-determines the best method for handling it. For range queries, for
-example, it may be optimal to divide the range into multiple parts with
-a hole for the exclusion. For string queries, such as this one, it is
-more optimal, however, to simply note which data to skip, or exclude,
-while scanning the index. Following this optimization the tree looks
-like this:
-
-....
- ┌───────┐
- ┌────────│ AND │──────┐
- │ └───────┘ │
- ▼ ▼
- ┌───────┐ ┌──────────┐
- ┌───────│ AND │────────┐ │age < 100 │
- │ └───────┘ │ └──────────┘
- ▼ ▼
- ┌──────────────────┐ ┌──────────┐
- │ fname=p* │ │ age > 21 │
- │ exclusions=[pa*] │ └──────────┘
- └──────────────────┘
-....
-
-The last type of optimization applied, for this query, is to merge range
-expressions across branches of the tree – without modifying the meaning
-of the query, of course. In this case, because the query contains all
-``AND``s the `age` expressions can be collapsed. Along with this
-optimization, the initial collapsing of unneeded ``AND``s can also be
-applied once more to result in this final tree using to execute the
-query:
-
-....
- ┌───────┐
- ┌──────│ AND │───────┐
- │ └───────┘ │
- ▼ ▼
- ┌──────────────────┐ ┌────────────────┐
- │ fname=p* │ │ 21 < age < 100 │
- │ exclusions=[pa*] │ └────────────────┘
- └──────────────────┘
-....
-
-===== Operations and Expressions
-
-As discussed, the
-https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/plan/QueryPlan.java[`QueryPlan`]
-optimizes a tree represented by
-https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/plan/Operation.java[`Operation`]s
-as interior nodes, and
-https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/plan/Expression.java[`Expression`]s
-as leaves. The
-https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/plan/Operation.java[`Operation`]
-class, more specifically, can have zero, one, or two
-https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/plan/Operation.java[`Operation`]s
-as children and an unlimited number of expressions. The iterators used
-to perform the queries, discussed below in the
-`Range(Union|Intersection)Iterator` section, implement the necessary
-logic to merge results transparently regardless of the
-https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/plan/Operation.java[`Operation`]s
-children.
-
-Besides participating in the optimizations performed by the
-https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/plan/QueryPlan.java[`QueryPlan`],
-https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/plan/Operation.java[`Operation`]
-is also responsible for taking a row that has been returned by the query
-and performing a final validation that it in fact does match. This
-`satisfiesBy` operation is performed recursively from the root of the
-https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/plan/Operation.java[`Operation`]
-tree for a given query. These checks are performed directly on the data
-in a given row. For more details on how `satisfiesBy` works, see the
-documentation
-https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/plan/Operation.java#L87-L123[in
-the code].
-
-===== Range(Union|Intersection)Iterator
-
-The abstract `RangeIterator` class provides a unified interface over the
-two main operations performed by SASI at various layers in the execution
-path: set intersection and union. These operations are performed in a
-iterated, or `streaming`, fashion to prevent unneeded reads of
-elements from either set. In both the intersection and union cases the
-algorithms take advantage of the data being pre-sorted using the same
-sort order, e.g. term or token order.
-
-The
-https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/utils/RangeUnionIterator.java[`RangeUnionIterator`]
-performs the `Merge-Join` portion of the
-https://en.wikipedia.org/wiki/Sort-merge_join[Sort-Merge-Join]
-algorithm, with the properties of an outer-join, or union. It is
-implemented with several optimizations to improve its performance over a
-large number of iterators – sets to union. Specifically, the iterator
-exploits the likely case of the data having many sub-groups of
-overlapping ranges and the unlikely case that all ranges will overlap
-each other. For more details see the
-https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/utils/RangeUnionIterator.java#L9-L21[javadoc].
-
-The
-https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/utils/RangeIntersectionIterator.java[`RangeIntersectionIterator`]
-itself is not a subclass of `RangeIterator`. It is a container for
-several classes, one of which, `AbstractIntersectionIterator`,
-sub-classes `RangeIterator`. SASI supports two methods of performing the
-intersection operation, and the ability to be adaptive in choosing
-between them based on some properties of the data.
-
-`BounceIntersectionIterator`, and the `BOUNCE` strategy, works like the
-https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/utils/RangeUnionIterator.java[`RangeUnionIterator`]
-in that it performs a `Merge-Join`, however, its nature is similar to
-a inner-join, where like values are merged by a data-specific merge
-function (e.g. merging two tokens in a list to lookup in a SSTable
-later). See the
-https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/utils/RangeIntersectionIterator.java#L88-L101[javadoc]
-for more details on its implementation.
-
-`LookupIntersectionIterator`, and the `LOOKUP` strategy, performs a
-different operation, more similar to a lookup in an associative data
-structure, or `hash lookup` in database terminology. Once again,
-details on the implementation can be found in the
-https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/utils/RangeIntersectionIterator.java#L199-L208[javadoc].
-
-The choice between the two iterators, or the `ADAPTIVE` strategy, is
-based upon the ratio of data set sizes of the minimum and maximum range
-of the sets being intersected. If the number of the elements in minimum
-range divided by the number of elements is the maximum range is less
-than or equal to `0.01`, then the `ADAPTIVE` strategy chooses the
-`LookupIntersectionIterator`, otherwise the `BounceIntersectionIterator`
-is chosen.
-
-==== The SASIIndex Class
-
-The above components are glued together by the
-https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/SASIIndex.java[`SASIIndex`]
-class which implements `Index`, and is instantiated per-table containing
-SASI indexes. It manages all indexes for a table via the
-https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/conf/DataTracker.java[`sasi.conf.DataTracker`]
-and
-https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/conf/view/View.java[`sasi.conf.view.View`]
-components, controls writing of all indexes for an SSTable via its
-https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/disk/PerSSTableIndexWriter.java[`PerSSTableIndexWriter`],
-and initiates searches with `Searcher`. These classes glue the
-previously mentioned indexing components together with Cassandra's
-SSTable life-cycle ensuring indexes are not only written when Memtable's
-flush, but also as SSTable's are compacted. For querying, the `Searcher`
-does little but defer to
-https://github.com/apache/cassandra/blob/trunk/src/java/org/apache/cassandra/index/sasi/plan/QueryPlan.java[`QueryPlan`]
-and update e.g. latency metrics exposed by SASI.
-
-==== Cassandra Internal Changes
-
-To support the above changes and integrate them into Cassandra a few
-minor internal changes were made to Cassandra itself. These are
-described here.
-
-===== SSTable Write Life-cycle Notifications
-
-The `SSTableFlushObserver` is an observer pattern-like interface, whose
-sub-classes can register to be notified about events in the life-cycle
-of writing out a SSTable. Sub-classes can be notified when a flush
-begins and ends, as well as when each next row is about to be written,
-and each next column. SASI's `PerSSTableIndexWriter`, discussed above,
-is the only current subclass.
-
-==== Limitations and Caveats
-
-The following are items that can be addressed in future updates but are
-not available in this repository or are not currently implemented.
-
-* The cluster must be configured to use a partitioner that produces
-``LongToken``s, e.g. `Murmur3Partitioner`. Other existing partitioners
-which don't produce LongToken e.g. `ByteOrderedPartitioner` and
-`RandomPartitioner` will not work with SASI.
-* Not Equals and OR support have been removed in this release while
-changes are made to Cassandra itself to support them.
-
-==== Contributors
-
-* https://github.com/xedin[Pavel Yaskevich]
-* https://github.com/jrwest[Jordan West]
-* https://github.com/mkjellman[Michael Kjellman]
-* https://github.com/jasobrown[Jason Brown]
-* https://github.com/mishail[Mikhail Stepura]
diff --git a/doc/modules/cassandra/pages/developing/cql/batch/batch-good-example.adoc b/doc/modules/cassandra/pages/developing/cql/batch/batch-good-example.adoc
index 93887253569b..86c248b3b328 100644
--- a/doc/modules/cassandra/pages/developing/cql/batch/batch-good-example.adoc
+++ b/doc/modules/cassandra/pages/developing/cql/batch/batch-good-example.adoc
@@ -24,7 +24,7 @@ All the `INSERT` and `UPDATE` statements in this batch write to the same partiti
include::cassandra:example$CQL/cyclist_expenses-table.cql[tag=batch_Vera]
----
+
-This batching example includes conditional updates combined with using xref:reference:static.adoc[static columns].
+This batching example includes conditional updates combined with using xref:cassandra:developing/cql/ddl.adoc#static-column[static columns].
Recall that single partition batches are not logged.
+
[NOTE]
diff --git a/doc/modules/cassandra/pages/developing/cql/collections/list.adoc b/doc/modules/cassandra/pages/developing/cql/collections/list.adoc
index 8113b5aa267e..6be7c9938f46 100644
--- a/doc/modules/cassandra/pages/developing/cql/collections/list.adoc
+++ b/doc/modules/cassandra/pages/developing/cql/collections/list.adoc
@@ -9,7 +9,7 @@ Use the `list` data type to store data that has a possible many-to-many relation
== Prerequisite
-* xref:developing/cql/keyspace-check.adoc[Keyspace] must exist
+* xref:cassandra:developing/cql/ddl.adoc#create-keyspace-statement[Keyspace] must exist
In the following example, a `list` called `events` stores all the race events on an upcoming calendar.
The table is called `upcoming_calendar`. Each event listed in the `list` will have a `text` data type.
diff --git a/doc/modules/cassandra/pages/developing/cql/collections/map.adoc b/doc/modules/cassandra/pages/developing/cql/collections/map.adoc
index f3ce072bf258..148b8b569733 100644
--- a/doc/modules/cassandra/pages/developing/cql/collections/map.adoc
+++ b/doc/modules/cassandra/pages/developing/cql/collections/map.adoc
@@ -11,7 +11,7 @@ Each element can have an individual time-to-live and expire when the TTL ends.
== Prerequisite
-* xref:developing/cql/keyspace-check.adoc[Keyspace] must exist
+* xref:cassandra:developing/cql/ddl.adoc#create-keyspace-statement[Keyspace] must exist
In the following example, each team listed in the `map` called `teams` will have a `year` of integer type and a `team name` of text type.
The table is named `cyclist_teams`.
diff --git a/doc/modules/cassandra/pages/developing/cql/collections/set.adoc b/doc/modules/cassandra/pages/developing/cql/collections/set.adoc
index 4834c8f21a1f..db5d85f4e4d6 100644
--- a/doc/modules/cassandra/pages/developing/cql/collections/set.adoc
+++ b/doc/modules/cassandra/pages/developing/cql/collections/set.adoc
@@ -8,7 +8,7 @@ Use the `set` data type to store data that has a many-to-one relationship with a
== Prerequisite
-* xref:developing/cql/keyspace-check.adoc[Keyspace] must exist
+* xref:cassandra:developing/cql/ddl.adoc#create-keyspace-statement[Keyspace] must exist
In the following example, a `set` called `teams` stores all the teams that a cyclist has been a member of during their career.
The table is `cyclist_career_teams`.
diff --git a/doc/modules/cassandra/pages/developing/cql/cql_singlefile.adoc b/doc/modules/cassandra/pages/developing/cql/cql_singlefile.adoc
index 4de16fbb713e..7cef8f5a6543 100644
--- a/doc/modules/cassandra/pages/developing/cql/cql_singlefile.adoc
+++ b/doc/modules/cassandra/pages/developing/cql/cql_singlefile.adoc
@@ -1227,6 +1227,8 @@ CREATE FUNCTION akeyspace.fname IF NOT EXISTS
( someArg int )
CALLED ON NULL INPUT
RETURNS text
+( DETERMINISTIC )?
+( MONOTONIC ( ON )? )?
LANGUAGE java
AS $$
// some Java code
@@ -1266,6 +1268,17 @@ exist.
`OR REPLACE` and `IF NOT EXIST` cannot be used together.
+The optional `DETERMINISTIC` keyword specifies that the function is
+deterministic. This means that given a particular input, the function
+will always produce the same output.
+
+The optional `MONOTONIC` keyword specifies that the function is monotonic.
+This means that it is either entirely nonincreasing or nondecreasing.
+Even if the function is not monotonic on all its arguments, it is possible
+to specify that it is monotonic `ON` one of its arguments, meaning that
+partial applications of the function over that argument will be monotonic.
+Monotonicity is required to use the function in a `GROUP BY` clause.
+
Functions belong to a keyspace. If no keyspace is specified in
``, the current keyspace is used (i.e. the keyspace
specified using the link:#useStmt[`USE`] statement). It is not possible
@@ -1318,6 +1331,7 @@ SFUNC
STYPE
( FINALFUNC )?
( INITCOND )?
+( DETERMINISTIC )?
----
_Sample:_
@@ -1346,6 +1360,10 @@ creates an aggregate if it does not already exist.
`OR REPLACE` and `IF NOT EXIST` cannot be used together.
+The optional `DETERMINISTIC` keyword specifies that the aggregate
+function is deterministic. This means that given a particular input,
+the function will always produce the same output.
+
Aggregates belong to a keyspace. If no keyspace is specified in
``, the current keyspace is used (i.e. the keyspace
specified using the link:#useStmt[`USE`] statement). It is not possible
@@ -1756,8 +1774,9 @@ FROM
( GROUP BY )?
( ORDER BY )?
( PER PARTITION LIMIT )?
-( LIMIT )?
+( LIMIT ( OFFSET )? )? +
( ALLOW FILTERING )?
+( WITH ann_options = )?
::= DISTINCT?
@@ -1985,12 +2004,12 @@ with a `GROUP BY`, the first value encounter in each group will be
returned.
[[selectLimit]]
-===== `LIMIT` and `PER PARTITION LIMIT`
+===== `LIMIT`, `OFFSET` and `PER PARTITION LIMIT`
-The `LIMIT` option to a `SELECT` statement limits the number of rows
-returned by a query, while the `PER PARTITION LIMIT` option limits the
-number of rows returned for a given partition by the query. Note that
-both type of limit can used in the same statement.
+The `LIMIT` option in a `SELECT` statement limits the number of rows returned by a query.
+The `LIMIT` option can include an `OFFSET` option to skip the first rows of the query result.
+The `PER PARTITION LIMIT` option limits the number of rows returned for a given partition by the query.
+Note that both types of limits can used in the same statement.
[[selectAllowFiltering]]
===== `ALLOW FILTERING`
diff --git a/doc/modules/cassandra/pages/developing/cql/create-custom-index.adoc b/doc/modules/cassandra/pages/developing/cql/create-custom-index.adoc
index 29fcfbc1bd27..5d91c31dc420 100644
--- a/doc/modules/cassandra/pages/developing/cql/create-custom-index.adoc
+++ b/doc/modules/cassandra/pages/developing/cql/create-custom-index.adoc
@@ -5,7 +5,7 @@ include::cassandra:partial$sai/support-databases.adoc[]
Creates a Storage-Attached Indexing (SAI) index.
You can create multiple secondary indexes on the same database table, with each SAI index based on any column in the table.
-All column date types except the following are supported for SAI indexes:
+All column data types except the following are supported for SAI indexes:
* `counter`
* geospatial types: `PointType`, `LineStringType`, `PolygonType`
@@ -55,7 +55,7 @@ SAI allows only alphanumeric characters and underscores in names.
SAI returns `InvalidRequestException` if you try to define an index on a column name that contains other characters, and does not create the index.
map_name::
-Used with xref:cassandra:developing/collections/collection-create.adoc[collections], identifier of the `map_name` specified in `CREATE TABLE` ...
+Used with xref:cassandra:developing/cql/collections/collection-create.adoc[collections], identifier of the `map_name` specified in `CREATE TABLE` ...
`map()`.
The regular column syntax applies for collection types `list` and `set`.
@@ -119,7 +119,7 @@ Also refer xref:cassandra:developing/cql/indexing/sai/sai-query.adoc[Examine SAI
=== SAI collection map examples with keys, values, and entries
The following examples demonstrate using collection maps of multiple types (`keys`, `values`, `entries`) in SAI indexes.
-For related information, see xref:cassandra:developing/collections/collection-create.adoc[Creating collections] and xref:cassandra:developing/collections/map.adoc[Using map type].
+For related information, see xref:cassandra:developing/cql/collections/collection-create.adoc[Creating collections] and xref:cassandra:developing/cql/collections/map.adoc[Using map type].
Also refer to the SAI collection examples of type xref:#saiCreateCustomIndexCollectionsListAndSetExamples[list and set] in this topic.
@@ -293,9 +293,9 @@ Remember that in CQL queries using SAI indexes, the `CONTAINS` clauses are suppo
These examples demonstrate using collections with the `list` and `set` types in SAI indexes.
For related information, see:
-* xref:cassandra:developing/collections/collection-create.adoc[Creating collections]
-* xref:cassandra:developing/collections/list.adoc[Using list type]
-* xref:cassandra:developing/collections/set.adoc[Using set type]
+* xref:cassandra:developing/cql/collections/collection-create.adoc[Creating collections]
+* xref:cassandra:developing/cql/collections/list.adoc[Using list type]
+* xref:cassandra:developing/cql/collections/set.adoc[Using set type]
[source,language-cql]
----
diff --git a/doc/modules/cassandra/pages/developing/cql/create-index.adoc b/doc/modules/cassandra/pages/developing/cql/create-index.adoc
index 88721030f052..d3114d6610e1 100644
--- a/doc/modules/cassandra/pages/developing/cql/create-index.adoc
+++ b/doc/modules/cassandra/pages/developing/cql/create-index.adoc
@@ -8,7 +8,7 @@ After an index has been created, it is automatically updated when data in the co
Indexing via this `CREATE INDEX` command can impact performance.
Before creating an index, be aware of when and xref:cassandra:developing/cql/indexing/2i/2i-when-to-use.adoc#when-no-index[when not to create an index].
-Use xref:cassandra:developing/cql/indexing/create-custom-index.adoc[CREATE CUSTOM INDEX] for Storage-Attached Indexes (SAI).
+Use xref:cassandra:developing/cql/create-custom-index.adoc[CREATE CUSTOM INDEX] for Storage-Attached Indexes (SAI).
*Restriction:* Indexing counter columns is not supported.
For maps, index the key, value, or entries.
@@ -90,7 +90,7 @@ To index map keys, use the `KEYS` keyword and map name in nested parentheses:
include::cassandra:example$CQL/sai/cyclist_teams-table.cql[tag=keysidx]
----
-To query the table, you can use xref:cassandra:reference/cql-commands/select.adoc#filtering-on-collections[CONTAINS KEY] in `WHERE` clauses.
+To query the table, you can use xref:cassandra:developing/cql/dml.adoc#allow-filtering[CONTAINS KEY] in `WHERE` clauses.
[source,language-cql]
----
diff --git a/doc/modules/cassandra/pages/developing/cql/ddl.adoc b/doc/modules/cassandra/pages/developing/cql/ddl.adoc
index d771f4e2bcb2..93d9f5c1ed06 100644
--- a/doc/modules/cassandra/pages/developing/cql/ddl.adoc
+++ b/doc/modules/cassandra/pages/developing/cql/ddl.adoc
@@ -166,7 +166,7 @@ will result in:
include::cassandra:example$RESULTS/autoexpand_exclude_dc.result[]
----
-If xref:new/transientreplication.adoc[transient replication] has been enabled, transient replicas can be
+If xref:cassandra:managing/operating/transientreplication.adoc[transient replication] has been enabled, transient replicas can be
configured for both `SimpleStrategy` and `NetworkTopologyStrategy` by
defining replication factors in the format
`'/'`
diff --git a/doc/modules/cassandra/pages/developing/cql/definitions.adoc b/doc/modules/cassandra/pages/developing/cql/definitions.adoc
index 3e0251cf4c52..64536cec6565 100644
--- a/doc/modules/cassandra/pages/developing/cql/definitions.adoc
+++ b/doc/modules/cassandra/pages/developing/cql/definitions.adoc
@@ -119,7 +119,7 @@ include::cassandra:example$BNF/term.bnf[]
A term is thus one of:
-* A xref:cassandra:developing/cql/defintions.adoc#constants[constant]
+* A xref:cassandra:developing/cql/definitions.adoc#constants[constant]
* A literal for either a xref:cassandra:developing/cql/types.adoc#collections[collection], a xref:cassandra:developing/cql/types.adoc#vectors[vector],
a xref:cassandra:developing/cql/types.adoc#udts[user-defined type] or a xref:cassandra:developing/cql/types.adoc#tuples[tuple]
* A xref:cassandra:developing/cql/functions.adoc#cql-functions[function] call, either a xref:cassandra:developing/cql/functions.adoc#scalar-native-functions[native function]
diff --git a/doc/modules/cassandra/pages/developing/cql/dml.adoc b/doc/modules/cassandra/pages/developing/cql/dml.adoc
index 674ede814518..3696f90b24b0 100644
--- a/doc/modules/cassandra/pages/developing/cql/dml.adoc
+++ b/doc/modules/cassandra/pages/developing/cql/dml.adoc
@@ -214,9 +214,10 @@ or the reverse
[[limit-clause]]
=== Limiting results
-The `LIMIT` option to a `SELECT` statement limits the number of rows
-returned by a query. The `PER PARTITION LIMIT` option limits the
-number of rows returned for a given partition by the query. Both types of limits can used in the same statement.
+The `LIMIT` option in a `SELECT` statement limits the number of rows returned by a query.
+The `LIMIT` option can include an `OFFSET` option to skip the first rows of the query result.
+The `PER PARTITION LIMIT` option limits the number of rows returned for a given partition by the query.
+Note that both types of limits can used in the same statement.
[[allow-filtering]]
=== Allowing filtering
@@ -264,6 +265,38 @@ execute:
include::cassandra:example$CQL/query_nofail_allow_filtering.cql[]
----
+[[ann-options]]
+=== ANN options
+
+`SELECT` queries using `ANN` ordering can provide a set of options to control the behavior of the ANN search:
+
+[source,cql]
+----
+include::example$CQL/query_with_ann_options.cql[]
+----
+
+[[index-hints]]
+=== Index hints
+
+`SELECT` statements allow to provide sets of included and excluded indexes:
+
+[source,cql]
+----
+include::example$CQL/query_with_index_hints.cql[]
+----
+The included indexes are indexes that should be used by the query.
+Queries will fail if it's not possible to use the included indexes.
+That might happen because the query doesn't have a restriction for those indexes,
+or because there is a restriction that can use the index,
+but it is not compatible with other restrictions,
+or because the underlying index implementation isn't able to use the index for whatever reason.
+
+The excluded indexes are indexes that should not be used by the query.
+Excluded indexes will never fail the query unless they reference a non-existent index,
+since it's always possible to exclude indexes that are not used by the query.
+
+The indexes mentioned in included or excluded sets must exist, otherwise the query will fail.
+
[[insert-statement]]
== INSERT
diff --git a/doc/modules/cassandra/pages/developing/cql/functions.adoc b/doc/modules/cassandra/pages/developing/cql/functions.adoc
index 9599b98a2434..d82a0f0e884e 100644
--- a/doc/modules/cassandra/pages/developing/cql/functions.adoc
+++ b/doc/modules/cassandra/pages/developing/cql/functions.adoc
@@ -288,6 +288,43 @@ A number of functions allow to obtain the similarity score between vectors of fl
include::cassandra:partial$vector-search/vector_functions.adoc[]
+[[index-functions]]
+===== Index functions
+
+====== `sai_analyze`
+
+The `sai_analyze` functions returns the tokens that a SAI index will generate for a certain text value. The arguments
+are that text value and the JSON configuration of the SAI analyzer. This JSON configuration is the same as the one used
+to create the SAI index. For example, this function call:
+
+[source,cql]
+----
+sai_analyze('johnny apples seedlings',
+ '{
+ "tokenizer": {"name": "whitespace"}
+ }')
+----
+Will return `['johnny', 'apples', 'seedlings']`
+
+This other function call:
+[source,cql]
+----
+sai_analyze('johnny apples seedlings',
+ '{
+ "tokenizer": {"name": "whitespace"},
+ "filters": [{"name": "porterstem"}]
+ }')
+----
+Will return `['johnni', 'appl', 'seedl']`
+
+
+[[vector-functions]]
+===== Vector functions
+
+A number of functions to operate with vectors of floats.
+
+include::cassandra:partial$vector-search/vector_functions.adoc[]
+
[[user-defined-scalar-functions]]
=== User-defined functions
@@ -378,6 +415,16 @@ If the optional `IF NOT EXISTS` keywords are used, the function will only be cre
exist.
`OR REPLACE` and `IF NOT EXISTS` cannot be used together.
+The optional `DETERMINISTIC` keyword specifies that the aggregate function is deterministic.
+This means that given a particular input, the function will always produce the same output.
+
+The optional `MONOTONIC` keyword specifies that the function is monotonic.
+This means that it is either entirely nonincreasing or nondecreasing.
+Even if the function is not monotonic on all its arguments, it is possible
+to specify that it is monotonic `ON` one of its arguments, meaning that
+partial applications of the function over that argument will be monotonic.
+Monotonicity is required to use the function in a `GROUP BY` clause.
+
Behavior for `null` input values must be defined for each function:
* `RETURNS NULL ON NULL INPUT` declares that the function will always return `null` if any of the input arguments is `null`.
@@ -540,6 +587,9 @@ A `CREATE AGGREGATE` without `OR REPLACE` fails if an aggregate with the same si
The `CREATE AGGREGATE` command with the optional `IF NOT EXISTS` keywords creates an aggregate if it does not already exist.
The `OR REPLACE` and `IF NOT EXISTS` phrases cannot be used together.
+The optional `DETERMINISTIC` keyword specifies that the aggregate function is deterministic.
+This means that given a particular input, the function will always produce the same output.
+
The `STYPE` value defines the type of the state value and must be specified.
The optional `INITCOND` defines the initial state value for the aggregate; the default value is `null`.
A non-null `INITCOND` must be specified for state functions that are declared with `RETURNS NULL ON NULL INPUT`.
diff --git a/doc/modules/cassandra/pages/developing/cql/indexing/2i/2i-working-with.adoc b/doc/modules/cassandra/pages/developing/cql/indexing/2i/2i-working-with.adoc
index 09fd5a638c7d..27cdafd6f202 100644
--- a/doc/modules/cassandra/pages/developing/cql/indexing/2i/2i-working-with.adoc
+++ b/doc/modules/cassandra/pages/developing/cql/indexing/2i/2i-working-with.adoc
@@ -4,8 +4,8 @@
== Prerequisites
-* xref:developing/keyspace-create.adoc[Keyspace created]
-* xref:developing/table-create.adoc[Table created]
+* xref:cassandra:developing/cql/ddl.adoc#create-keyspace-statement[Keyspace created]
+* xref:cassandra:developing/cql/ddl.adoc#create-table-statement[Table created]
include::_2i-create.adoc[leveloffset=+1]
diff --git a/doc/modules/cassandra/pages/developing/cql/indexing/2i/operations/2i-build.adoc b/doc/modules/cassandra/pages/developing/cql/indexing/2i/operations/2i-build.adoc
index 80201b133834..37caa0977542 100644
--- a/doc/modules/cassandra/pages/developing/cql/indexing/2i/operations/2i-build.adoc
+++ b/doc/modules/cassandra/pages/developing/cql/indexing/2i/operations/2i-build.adoc
@@ -7,4 +7,4 @@ Indexes are built in the background automatically, without blocking reads or wri
Client-maintained _tables as indexes_ must be created manually;
for example, if the artists column had been indexed by creating a table such as `songs_by_artist`, your client application would have to populate the table with data from the songs table.
-To perform a hot rebuild of an index, use the xref:cassandra:tools/nodetool/rebuild_index.adoc[nodetool rebuild_index] command.
+To perform a hot rebuild of an index, use the xref:cassandra:managing/tools/nodetool/rebuild_index.adoc[nodetool rebuild_index] command.
diff --git a/doc/modules/cassandra/pages/developing/cql/indexing/sai/_sai-create.adoc b/doc/modules/cassandra/pages/developing/cql/indexing/sai/_sai-create.adoc
index d7a5c708f8cc..45e9362837d0 100644
--- a/doc/modules/cassandra/pages/developing/cql/indexing/sai/_sai-create.adoc
+++ b/doc/modules/cassandra/pages/developing/cql/indexing/sai/_sai-create.adoc
@@ -1,7 +1,9 @@
= Create SAI index
:description: Create SAI index for CQL table schema using `cqlsh`.
-To create an SAI index, you must define the index name, table name, and column name for the column to be indexed.
+To create an SAI index, you must define the index name, table name, and column name for the column to be indexed.
+
+include::cassandra:partial$index-naming.adoc[]
To create a simple SAI index:
@@ -74,4 +76,4 @@ include::cassandra:example$CQL/sai/index-sai-similarity-function.cql[]
'''
Other resources
-See xref:developing/cql/indexing/create-custom-index.adoc[CREATE CUSTOM INDEX] for more information about creating SAI indexes.
\ No newline at end of file
+See xref:cassandra:developing/cql/create-custom-index.adoc[CREATE CUSTOM INDEX] for more information about creating SAI indexes.
\ No newline at end of file
diff --git a/doc/modules/cassandra/pages/developing/cql/indexing/sai/operations/configuring.adoc b/doc/modules/cassandra/pages/developing/cql/indexing/sai/operations/configuring.adoc
index 0acd114746cd..7293c099abcd 100644
--- a/doc/modules/cassandra/pages/developing/cql/indexing/sai/operations/configuring.adoc
+++ b/doc/modules/cassandra/pages/developing/cql/indexing/sai/operations/configuring.adoc
@@ -3,8 +3,9 @@
// LLP: *NOT DONE*
-Configuring your {product} environment for Storage-Attached Indexing (SAI) may require some customization of the `cassandra.yaml` file.
+Configuring your {product} environment for Storage-Attached Indexing (SAI) may require some customization of the `cassandra.yaml` file.
+[[saiConfigure__saiCompactionStrategies]]
== Compaction strategies
Read queries perform better with compaction strategies that produce fewer SSTables.
@@ -20,13 +21,13 @@ While in a time window, TWCS compacts all SSTables flushed from memory into larg
At the end of the time window, all of these SSTables are compacted into a single SSTable.
Then the next time window starts and the process repeats.
The duration of the time window is the only setting required.
-See xref:reference:cql-commands/create-table.adoc#compactSubprop__TWCS[TimeWindowCompactionStrategy].
+See xref:cassandra:reference/cql-commands/compact-subproperties.adoc#TWCS[TimeWindowCompactionStrategy].
For more information about TWCS, see xref:cassandra:managing/operating/compaction/twcs.adoc[Time Window Compaction Strategy].
In general, do not use `LeveledCompactionStrategy` (LCS) unless your index queries restrict the token range, either directly or by providing a restriction on the partition key.
However, if you decide to use LCS, use the following guidelines:
-* The `160` MB default for the `CREATE TABLE` command's `sstable_size_in_mb` option, described in this xref:reference:cql-commands/create-table.adoc#compactSubprop__LCS[topic], may result in suboptimal performance for index queries that do not restrict on token range or partition key.
+* The `160` MB default for the `CREATE TABLE` command's `sstable_size_in_mb` option, described in this xref:cassandra:reference/cql-commands/compact-subproperties.adoc#LCS[topic], may result in suboptimal performance for index queries that do not restrict on token range or partition key.
* While even higher values may be appropriate, depending on your hardware, we recommend at least doubling the default value of `sstable_size_in_mb`.
Example:
diff --git a/doc/modules/cassandra/pages/developing/cql/indexing/sai/sai-faq.adoc b/doc/modules/cassandra/pages/developing/cql/indexing/sai/sai-faq.adoc
index 95f6783a25f6..e91a17b08647 100644
--- a/doc/modules/cassandra/pages/developing/cql/indexing/sai/sai-faq.adoc
+++ b/doc/modules/cassandra/pages/developing/cql/indexing/sai/sai-faq.adoc
@@ -314,7 +314,7 @@ the post filtering on `col2` is carried out on the replicas themselves.
Another case were post-filtering comes into play is when constructing a query that involves more than two SAI indexes.
Refer to this xref:#saiAndQueriesFaq[related FAQ] about `AND` queries.
-== Can I create an SAI index based on a xref:reference:static.adoc[static column]?
+== Can I create an SAI index based on a xref:cassandra:developing/cql/ddl.adoc#static-column[static column]?
Yes.
For example, consider a `transaction_by_customer` table where you have a primary key `customer_id`, plus static columns to contain each customer's `address`, `phone_number`, and `date_of_birth`.
diff --git a/doc/modules/cassandra/pages/developing/cql/indexing/sai/sai-read-write-paths.adoc b/doc/modules/cassandra/pages/developing/cql/indexing/sai/sai-read-write-paths.adoc
index e735a9d210c9..4a34688d853e 100644
--- a/doc/modules/cassandra/pages/developing/cql/indexing/sai/sai-read-write-paths.adoc
+++ b/doc/modules/cassandra/pages/developing/cql/indexing/sai/sai-read-write-paths.adoc
@@ -27,7 +27,7 @@ SAI calculates an estimate of the incremental heap consumption of the new entry.
This estimate counts against the heap usage of the underlying Memtable.
This feature also means that as more columns are indexed on a table, the Memtable flush rate will increase, and the size of flushed SSTables will decrease.
The number of total writes and the estimated heap usage of all live Memtable indexes are exposed as metrics.
-See xref:cassandra:developing/cql/indexing/sai/monitoring.adoc#saiMonitorMetrics[SAI metrics].
+See xref:cassandra:developing/cql/indexing/sai/operations/monitoring.adoc#saiMonitorMetrics[SAI metrics].
=== Memtable flush
@@ -94,7 +94,7 @@ image::sai/saiOnDiskStructureWithOffsets.png[alt=SAI on-disk layout as described
The actual segment flushing process is very similar to a Memtable flush.
However, buffered terms are sorted before they can be written with their postings to their respective type-specific on-disk structures.
At the end of compaction for a given index, a special empty marker file is flagged to indicate success, and the number of segments is recorded in SAI metrics.
-See xref:developing:indexing/sai/monitoring.adoc#saiGlobalIndexingMetrics[Global indexing metrics].
+See xref:cassandra:developing/cql/indexing/sai/operations/monitoring.adoc#saiGlobalIndexingMetrics[Global indexing metrics].
When the entire compaction task finishes, SAI receives an SSTable List Changed Notification that contains the SSTables added and removed during the transaction.
SSTable Context Manager and Index View Manager are responsible for replacing old SSTable indexes with new ones atomically.
diff --git a/doc/modules/cassandra/pages/developing/cql/indexing/sai/sai-working-with.adoc b/doc/modules/cassandra/pages/developing/cql/indexing/sai/sai-working-with.adoc
index 98d8e2653a3e..6117786dbe43 100644
--- a/doc/modules/cassandra/pages/developing/cql/indexing/sai/sai-working-with.adoc
+++ b/doc/modules/cassandra/pages/developing/cql/indexing/sai/sai-working-with.adoc
@@ -4,8 +4,8 @@
== Prerequisites
-* xref:developing/keyspace-create.adoc[Keyspace created]
-* xref:developing/table-create.adoc[Table created]
+* xref:cassandra:developing/cql/ddl.adoc#create-keyspace-statement[Keyspace created]
+* xref:cassandra:developing/cql/ddl.adoc#create-table-statement[Table created]
include::_sai-create.adoc[leveloffset=+1]
diff --git a/doc/modules/cassandra/pages/developing/cql/security.adoc b/doc/modules/cassandra/pages/developing/cql/security.adoc
index 0af30a9d1541..b94ae3379c44 100644
--- a/doc/modules/cassandra/pages/developing/cql/security.adoc
+++ b/doc/modules/cassandra/pages/developing/cql/security.adoc
@@ -167,11 +167,11 @@ used and the role does not exist the statement is a no-op.
DROP ROLE intentionally does not terminate any open user sessions.
Currently connected sessions will remain connected and will retain the
ability to perform any database actions which do not require
-xref:cassandra:developing/cql/security.adoc#authorization[authorization].
+xref:cassandra:managing/operating/security.adoc#authorization[authorization].
However, if authorization is enabled, xref:cassandra:developing/cql/security.adoc#cql-permissions[permissions] of the dropped role are also revoked,
-subject to the xref:cassandra:developing/cql/security.adoc#auth-caching[caching options] configured in xref:cassandra:developing/cql/configuring.adoc#cassandra.yaml[cassandra-yaml] file.
-Should a dropped role be subsequently recreated and have new xref:security.adoc#grant-permission-statement[permissions] or
-xref:security.adoc#grant-role-statement[roles] granted to it, any client sessions still
+subject to the xref:cassandra:managing/operating/security.adoc#auth-caching[caching options] configured in xref:cassandra:managing/configuration/cass_yaml_file.adoc[cassandra-yaml] file.
+Should a dropped role be subsequently recreated and have new xref:cassandra:developing/cql/security.adoc#grant-permission-statement[permissions] or
+xref:cassandra:developing/cql/security.adoc#grant-role-statement[roles] granted to it, any client sessions still
connected will acquire the newly granted permissions and roles.
====
@@ -332,7 +332,7 @@ Existing users can be listed using the `LIST USERS` statement:
include::cassandra:example$BNF/list_users_statement.bnf[]
----
-Note that this statement is equivalent to xref:security.adoc#list-roles-statement[LIST ROLES], but only roles with the `LOGIN` privilege are included in the output.
+Note that this statement is equivalent to xref:cassandra:developing/cql/security.adoc#list-roles-statement[LIST ROLES], but only roles with the `LOGIN` privilege are included in the output.
== Data Control
diff --git a/doc/modules/cassandra/pages/developing/data-modeling/data-modeling_logical.adoc b/doc/modules/cassandra/pages/developing/data-modeling/data-modeling_logical.adoc
index 80ddf3b6f0d6..ba91fae929d7 100644
--- a/doc/modules/cassandra/pages/developing/data-modeling/data-modeling_logical.adoc
+++ b/doc/modules/cassandra/pages/developing/data-modeling/data-modeling_logical.adoc
@@ -34,7 +34,7 @@ informative way to visualize the relationships between queries and
tables in your designs. This figure shows the Chebotko notation for a
logical data model.
-image::cassandra:developing/data-modeling/data_modeling_chebotko_logical.png[image]
+image::data_modeling_chebotko_logical.png[image]
Each table is shown with its title and a list of columns. Primary key
columns are identified via symbols such as *K* for partition key columns
@@ -51,7 +51,7 @@ dedicated tables for rooms or amenities, as you had in the relational
design. This is because the workflow didn't identify any queries
requiring this direct access.
-image::cassandra:developing/data-modeling/data_modeling_hotel_logical.png[image]
+image::data_modeling_hotel_logical.png[image]
Let's explore the details of each of these tables.
@@ -127,7 +127,7 @@ shows a logical data model for reservations. You'll notice that these
tables represent a denormalized design; the same data appears in
multiple tables, with differing keys.
-image::cassandra:developing/data-modeling/data_modeling_reservation_logical.png[image]
+image::data_modeling_reservation_logical.png[image]
In order to satisfy Q6, the `reservations_by_guest` table can be used to
look up the reservation by guest name. You could envision query Q7 being
diff --git a/doc/modules/cassandra/pages/developing/data-modeling/data-modeling_queries.adoc b/doc/modules/cassandra/pages/developing/data-modeling/data-modeling_queries.adoc
index b33e91e05e4f..88f47758c746 100644
--- a/doc/modules/cassandra/pages/developing/data-modeling/data-modeling_queries.adoc
+++ b/doc/modules/cassandra/pages/developing/data-modeling/data-modeling_queries.adoc
@@ -28,7 +28,7 @@ here, however, you'll want to think not only from the customer
perspective in terms of how the data is written, but also in terms of
how the data will be queried by downstream use cases.
-You natural tendency as might be to focus first on designing the tables
+Your natural tendency might be to focus first on designing the tables
to store reservation and guest records, and only then start thinking
about the queries that would access them. You may have felt a similar
tension already when discussing the shopping queries before, thinking
diff --git a/doc/modules/cassandra/pages/developing/data-modeling/data-modeling_schema.adoc b/doc/modules/cassandra/pages/developing/data-modeling/data-modeling_schema.adoc
index 68c0cfcea708..cf788933c011 100644
--- a/doc/modules/cassandra/pages/developing/data-modeling/data-modeling_schema.adoc
+++ b/doc/modules/cassandra/pages/developing/data-modeling/data-modeling_schema.adoc
@@ -114,12 +114,11 @@ CREATE TABLE reservation.guests (
first_name text,
last_name text,
title text,
- emails set,
- phone_numbers list,
- addresses map,
- confirm_number text )
- WITH comment = 'Q9. Find guest by ID';
+ emails set,
+ phone_numbers list,
+ addresses map>,
+ confirm_number text
+) WITH comment = 'Q9. Find guest by ID';
----
You now have a complete Cassandra schema for storing data for a hotel
diff --git a/doc/modules/cassandra/pages/getting-started/production.adoc b/doc/modules/cassandra/pages/getting-started/production.adoc
index ad28a35f5638..475c1f7fb8f1 100644
--- a/doc/modules/cassandra/pages/getting-started/production.adoc
+++ b/doc/modules/cassandra/pages/getting-started/production.adoc
@@ -118,7 +118,7 @@ https://thelastpickle.com/blog/2018/08/08/compression_performance.html[The Last
== Compaction
-There are different xref:compaction/index.adoc[compaction] strategies available
+There are different xref:cassandra:managing/operating/compaction/index.adoc[compaction] strategies available
for different workloads.
We recommend reading about the different strategies to understand which is the
best for your environment.
diff --git a/doc/modules/cassandra/pages/getting-started/querying.adoc b/doc/modules/cassandra/pages/getting-started/querying.adoc
index 78866cef0e69..2e1fe2361bea 100644
--- a/doc/modules/cassandra/pages/getting-started/querying.adoc
+++ b/doc/modules/cassandra/pages/getting-started/querying.adoc
@@ -22,7 +22,7 @@ include::cassandra:example$BASH/cqlsh_localhost.sh[]
----
include::cassandra:example$RESULTS/cqlsh_localhost.result[]
----
-If the command is used without specifying a node, `localhost` is the default. See the xref:tools/cqlsh.adoc[`cqlsh` section] for full documentation.
+If the command is used without specifying a node, `localhost` is the default. See the xref:cassandra:managing/tools/cqlsh.adoc[`cqlsh` section] for full documentation.
== Client drivers
diff --git a/doc/modules/cassandra/pages/getting-started/sai-quickstart.adoc b/doc/modules/cassandra/pages/getting-started/sai-quickstart.adoc
index 12cb6092e69f..16cd3e66ff3e 100644
--- a/doc/modules/cassandra/pages/getting-started/sai-quickstart.adoc
+++ b/doc/modules/cassandra/pages/getting-started/sai-quickstart.adoc
@@ -40,6 +40,8 @@ Use xref:reference/cql-commands/create-custom-index.adoc[CREATE CUSTOM INDEX] co
include::cassandra:example$CQL/sai/cyclist_semi_pro_sai_indices.cql[tag=createQuickStartIndices]
----
+include::cassandra:partial$index-naming.adoc[]
+
Let's take a look at the description of the table and its indexes:
[tabs]
====
diff --git a/doc/modules/cassandra/pages/installing/installing.adoc b/doc/modules/cassandra/pages/installing/installing.adoc
index e5043914e00c..7ee741ea214f 100644
--- a/doc/modules/cassandra/pages/installing/installing.adoc
+++ b/doc/modules/cassandra/pages/installing/installing.adoc
@@ -279,4 +279,4 @@ include::cassandra:partial$nodetool_and_cqlsh_nobin.adoc[]
== Further installation info
-For help with installation issues, see the xref:cassandra:troubleshooting/index.html[Troubleshooting] section.
+For help with installation issues, see the xref:cassandra:troubleshooting/index.adoc[Troubleshooting] section.
diff --git a/doc/modules/cassandra/pages/managing/configuration/cass_env_sh_file.adoc b/doc/modules/cassandra/pages/managing/configuration/cass_env_sh_file.adoc
index 309b15b17dd0..b1bfb9639305 100644
--- a/doc/modules/cassandra/pages/managing/configuration/cass_env_sh_file.adoc
+++ b/doc/modules/cassandra/pages/managing/configuration/cass_env_sh_file.adoc
@@ -31,11 +31,6 @@ In a multi-instance deployment, multiple Cassandra instances will
independently assume that all CPU processors are available to it. This
setting allows you to specify a smaller set of processors.
-== `cassandra.boot_without_jna=true`
-
-If JNA fails to initialize, Cassandra fails to boot. Use this command to
-boot Cassandra without JNA.
-
== `cassandra.config=`
The directory location of the `cassandra.yaml file`. The default
diff --git a/doc/modules/cassandra/pages/managing/operating/audit_logging.adoc b/doc/modules/cassandra/pages/managing/operating/audit_logging.adoc
index a40fcc5b2e43..023a8e07fd54 100644
--- a/doc/modules/cassandra/pages/managing/operating/audit_logging.adoc
+++ b/doc/modules/cassandra/pages/managing/operating/audit_logging.adoc
@@ -150,8 +150,12 @@ auditlogviewer [...] [options]
waiting for more records
`-r,--roll_cycle`::
How often to roll the log file was rolled. May be;;
- necessary for Chronicle to correctly parse file names. (MINUTELY,
- HOURLY, DAILY). Default HOURLY.
+ necessary for Chronicle to correctly parse file names. Some available options are:
+FIVE_MINUTELY, FAST_HOURLY, FAST_DAILY, LargeRollCycles.LARGE_DAILY, LargeRollCycles.XLARGE_DAILY,
+LargeRollCycles.HUGE_DAILY. Deprecated ones still availble but not recommended for new deployments:
+MINUTELY, HOURLY, DAILY
+For more options, refer: net.openhft.chronicle.queue.RollCycles.
+Default is set to FAST_HOURLY
`-h,--help`::
display this help message
diff --git a/doc/modules/cassandra/pages/managing/operating/auto_repair.adoc b/doc/modules/cassandra/pages/managing/operating/auto_repair.adoc
new file mode 100644
index 000000000000..7971263b5027
--- /dev/null
+++ b/doc/modules/cassandra/pages/managing/operating/auto_repair.adoc
@@ -0,0 +1,473 @@
+= Auto Repair
+:navtitle: Auto Repair
+:description: Auto Repair concepts - How it works, how to configure it, and more.
+:keywords: CEP-37, Repair, Incremental, Preview
+
+Auto Repair is a fully automated scheduler that provides repair orchestration within Apache Cassandra. This
+significantly reduces operational overhead by eliminating the need for operators to deploy external tools to submit and
+manage repairs.
+
+At a high level, a dedicated thread pool is assigned to the repair scheduler. The repair scheduler in Cassandra
+maintains a new replicated table, `system_distributed.auto_repair_history`, which stores the repair history for all
+nodes, including details such as the last repair time. The scheduler selects the node(s) to begin repairs and
+orchestrates the process to ensure that every table and its token ranges are repaired.
+
+The algorithm can run repairs simultaneously on multiple nodes and splits token ranges into subranges, with necessary
+retries to handle transient failures. Automatic repair starts as soon as a Cassandra cluster is launched, similar to
+compaction, and if configured appropriately, does not require human intervention.
+
+The scheduler currently supports Full, Incremental, and Preview repair types with the following features. New repair
+types, such as Paxos repair or other future repair mechanisms, can be integrated with minimal development effort!
+
+
+== Features
+- Capability to run repairs on multiple nodes simultaneously.
+- A default implementation and an interface to override the dataset being repaired per session.
+- Extendable token split algorithms with two implementations readily available:
+. Splits token ranges by placing a cap on the size of data repaired in one session and a maximum cap at the schedule
+level using xref:#repair-token-range-splitter[RepairTokenRangeSplitter] (default).
+. Splits tokens evenly based on the specified number of splits using
+xref:#fixed-split-token-range-splitter[FixedSplitTokenRangeSplitter].
+- A new xref:#table-configuration[CQL table property] (`auto_repair`) offering:
+. The ability to disable specific repair types at the table level, allowing the scheduler to skip one or more tables.
+. Configuring repair priorities for certain tables to prioritize them over others.
+- Dynamic enablement or disablement of the scheduler for each repair type.
+- Configurable settings tailored to each repair job.
+- Rich configuration options for each repair type (e.g., Full, Incremental, or Preview repairs).
+- Comprehensive observability features that allow operators to configure alarms as needed.
+
+== Availability
+
+Auto Repair was introduced in Cassandra 6.0 via CEP-37 and backported to 5.0.8.
+
+In 5.0.8, auto-repair requires enabling the JVM property `-Dcassandra.autorepair.enable=true` before starting the
+node. This property creates the required schema elements (the `auto_repair` column in `system_schema.tables` and
+`system_schema.views`, and the `auto_repair_history` and `auto_repair_priority` tables in `system_distributed`).
+After enabling this property, auto-repair scheduling still needs to be enabled either in `cassandra.yaml` under
+the `auto_repair` section or at runtime via JMX.
+
+WARNING: The `cassandra.autorepair.enable` property is non-reversible. Once enabled, it cannot be disabled.
+See the xref:#upgrading[Upgrading] section in NEWS.txt for details.
+
+== Considerations
+
+Before enabling Auto Repair, please consult the xref:managing/operating/repair.adoc[Repair] guide to establish a base
+understanding of repairs.
+
+=== Full Repair
+
+Full Repairs operate over all data in the token range being repaired. It is therefore important to run full repair
+with a longer schedule and with smaller assignments.
+
+=== Incremental Repair
+
+When enabled from the inception of a cluster, incremental repairs operate over unrepaired data and should finish
+quickly when run more frequently.
+
+Once incremental repair has been run, SSTables will be separated between data that have been incrementally repaired
+and data that have not. Therefore, it is important to continually run incremental repair once it has been enabled so
+newly written data can be compacted together with previously repaired data, allowing overwritten and expired data to
+be eventually purged.
+
+Running incremental repair more frequently keeps the unrepaired set smaller and thus causes repairs to operate over
+a smaller set of data, so a shorter `min_repair_interval` such as `1h` is recommended for new clusters.
+
+==== Enabling Incremental Repair on existing clusters with a large amount of data
+[#enabling-ir]
+One should be careful when enabling incremental repair on a cluster for the first time. While
+xref:#repair-token-range-splitter[RepairTokenRangeSplitter] includes a default configuration to attempt to gracefully
+migrate to incremental repair over time, failure to take proper precaution could overwhelm the cluster with
+xref:managing/operating/compaction/overview.adoc#types-of-compaction[anticompactions].
+
+No matter how one goes about enabling and running incremental repair, it is recommended to run a cycle of full repairs
+for the entire cluster as pre-flight step to running incremental repair. This will put the cluster into a more
+consistent state which will reduce the amount of streaming between replicas when incremental repair initially runs.
+
+If you do not have strong data consistency requirements, one may consider using
+xref:managing/tools/sstable/sstablerepairedset.adoc[nodetool sstablerepairedset] to mark all SSTables as repaired
+before enabling incremental repair scheduling using Auto Repair. This will reduce the burden of initially running
+incremental repair because all existing data will be considered as repaired, so subsequent incremental repairs will
+only run against new data.
+
+If you do have strong data consistency requirements, then one must treat all data as initially unrepaired and run
+incremental repair against it. Consult
+xref:#incremental-repair-defaults[RepairTokenRangeSplitter's Incremental repair defaults].
+
+In particular one should be mindful of the xref:managing/operating/compaction/overview.adoc[compaction strategy]
+you use for your tables and how it might impact incremental repair before running incremental repair for the first
+time:
+
+- *Large SSTables*: When using xref:managing/operating/compaction/stcs.adoc[SizeTieredCompactionStrategy] or any
+ compaction strategy which can create large SSTables including many partitions the amount of
+ xref:managing/operating/compaction/overview.adoc#types-of-compaction[anticompaction] that might be required could be
+ excessive. Using a small `bytes_per_assignment` might contribute to repeated anticompactions over the same
+ unrepaired data.
+- *Partitions overlapping many SSTables*: If partitions overlap between many SSTables, the amount of SSTables included
+ in a repair might be large. Therefore it is important to consider that many SSTables may be included in a repair
+ session and must all be anticompacted. xref:managing/operating/compaction/lcs.adoc[LeveledCompactionStrategy] is less
+ susceptible to this issue as it prevents overlapping of partitions within levels outside of L0, but if SSTables
+ start accumulating in L0 between incremental repairs, the cost of anticompaction will increase.
+ xref:managing/operating/compaction/ucs#sharding[UnifiedCompactionStrategy's sharding] can also be used to avoid
+ partitions overlapping SSTables.
+
+The xref:#repair-token-range-splitter[token_range_splitter] configuration for incremental repair includes a default
+configuration that attempts to conservatively migrate 100GiB of compressed data every day per node. Depending on
+requirements, data set and capability of a cluster's hardware, one may consider tuning these values to be more
+aggressive or conservative.
+
+=== Previewing Repaired Data
+
+The `preview_repaired` repair type executes repairs over the repaired data set to detect possible data inconsistencies.
+
+Inconsistencies in the repaired data set should not happen in practice and could indicate a possible bug in incremental
+repair.
+
+Running preview repairs is useful when considering using the
+xref:cassandra:managing/operating/compaction/tombstones.adoc#deletion[only_purge_repaired_tombstones] table compaction
+option to prevent data from possibly being resurrected when inconsistent replicas are missing tombstones from deletes.
+
+When enabled, the `BytesPreviewedDesynchronized` and `TokenRangesPreviewedDesynchronized`
+xref:cassandra:managing/operating/metrics.adoc#table-metrics[table metrics] can be used to detect inconsistencies in the
+repaired data set.
+
+== Configuring Auto Repair in cassandra.yaml
+
+Configuration for Auto Repair is managed in the `cassandra.yaml` file by the `auto_repair` property.
+
+A rich set of configuration exists for configuring Auto Repair with sensible defaults. However, the expectation
+is that some tuning might be needed particulary when it comes to tuning how often repair should run
+(`min_repair_interval`) and how repair assignments as created (`token_range_splitter`).
+
+The following is a practical example of an auto_repair configuration that one might use.
+
+[source, yaml]
+----
+auto_repair:
+ enabled: true
+ repair_type_overrides:
+ full:
+ enabled: true
+ min_repair_interval: 5d
+ incremental:
+ enabled: true
+ min_repair_interval: 1h
+ token_range_splitter:
+ parameters:
+ bytes_per_assignment: 50GiB
+ max_bytes_per_schedule: 100GiB
+ preview_repaired:
+ enabled: true
+ min_repair_interval: 1d
+ global_settings:
+ repair_by_keyspace: true
+ parallel_repair_count: 1
+----
+
+
+=== Top level settings
+The following settings are defined at the top level of the configuration file and apply universally across all
+repair types.
+
+[cols=",,",options="header",]
+|===
+| Name | Default | Description
+| enabled | false | Enable/Disable the auto-repair scheduler. If set to false, the scheduler thread will not be started.
+If set to true, the repair scheduler thread will be created. The thread will check for secondary configuration available
+for each repair type (full, incremental, and preview_repaired), and based on that, it will schedule repairs.
+| repair_check_interval | 5m | Time interval between successive checks to see if ongoing repairs are complete or if it
+is time to schedule repairs.
+| repair_max_retries | 3 | Maximum number of retries for a repair session.
+| history_clear_delete_hosts_buffer_interval | 2h | The scheduler needs to adjust its order when nodes leave the ring.
+Deleted hosts are tracked in metadata for a specified duration to ensure they are indeed removed before adjustments
+are made to the schedule.
+| mixed_major_version_repair_enabled | false | Enable/Disable running repairs on the cluster when there are mixed
+major versions detected, which usually occurs when the cluster is being upgraded. Repairs between nodes of
+different major versions is not something that is tested, so this may lead to data compatibility issues.
+It is strongly discouraged to set this to true without doing extensive testing beforehand.
+|===
+
+
+=== Repair level settings
+The following settings can be configured globally using `global_settings` or tailored individually for each repair
+type by using `repair_type_overrides`.
+
+[cols=",,",options="header",]
+|===
+| Name | Default | Description
+| enabled | false | Whether the given repair types should be enabled
+| min_repair_interval | 24h | Minimum duration between repairing the same node again. This is useful for tiny clusters,
+such as clusters with 5 nodes that finish repairs quickly. This means that if the scheduler completes one round on all
+nodes in less than this duration, it will not start a new repair round on a given node until this much time has
+passed since the last repair completed. Consider increasing to a larger value to reduce the impact of repairs,
+however note that one should attempt to run repairs at a smaller interval than gc_grace_seconds to
+avoid xref:cassandra:managing/operating/compaction/tombstones.adoc#zombies[data resurrection].
+| token_range_splitter.class_name | org.apache.cassandra.repair.autorepair.RepairTokenRangeSplitter | Implementation of
+IAutoRepairTokenRangeSplitter to use; responsible for splitting token ranges for repair assignments. Out of the box,
+Cassandra provides org.apache.cassandra.repair.autorepair.{RepairTokenRangeSplitter,FixedTokenRangeSplitter}.
+| repair_by_keyspace | true | If true, attempts to group tables in the same keyspace into one repair; otherwise,
+each table is repaired individually.
+| number_of_repair_threads | 1 | Number of threads to use for each repair job scheduled by the scheduler. Similar to
+the -j option in nodetool repair.
+| parallel_repair_count | 3 | Number of nodes running repair in parallel. If `parallel_repair_percentage` is set, the
+larger value is used.
+| parallel_repair_percentage | 3 | Percentage of nodes in the cluster running repair in parallel. If
+`parallel_repair_count is set`, the larger value is used.
+| allow_parallel_replica_repair | false | Whether to allow a node to take its turn running repair while one or more of
+its replicas are running repair. Defaults to false, as running repairs concurrently on replicas can increase load and
+also cause anticompaction conflicts while running incremental repair.
+| allow_parallel_replica_repair_across_schedules | true | An addition to allow_parallel_repair that also blocks repairs
+when replicas (including this node itself) are repairing in any schedule.
+For example, if a replica is executing full repairs, a value of false will prevent starting incremental repairs for this
+node. Defaults to true and is only evaluated when allow_parallel_replica_repair is false.
+| materialized_view_repair_enabled | false | Repairs materialized views if true.
+| initial_scheduler_delay | 5m | Delay before starting repairs after a node restarts to avoid repairs starting
+immediately after a restart.
+| repair_session_timeout | 3h | Timeout for retrying stuck repair sessions.
+| force_repair_new_node | false | Force immediate repair on new nodes after they join the ring.
+| sstable_upper_threshold | 50000 | Threshold to skip repairing tables with too many SSTables.
+| table_max_repair_time | 6h | Maximum time allowed for repairing one table on a given node. If exceeded, the repair
+proceeds to the next table.
+| ignore_dcs | [] | Avoid running repairs in specific data centers. By default, repairs run in all data centers. Specify
+data centers to exclude in this list. Note that repair sessions will still consider all replicas from excluded data
+centers. Useful if you have keyspaces that are not replicated in certain data centers, and you want to not run repair
+schedule in certain data centers.
+| repair_primary_token_range_only | true | Repair only the primary ranges owned by a node. Equivalent to the -pr option
+in nodetool repair. General advice is to keep this true.
+| repair_retry_backoff | 30s | Backoff time before retrying a repair session.
+| repair_task_min_duration | 5s | Minimum duration for the execution of a single repair task. This prevents the
+scheduler from overwhelming the node by scheduling too many repair tasks in a short period of time.
+|===
+
+=== `RepairTokenRangeSplitter` configuration
+[#repair-token-range-splitter]
+
+`RepairTokenRangeSplitter` is the default implementation of `IAutoRepairTokenRangeSplitter` that attempts to create
+token range assignments meeting the following goals:
+
+- *Create smaller, consistent repair times*: Long repairs, such as those lasting 15 hours, can be problematic. If a
+node fails 14 hours into the repair, the entire process must be restarted. The goal is to reduce the impact of
+disturbances or failures. However, making the repairs too short can lead to overhead from repair orchestration becoming
+the main bottleneck.
+
+- *Minimize the impact on hosts*: Repairs should not heavily affect the host systems. For incremental repairs, this
+might involve anti-compaction work. In full repairs, streaming large amounts of data—especially with wide partitions
+can lead to issues with disk usage and higher compaction costs.
+
+- *Reduce overstreaming*: The Merkle tree, which represents data within each partition and range, has a maximum size.
+If a repair covers too many partitions, the tree’s leaves represent larger data ranges. Even a small change in a leaf
+can trigger excessive data streaming, making the process inefficient.
+
+- *Reduce number of repairs*: If there are many small tables, it's beneficial to batch these tables together under a
+single parent repair. This prevents the repair overhead from becoming a bottleneck, especially when dealing with
+hundreds of tables. Running individual repairs for each table can significantly impact performance and efficiency.
+
+To achieve these goals, this implementation inspects SSTable metadata to estimate the bytes and number of partitions
+within a range and splits it accordingly to bound the size of the token ranges used for repair assignments.
+
+==== Parameter defaults
+
+The following `parameters` include the same defaults for all repair types.
+
+[cols=",,",options="header",]
+|===
+| Name | Default | Description
+| partitions_per_assignment | 1048576 | Maximum number of partitions to include in a repair
+assignment. Used to reduce number of partitions present in merkle tree leaf nodes to avoid overstreaming.
+| max_tables_per_assignment | 64 | Maximum number of tables to include in a repair assignment.
+This reduces the number of repairs, especially in keyspaces with many tables. The splitter avoids batching tables
+together if they exceed other configuration parameters like `bytes_per_assignment` or `partitions_per_assignment`.
+|===
+
+==== Full & Preview Repaired repair defaults
+
+The following `parameters` defaults are established for both `full` and `preview_repaired` repair scheduling:
+
+[cols=",,",options="header",]
+|===
+| Name | Default | Description
+| bytes_per_assignment | 50GiB | The target and maximum amount of *compressed* bytes that should be included in a
+repair assignment. *Note*: For full and preview_repaired, only the portion of an SSTable that covers the ranges
+being repaired are accounted for in this calculation.
+| max_bytes_per_schedule | 100000GiB | The maximum number of bytes to cover in an individual
+schedule. This serves as a mechanism to throttle the work done in each repair cycle. You may reduce this value if the
+impact of repairs is causing too much load on the cluster or increase it if writes outpace the amount of data being
+repaired. Alternatively, adjust the `min_repair_interval`. This is set to a large value for full repair to attempt to
+repair all data per repair schedule.
+|===
+
+==== Incremental repair defaults
+
+The following `parameters` defaults are established for `incremental` repair scheduling:
+
+[cols=",,",options="header",]
+|===
+| Name | Default | Description
+| bytes_per_assignment | 50GiB | The target and maximum amount of *compressed* bytes that should be
+included in a repair assignment. *Note*: For incremental repair, the *entire size* of *unrepaired* SSTables
+including ranges being repaired are accounted for in this calculation. This is to account for the anticompaction
+work required to split the candidate data to repair from the data that won't be repaired.
+| max_bytes_per_schedule | 100GiB | The maximum number of bytes to cover in an individual schedule.
+Consider increasing if more data is written than this limit within the `min_repair_interval`.
+|===
+
+=== `FixedSplitTokenRangeSplitter` configuration
+[#fixed-split-token-range-splitter]
+
+`FixedSplitTokenRangeSplitter` is a more simple implementation of `IAutoRepairTokenRangeSplitter` that creates repair
+assignments by splitting a node's token ranges into an even number of splits.
+
+The following `parameters` apply for `FixedSplitTokenRangeSplitter` configuration:
+
+[cols=",,",options="header",]
+|===
+| Name | Default | Description
+| number_of_subranges | 32 | Number of evenly split subranges to create for each node that repair runs for.
+If vnodes are configured using `num_tokens`, attempts to evenly subdivide subranges by each range. For example, for
+`num_tokens: 16` and `number_of_subranges: 32`, 2 (32/16) repair assignments will be created for each token range. At
+least one repair assignment will be created for each token range.
+|===
+
+=== Other cassandra.yaml Considerations
+
+==== Enable `reject_repair_compaction_threshold`
+
+When enabling auto_repair, it is advisable to configure the top level `reject_repair_compaction_threshold`
+configuration in cassandra.yaml as a backpressure mechanism to reject new repairs on instances that have many
+pending compactions.
+
+==== Tune `repair_disk_headroom_reject_ratio`
+
+By default, repairs will be rejected if less than 20% of disk is available. If one wishes to be
+conservative this top level configuration could be increased to a larger value to prevent filling your data directories.
+
+== Table configuration
+
+If Auto Repair is enabled in cassandra.yaml, the `auto_repair` property may be optionally configured at the table
+level, e.g.:
+
+[source,cql]
+----
+ALTER TABLE cycling.cyclist_races
+WITH auto_repair = {'incremental_enabled': 'false', 'priority': '0'};
+----
+
+[cols=",,",options="header",]
+|===
+| Name | Default | Description
+| priority | 0 | Indicates the priority at which this table should be given when issuing repairs. The higher the number
+the more priority will be given to repair the table (e.g. 3 will be repaired before 2). When `repair_by_keyspace` is
+set to `true` tables sharing the same priority may be grouped in the same repair assignment.
+| full_enabled | true | Whether full repair is enabled for this table. If full.enabled is not true in cassandra.yaml
+this will not be evaluated.
+| incremental_enabled | true | Whether incremental repair is enabled for this table. If incremental.enabled is not
+true in cassandra.yaml this will not be evaluated.
+| preview_repaired_enabled | true | Whether preview repair is enabled for this table. If preview_repaired.enabled is
+not true in cassandra.yaml this will not be evaluated.
+|===
+
+== Nodetool Configuration
+=== nodetool getautorepairconfig
+
+Retrieves the runtime configuration of Auto Repair for the targeted node.
+
+[source,none]
+----
+$> nodetool getautorepairconfig
+repair scheduler configuration:
+ repair_check_interval: 5m
+ repair_max_retries: 3
+ history_clear_delete_hosts_buffer_interval: 2h
+configuration for repair_type: full
+ enabled: true
+ min_repair_interval: 24h
+ repair_by_keyspace: true
+ number_of_repair_threads: 1
+ sstable_upper_threshold: 50000
+ table_max_repair_time: 6h
+ ignore_dcs: []
+ repair_primary_token_range_only: true
+ parallel_repair_count: 3
+ parallel_repair_percentage: 3
+ materialized_view_repair_enabled: false
+ initial_scheduler_delay: 5m
+ repair_session_timeout: 3h
+ force_repair_new_node: false
+ repair_retry_backoff: 30s
+ repair_task_min_duration: 5s
+ token_range_splitter: org.apache.cassandra.repair.autorepair.RepairTokenRangeSplitter
+ token_range_splitter.bytes_per_assignment: 50GiB
+ token_range_splitter.partitions_per_assignment: 1048576
+ token_range_splitter.max_tables_per_assignment: 64
+ token_range_splitter.max_bytes_per_schedule: 100000GiB
+configuration for repair_type: incremental
+ enabled: true
+ min_repair_interval: 1h
+ repair_by_keyspace: true
+ number_of_repair_threads: 1
+ sstable_upper_threshold: 50000
+ table_max_repair_time: 6h
+ ignore_dcs: []
+ repair_primary_token_range_only: true
+ parallel_repair_count: 3
+ parallel_repair_percentage: 3
+ materialized_view_repair_enabled: false
+ initial_scheduler_delay: 5m
+ repair_session_timeout: 3h
+ force_repair_new_node: false
+ repair_retry_backoff: 30s
+ repair_task_min_duration: 5s
+ token_range_splitter: org.apache.cassandra.repair.autorepair.RepairTokenRangeSplitter
+ token_range_splitter.bytes_per_assignment: 50GiB
+ token_range_splitter.partitions_per_assignment: 1048576
+ token_range_splitter.max_tables_per_assignment: 64
+ token_range_splitter.max_bytes_per_schedule: 100GiB
+configuration for repair_type: preview_repaired
+ enabled: false
+----
+
+=== nodetool autorepairstatus
+
+Provides currently running Auto Repair status.
+
+[source,none]
+----
+$> nodetool autorepairstatus -t incremental
+Active Repairs
+425cea55-09aa-46e0-8911-9f37a4424574
+
+
+$> nodetool autorepairstatus -t full
+Active Repairs
+NONE
+
+----
+
+=== nodetool setautorepairconfig
+
+Dynamic configuration changes can be made by using `setautorepairconfig`. Note that this only applies on the node being
+targeted and these changes are not retained when a node is bounced.
+
+The following disables the `incremental` repair schedule:
+
+[source,none]
+----
+$> nodetool setautorepairconfig -t incremental enabled false
+----
+
+The following adjusts the `min_repair_interval` option to `5d` specifically for the `full` repair schedule:
+
+[source,none]
+----
+$> nodetool setautorepairconfig -t full min_repair_interval 5d
+----
+
+The following configures the `bytes_per_assignment` parameter for `incremental` repair's `token_range_splitter` to
+`10GiB`:
+
+[source,none]
+----
+$> nodetool setautorepairconfig -t incremental token_range_splitter.bytes_per_assignment 10GiB
+----
+
+==== More details
+https://cwiki.apache.org/confluence/display/CASSANDRA/CEP-37+Apache+Cassandra+Unified+Repair+Solution[CEP-37]
diff --git a/doc/modules/cassandra/pages/managing/operating/bulk_loading.adoc b/doc/modules/cassandra/pages/managing/operating/bulk_loading.adoc
index 2630fbf76c4a..fbddb149dd94 100644
--- a/doc/modules/cassandra/pages/managing/operating/bulk_loading.adoc
+++ b/doc/modules/cassandra/pages/managing/operating/bulk_loading.adoc
@@ -25,7 +25,7 @@ The `sstableloader` and `nodetool import` are accessible if the
Cassandra installation `bin` directory is in the `PATH` environment
variable.
Or these may be accessed directly from the `bin` directory.
-The examples use the keyspaces and tables created in xref:cassandra:developing/cql/operating/backups.adoc[Backups].
+The examples use the keyspaces and tables created in xref:cassandra:managing/operating/backups.adoc[Backups].
== Using sstableloader
diff --git a/doc/modules/cassandra/pages/managing/operating/compaction/overview.adoc b/doc/modules/cassandra/pages/managing/operating/compaction/overview.adoc
index 3fdbb6d5a7d7..6a396106b8ee 100644
--- a/doc/modules/cassandra/pages/managing/operating/compaction/overview.adoc
+++ b/doc/modules/cassandra/pages/managing/operating/compaction/overview.adoc
@@ -2,18 +2,18 @@
== What is compaction?
-Data in {cassandra} is created in xref:cassandra:architecture/storage-engine.adoc#memtables[memtables].
+Data in {cassandra} is created in xref:cassandra:architecture/storage-engine.adoc#memtables[memtables].
Once a memory threshold is reached, to free up memory again, the data is written to an xref:cassandra:architecture/storage-engine.adoc#SSTables[SSTable], an https://cassandra.apache.org/_/glossary.html#immutable[immutable] file residing on disk.
-Because SSTables are immutable, when data is updated or deleted, the old data is not overwritten with inserts or updates, or removed from the SSTable.
-Instead, a new SSTable is created with the updated data with a new timestamp, and the old SSTable is marked for deletion.
+Because SSTables are immutable, when data is updated or deleted, the old data is not overwritten with inserts or updates, or removed from the SSTable.
+Instead, a new SSTable is created with the updated data with a new timestamp, and the old SSTable is marked for deletion.
The piece of deleted data is known as a https://cassandra.apache.org/_/glossary.html#tombstone[tombstone].
-Over time, Cassandra may write many versions of a row in different SSTables.
-Each version may have a unique set of columns stored with a different timestamp.
+Over time, Cassandra may write many versions of a row in different SSTables.
+Each version may have a unique set of columns stored with a different timestamp.
As SSTables accumulate, the distribution of data can require accessing more and more SSTables to retrieve a complete row.
-To keep the database healthy, Cassandra periodically merges SSTables and discards old data.
+To keep the database healthy, Cassandra periodically merges SSTables and discards old data.
This process is called https://cassandra.apache.org/_/glossary.html#compaction[compaction].
== Why must compaction be run?
@@ -26,22 +26,22 @@ Deleting, updating, or expiring data are all valid triggers for compaction.
== What does compaction accomplish?
Two important factors accomplished by compaction are performance improvement and disk space reclamation.
-If SSTables have duplicate data that must be read, read operations are slower.
+If SSTables have duplicate data that must be read, read operations are slower.
Once tombstones and duplicates are removed, read operations are faster.
SSTables use disk space, and reducing the size of SSTables through compaction frees up disk space.
== How does compaction work?
-Compaction works on a collection of SSTables.
-From these SSTables, compaction collects all versions of each unique row and assembles one complete row, using the most up-to-date version (by timestamp) of each of the row's columns.
-The merge process is performant, because rows are sorted by partition key within each SSTable, and the merge process does not use random I/O.
-The new versions of each row is written to a new SSTable.
+Compaction works on a collection of SSTables.
+From these SSTables, compaction collects all versions of each unique row and assembles one complete row, using the most up-to-date version (by timestamp) of each of the row's columns.
+The merge process is performant, because rows are sorted by partition key within each SSTable, and the merge process does not use random I/O.
+The new versions of each row is written to a new SSTable.
The old versions, along with any rows that are ready for deletion, are left in the old SSTables, and are deleted as soon as pending reads are completed.
== Types of compaction
The concept of compaction is used for different kinds of operations in
-{cassandra}, the common thing about these operations is that it takes one
+{cassandra}, the common thing about these operations is that they take one
or more SSTables, merges, and outputs new SSTables. The types of compactions are:
Minor compaction::
@@ -56,11 +56,11 @@ A major compaction is triggered when a user executes a compaction over all SSTab
User defined compaction::
Similar to a major compaction, a user-defined compaction executes when a user triggers a compaction on a given set of SSTables.
Scrub::
-A scrub triggers a compaction to try to fix any broken SSTables.
+A scrub triggers a compaction to try to fix any broken SSTables.
This can actually remove valid data if that data is corrupted.
If that happens you will need to run a full repair on the node.
UpgradeSSTables::
-A compaction occurs when you upgrade SSTables to the latest version.
+A compaction occurs when you upgrade SSTables to the latest version.
Run this after upgrading to a new major version.
Cleanup::
Compaction executes to remove any ranges that a node no longer owns.
@@ -71,8 +71,8 @@ Anticompaction::
After repair, the ranges that were actually repaired are split out of the SSTables that existed when repair started. This type of compaction rewrites SSTables to accomplish this task.
Sub range compaction::
It is possible to only compact a given sub range - this action is useful if you know a token that has been misbehaving - either gathering many updates or many deletes.
-The command `nodetool compact -st x -et y` will pick all SSTables containing the range between x and y and issue a compaction for those SSTables.
-For Size Tiered Compaction Strategy, this will most likely include all SSTables, but with Leveled Compaction Strategy, it can issue the compaction for a subset of the SSTables.
+The command `nodetool compact -st x -et y` will pick all SSTables containing the range between x and y and issue a compaction for those SSTables.
+For Size Tiered Compaction Strategy, this will most likely include all SSTables, but with Leveled Compaction Strategy, it can issue the compaction for a subset of the SSTables.
With LCS the resulting SSTable will end up in L0.
== Strategies
@@ -82,14 +82,14 @@ Picking the right compaction strategy for your workload will ensure the best per
xref:cassandra:managing/operating/compaction/ucs.adoc[`Unified Compaction Strategy (UCS)`]::
UCS is a good choice for most workloads and is recommended for new workloads.
-This compaction strategy is designed to handle a wide variety of workloads.
-It is designed to be able to handle both immutable time-series data and workloads with lots of updates and deletes.
-It is also designed to be able to handle both spinning disks and SSDs.
-xref:cassandra:managing/operating/compaction/stcs.adoc[`Size Tiered Compaction Strategy (STCS)`]::
-STCS is the default compaction strategy, because it is useful as a fallback when other strategies don't fit the workload.
+This compaction strategy is designed to handle a wide variety of workloads.
+It is designed to be able to handle both immutable time-series data and workloads with lots of updates and deletes.
+It is also designed to be able to handle both spinning disks and SSDs.
+xref:cassandra:managing/operating/compaction/stcs.adoc[`Size Tiered Compaction Strategy (STCS)`]::
+STCS is the default compaction strategy, because it is useful as a fallback when other strategies don't fit the workload.
Most useful for not strictly time-series workloads with spinning disks, or when the I/O from `LCS` is too high.
xref:cassandra:managing/operating/compaction/lcs.adoc[`Leveled Compaction Strategy (LCS)`]::
-Leveled Compaction Strategy (LCS) is optimized for read heavy workloads, or workloads with lots of updates and deletes.
+Leveled Compaction Strategy (LCS) is optimized for read heavy workloads, or workloads with lots of updates and deletes.
It is not a good choice for immutable time-series data.
xref:cassandra:managing/operating/compaction/twcs.adoc[`Time Window Compaction Strategy (TWCS)`]::
Time Window Compaction Strategy is designed for TTL'ed, mostly immutable time-series data.
@@ -107,19 +107,6 @@ of the TTL) Cassandra will have a hard time dropping the tombstones
created since the partition might span many SSTables and not all are
compacted at once.
-== Fully expired SSTables
-
-If an SSTable contains only tombstones and it is guaranteed that
-SSTable is not shadowing data in any other SSTable, then the compaction can drop
-that SSTable. If you see SSTables with only tombstones (note that TTL-ed
-data is considered tombstones once the time-to-live has expired), but it
-is not being dropped by compaction, it is likely that other SSTables
-contain older data. There is a tool called `sstableexpiredblockers` that
-will list which SSTables are droppable and which are blocking them from
-being dropped. With `TimeWindowCompactionStrategy` it
-is possible to remove the guarantee (not check for shadowing data) by
-enabling `unsafe_aggressive_sstable_expiration`.
-
== Repaired/unrepaired data
With incremental repairs Cassandra must keep track of what data is
@@ -161,8 +148,8 @@ When an SSTable is written a histogram with the tombstone expiry times
is created and this is used to try to find SSTables with very many
tombstones and run single SSTable compaction on that SSTable in hope of
being able to drop tombstones in that SSTable. Before starting this it
-is also checked how likely it is that any tombstones will actually will
-be able to be dropped how much this SSTable overlaps with other
+is also checked how likely it is that any tombstones will actually
+be able to be dropped and how much this SSTable overlaps with other
SSTables. To avoid most of these checks the compaction option
`unchecked_tombstone_compaction` can be enabled.
@@ -178,11 +165,11 @@ How much of the SSTable should be tombstones for us to consider doing a single S
`tombstone_compaction_interval` (default: 86400s (1 day))::
Since it might not be possible to drop any tombstones when doing a single SSTable compaction we need to make sure that one SSTable is not constantly getting recompacted - this option states how often we should try for a given SSTable.
`log_all` (default: false)::
-New detailed compaction logging, see `below `.
+New detailed compaction logging, see <>.
`unchecked_tombstone_compaction` (default: false)::
-The single SSTable compaction has quite strict checks for whether it should be started, this option disables those checks and for some use cases this might be needed.
+The single SSTable compaction has quite strict checks for whether it should be started, this option disables those checks and for some use cases this might be needed.
Note that this does not change anything for the actual compaction, tombstones are only dropped if it is safe to do so - it might just rewrite an SSTable without being able to drop any tombstones.
-`only_purge_repaired_tombstone` (default: false)::
+`only_purge_repaired_tombstones` (default: false)::
Option to enable the extra safety of making sure that tombstones are only dropped if the data has been repaired.
`min_threshold` (default: 4)::
Lower limit of number of SSTables before a compaction is triggered.
@@ -195,7 +182,7 @@ Further, see the section on each strategy for specific additional options.
== Compaction nodetool commands
-The `nodetool ` utility provides a number of commands related to compaction:
+The `nodetool` utility provides a number of commands related to compaction:
`enableautocompaction`::
Enable compaction.
@@ -212,7 +199,7 @@ Set the min/max SSTable count for when to trigger compaction, defaults to 4/32.
== Switching the compaction strategy and options using JMX
-It is possible to switch compaction strategies and its options on just a single node using JMX, this is a great way to experiment with settings without affecting the whole cluster.
+It is possible to switch compaction strategies and its options on just a single node using JMX, this is a great way to experiment with settings without affecting the whole cluster.
The mbean is:
[source,console]
diff --git a/doc/modules/cassandra/pages/managing/operating/compaction/tombstones.adoc b/doc/modules/cassandra/pages/managing/operating/compaction/tombstones.adoc
index 9e0dcb6f7879..e48c02d209df 100644
--- a/doc/modules/cassandra/pages/managing/operating/compaction/tombstones.adoc
+++ b/doc/modules/cassandra/pages/managing/operating/compaction/tombstones.adoc
@@ -69,22 +69,21 @@ This is basically the same as in the "Deletes without Tombstones" section.
=== Deletes without tombstones
-Imagine a three node cluster which has the value [A] replicated to every
-node.:
+Imagine a three node cluster which has the value [A] replicated to every node:
[source,none]
----
[A], [A], [A]
----
-If one of the nodes fails and and our delete operation only removes existing values, we can end up with a cluster that looks like:
+If one of the nodes fails and our delete operation only removes existing values, we can end up with a cluster that looks like:
[source,none]
----
[], [], [A]
----
-Then a repair operation would replace the value of [A] back onto the two nodes which are missing the value.:
+Then a repair operation would replace the value of [A] back onto the two nodes which are missing the value:
[source,none]
----
@@ -95,7 +94,7 @@ This would cause our data to be resurrected as a zombie even though it had been
=== Deletes with tombstones
-Starting again with a three node cluster which has the value [A] replicated to every node.:
+Starting again with a three node cluster which has the value [A] replicated to every node:
[source,none]
----
diff --git a/doc/modules/cassandra/pages/managing/operating/security.adoc b/doc/modules/cassandra/pages/managing/operating/security.adoc
index 1de19d86414d..783cac6d60c4 100644
--- a/doc/modules/cassandra/pages/managing/operating/security.adoc
+++ b/doc/modules/cassandra/pages/managing/operating/security.adoc
@@ -281,7 +281,7 @@ xref:cassandra:developing/cql/security.adoc#operation-roles[`CassandraRoleManage
See also: `setting-credentials-for-internal-authentication`,
xref:cassandra:developing/cql/security.adoc#create-role[`CREATE ROLE`],
xref:cassandra:developing/cql/security.adoc#alter-role[`ALTER ROLE`],
-xref:xref:cassandra:developing/cql/security.adoc#alter-keyspace[`ALTER KEYSPACE`] and
+xref:cassandra:developing/cql/ddl.adoc#alter-keyspace-statement[`ALTER KEYSPACE`] and
xref:cassandra:developing/cql/security.adoc#grant-permission[`GRANT PERMISSION`].
== Authorization
@@ -411,7 +411,7 @@ If enabling remote connections, it is recommended to also use
xref:cassandra:managing/operating/security.adoc#jmx-with-ssl[`SSL`] connections.
Finally, after enabling auth and/or SSL, ensure that tools which use
-JMX, such as xref:tools/nodetool/nodetools.adoc[`nodetool`] are correctly configured and working
+JMX, such as xref:cassandra:managing/tools/nodetool/nodetool.adoc[`nodetool`] are correctly configured and working
as expected.
=== Standard JMX Auth
diff --git a/doc/modules/cassandra/pages/managing/tools/cqlsh.adoc b/doc/modules/cassandra/pages/managing/tools/cqlsh.adoc
index 61ab25de3358..7143706058eb 100644
--- a/doc/modules/cassandra/pages/managing/tools/cqlsh.adoc
+++ b/doc/modules/cassandra/pages/managing/tools/cqlsh.adoc
@@ -22,7 +22,7 @@ of `cqlsh`.
By default, `cqlsh` displays all timestamps with a UTC timezone.
For Python 3.9 or higher, timestamps can be displayed in different timezones by modifying the
-`timezone` option in xref:cassandra:developing/cql/tools/cqlsh.adoc#cqlshrc[cqlshrc] or by setting the environment
+`timezone` option in xref:cassandra:managing/tools/cqlsh.adoc#cqlshrc[cqlshrc] or by setting the environment
variable `TZ`.
Python 3.8 or lower, however, will also require the installation of http://pytz.sourceforge.net/[pytz] library.
diff --git a/doc/modules/cassandra/pages/new/index.adoc b/doc/modules/cassandra/pages/new/index.adoc
index 5d36e59b73b5..d2516661967e 100644
--- a/doc/modules/cassandra/pages/new/index.adoc
+++ b/doc/modules/cassandra/pages/new/index.adoc
@@ -10,7 +10,7 @@ This section covers the new features in Apache Cassandra 5.0.
* Trie SSTables: https://cwiki.apache.org/confluence/display/CASSANDRA/CEP-25%3A+Trie-indexed+SSTable+format[CEP-25], https://issues.apache.org/jira/browse/CASSANDRA-18398[JIRA ticket]
* JDK 17: xref:cassandra:reference/java17.adoc[Docs], https://issues.apache.org/jira/browse/CASSANDRA-16895[JIRA ticket]
* More guardrails: https://github.com/apache/cassandra/blob/trunk/NEWS.txt[NEWS.txt]
-* TTL and writetime on collections and UDTs: xref:cassandra:developing/cql/dml.html#writetime-and-ttl-function[Docs], https://issues.apache.org/jira/browse/CASSANDRA-8877[JIRA ticket]
+* TTL and writetime on collections and UDTs: xref:cassandra:developing/cql/functions.adoc#writetime-and-ttl-functions[Docs], https://issues.apache.org/jira/browse/CASSANDRA-8877[JIRA ticket]
* New vector data type: xref:cassandra:reference/vector-data-type.adoc[Docs], https://cwiki.apache.org/confluence/display/CASSANDRA/CEP-30%3A+Approximate+Nearest+Neighbor%28ANN%29+Vector+Search+via+Storage-Attached+Indexes[CEP-30], https://issues.apache.org/jira/browse/CASSANDRA-18504[JIRA ticket]
* New vector similarity functions: xref:cassandra:vector-search/overview.adoc[Docs], https://cwiki.apache.org/confluence/display/CASSANDRA/CEP-30%3A+Approximate+Nearest+Neighbor%28ANN%29+Vector+Search+via+Storage-Attached+Indexes[CEP-30], https://issues.apache.org/jira/browse/CASSANDRA-18640[JIRA ticket]
* Unified Compaction Strategy: xref:cassandra:managing/operating/compaction/ucs.adoc[Docs], https://cwiki.apache.org/confluence/display/CASSANDRA/CEP-26%3A+Unified+Compaction+Strategy[CEP-26], https://issues.apache.org/jira/browse/CASSANDRA-18397[JIRA ticket]
diff --git a/doc/modules/cassandra/pages/reference/cql-commands/alter-table.adoc b/doc/modules/cassandra/pages/reference/cql-commands/alter-table.adoc
index 2c6071c52944..be4dff203927 100644
--- a/doc/modules/cassandra/pages/reference/cql-commands/alter-table.adoc
+++ b/doc/modules/cassandra/pages/reference/cql-commands/alter-table.adoc
@@ -98,7 +98,7 @@ Enclose the value for a string property in single quotation marks.
+
Other table properties are set using a JSON map: `+option_name = { : [ , ... ] }+`
+
-See xref:reference:cql-commands/create-table.adoc#table_options[table_options] for more details.
+See xref:cassandra:reference/cql-commands/create-table.adoc#table_options[table_options] for more details.
== Usage notes
@@ -190,7 +190,7 @@ To change an existing table's properties, use `ALTER TABLE` and `WITH`.
You can specify a:
* Single property name and value.
-* Property map to set the names and values, as shown in the xref:cql-commands/alter-table.adoc#alter-compression[next section on compression and compaction].
+* Property map to set the names and values, as shown in the xref:cassandra:reference/cql-commands/alter-table.adoc#alter-compression[next section on compression and compaction].
For example, to add a comment to the xref:cassandra:cyclist_base-table.adoc[cyclist_base] table using WITH:
@@ -204,7 +204,7 @@ Enclose a text property value in single quotation marks.
[[alter-compression]]
=== Modifying compression and compaction
-Use a property map to alter the xref:cassandra:comments-table.adoc[comments] table's compression or compaction setting:
+Use a property map to alter the comments table's compression or compaction setting:
[source,language-cql]
----
@@ -223,7 +223,7 @@ For strategies to minimize this disruption, see http://blog.alteroot.org/article
=== Changing caching
-Set the number of rows per partition to store in the row cache for the xref:cassandra:comments-table.adoc[comments] table to 10 rows:
+Set the number of rows per partition to store in the row cache for the comments table to 10 rows:
[source,language-cql]
----
diff --git a/doc/modules/cassandra/pages/reference/cql-commands/commands-toc.adoc b/doc/modules/cassandra/pages/reference/cql-commands/commands-toc.adoc
index 89ec71787679..b5f087671c3b 100644
--- a/doc/modules/cassandra/pages/reference/cql-commands/commands-toc.adoc
+++ b/doc/modules/cassandra/pages/reference/cql-commands/commands-toc.adoc
@@ -4,123 +4,111 @@
This section describes the Cassandra Query Language (CQL) commands supported by the {product} database.
'''
-xref:reference/cql-commands/alter-keyspace.adoc[ALTER KEYSPACE] ::
+xref:cassandra:developing/cql/ddl.adoc#alter-keyspace-statement[ALTER KEYSPACE] ::
Changes keyspace replication strategy and enables or disables commit log.
-xref:reference/cql-commands/alter-materialized-view.adoc[ALTER MATERIALIZED VIEW] ::
+xref:cassandra:developing/cql/mvs.adoc#alter-materialized-view-statement[ALTER MATERIALIZED VIEW] ::
Changes the table properties of a materialized view.
-xref:reference/cql-commands/alter-role.adoc[ALTER ROLE] ::
+xref:cassandra:developing/cql/security.adoc#alter-role-statement[ALTER ROLE] ::
Changes password and sets superuser or login options.
-xref:reference/cql-commands/alter-table.adoc[ALTER TABLE] ::
+xref:cassandra:reference/cql-commands/alter-table.adoc[ALTER TABLE] ::
Modifies the columns and properties of a table, or modify
-xref:reference/cql-commands/alter-type.adoc[ALTER TYPE] ::
+xref:cassandra:developing/cql/types.adoc#udts[ALTER TYPE] ::
Modifies an existing user-defined type (UDT).
-xref:reference/cql-commands/alter-user.adoc[ALTER USER (Deprecated)] ::
+xref:cassandra:developing/cql/security.adoc#alter-user-statement[ALTER USER (Deprecated)] ::
Deprecated. Alter existing user options.
-xref:reference/cql-commands/batch.adoc[BATCH] ::
+xref:cassandra:developing/cql/dml.adoc#batch_statement[BATCH] ::
Applies multiple data modification language (DML) statements with atomicity and/or in isolation.
-xref:reference/cql-commands/create-aggregate.adoc[CREATE AGGREGATE] ::
+xref:cassandra:developing/cql/functions.adoc#create-aggregate-statement[CREATE AGGREGATE] ::
Defines a user-defined aggregate.
-xref:reference/cql-commands/create-custom-index.adoc[CREATE CUSTOM INDEX] ::
+xref:cassandra:reference/cql-commands/create-custom-index.adoc[CREATE CUSTOM INDEX] ::
Creates a storage-attached index.
-xref:reference/cql-commands/create-function.adoc[CREATE FUNCTION] ::
+xref:cassandra:developing/cql/functions.adoc#create-function-statement[CREATE FUNCTION] ::
Creates custom function to execute user provided code.
-xref:reference/cql-commands/create-index.adoc[CREATE INDEX] ::
+xref:cassandra:reference/cql-commands/create-index.adoc[CREATE INDEX] ::
Defines a new index for a single column of a table.
-xref:reference/cql-commands/create-keyspace.adoc[CREATE KEYSPACE] ::
+xref:cassandra:developing/cql/ddl.adoc#create-keyspace-statement[CREATE KEYSPACE] ::
-xref:reference/cql-commands/create-materialized-view.adoc[CREATE MATERIALIZED VIEW] ::
+xref:cassandra:developing/cql/mvs.adoc#create-materialized-view-statement[CREATE MATERIALIZED VIEW] ::
Optimizes read requests and eliminates the need for multiple write requests by duplicating data from a base table.
-xref:reference/cql-commands/create-role.adoc[CREATE ROLE] ::
+xref:cassandra:developing/cql/security.adoc#create-role-statement[CREATE ROLE] ::
Creates a cluster wide database object used for access control.
-xref:reference/cql-commands/create-table.adoc[CREATE TABLE] ::
+xref:cassandra:reference/cql-commands/create-table.adoc[CREATE TABLE] ::
Creates a new table.
-xref:reference/cql-commands/create-type.adoc[CREATE TYPE] ::
+xref:cassandra:developing/cql/types.adoc#udts[CREATE TYPE] ::
Creates a custom data type in the keyspace that contains one or more fields of related information.
-xref:reference/cql-commands/create-user.adoc[CREATE USER (Deprecated)] ::
+xref:cassandra:developing/cql/security.adoc#create-user-statement[CREATE USER (Deprecated)] ::
Deprecated. Creates a new user.
-xref:reference/cql-commands/delete.adoc[DELETE] ::
+xref:cassandra:developing/cql/dml.adoc#delete_statement[DELETE] ::
Removes data from one or more columns or removes the entire row
-xref:reference/cql-commands/drop-aggregate.adoc[DROP AGGREGATE] ::
+xref:cassandra:developing/cql/functions.adoc#drop-aggregate-statement[DROP AGGREGATE] ::
Deletes a user-defined aggregate from a keyspace.
-xref:reference/cql-commands/drop-function.adoc[DROP FUNCTION] ::
+xref:cassandra:developing/cql/functions.adoc#drop-function-statement[DROP FUNCTION] ::
Deletes a user-defined function (UDF) from a keyspace.
-xref:reference/cql-commands/drop-index.adoc[DROP INDEX] ::
+xref:cassandra:reference/cql-commands/drop-index.adoc[DROP INDEX] ::
Removes an index from a table.
-xref:reference/cql-commands/drop-keyspace.adoc[DROP KEYSPACE] ::
+xref:cassandra:developing/cql/ddl.adoc#drop-keyspace-statement[DROP KEYSPACE] ::
Removes the keyspace.
-xref:reference/cql-commands/drop-materialized-view.adoc[DROP MATERIALIZED VIEW] ::
+xref:cassandra:developing/cql/mvs.adoc#drop-materialized-view-statement[DROP MATERIALIZED VIEW] ::
Removes the named materialized view.
-xref:reference/cql-commands/drop-role.adoc[DROP ROLE] ::
+xref:cassandra:developing/cql/security.adoc#drop-role-statement[DROP ROLE] ::
Removes a role.
-xref:reference/cql-commands/drop-table.adoc[DROP TABLE] ::
+xref:cassandra:reference/cql-commands/drop-table.adoc[DROP TABLE] ::
Removes the table.
-xref:reference/cql-commands/drop-type.adoc[DROP TYPE] ::
+xref:cassandra:developing/cql/types.adoc#udts[DROP TYPE] ::
Drop a user-defined type.
-xref:reference/cql-commands/drop-user.adoc[DROP USER (Deprecated)] ::
+xref:cassandra:developing/cql/security.adoc#drop-user-statement[DROP USER (Deprecated)] ::
Removes a user.
-xref:reference/cql-commands/grant.adoc[GRANT] ::
+xref:cassandra:developing/cql/security.adoc#grant-permission-statement[GRANT] ::
Allow access to database resources.
-xref:reference/cql-commands/insert.adoc[INSERT] ::
+xref:cassandra:developing/cql/dml.adoc#insert-statement[INSERT] ::
Inserts an entire row or upserts data into existing rows.
-xref:reference/cql-commands/list-permissions.adoc[LIST PERMISSIONS] ::
+xref:cassandra:developing/cql/security.adoc#list-permissions-statement[LIST PERMISSIONS] ::
Lists permissions on resources.
-xref:reference/cql-commands/list-roles.adoc[LIST ROLES] ::
+xref:cassandra:developing/cql/security.adoc#list-roles-statement[LIST ROLES] ::
Lists roles and shows superuser and login status.
-xref:reference/cql-commands/list-users.adoc[LIST USERS (Deprecated)] ::
+xref:cassandra:developing/cql/security.adoc#list-users-statement[LIST USERS (Deprecated)] ::
Lists existing internal authentication users and their superuser status.
-xref:reference/cql-commands/restrict.adoc[RESTRICT] ::
-Denies the permission on a resource, even if the role is directly granted or inherits permissions.
-
-xref:reference/cql-commands/restrict-rows.adoc[RESTRICT ROWS] ::
-Configures the column used for row-level access control.
-
-xref:reference/cql-commands/revoke.adoc[REVOKE] ::
+xref:cassandra:developing/cql/security.adoc#revoke-permission-statement[REVOKE] ::
Removes privileges on database objects from roles.
-xref:reference/cql-commands/select.adoc[SELECT] ::
+xref:cassandra:developing/cql/dml.adoc#select-statement[SELECT] ::
Returns data from a table.
-xref:reference/cql-commands/truncate.adoc[TRUNCATE] ::
+xref:cassandra:developing/cql/ddl.adoc#truncate-statement[TRUNCATE] ::
Removes all data from a table.
-xref:reference/cql-commands/unrestrict.adoc[UNRESTRICT] ::
-Removes a restriction from a role.
-
-xref:reference/cql-commands/unrestrict-rows.adoc[UNRESTRICT ROWS] ::
-Removes the column definition for row-level access control.
-
-xref:reference/cql-commands/update.adoc[UPDATE] ::
+xref:cassandra:developing/cql/dml.adoc#update-statement[UPDATE] ::
Modifies one or more column values to a row in a table.
-xref:reference/cql-commands/use.adoc[USE] ::
-Selects the keyspace for the current client session.
\ No newline at end of file
+xref:cassandra:developing/cql/ddl.adoc#use-statement[USE] ::
+Selects the keyspace for the current client session.
diff --git a/doc/modules/cassandra/pages/reference/cql-commands/compact-subproperties.adoc b/doc/modules/cassandra/pages/reference/cql-commands/compact-subproperties.adoc
index 3df664bbc172..91763fd0e790 100644
--- a/doc/modules/cassandra/pages/reference/cql-commands/compact-subproperties.adoc
+++ b/doc/modules/cassandra/pages/reference/cql-commands/compact-subproperties.adoc
@@ -235,7 +235,7 @@ Default: `160`
====
The default value, 160 MB, may be inefficient and negatively impact database indexing and the queries that rely on indexes.
For example, consider the benefit of using higher values for sstable_size_in_mb in tables that use (SAI) indexes.
-For related information, see xref:developing:indexing/sai/configuring.adoc#saiConfigure__saiCompactionStrategies[Compaction strategies].
+For related information, see xref:cassandra:developing/cql/indexing/sai/operations/configuring.adoc#saiConfigure__saiCompactionStrategies[Compaction strategies].
====
fanout_size::
diff --git a/doc/modules/cassandra/pages/reference/cql-commands/create-custom-index.adoc b/doc/modules/cassandra/pages/reference/cql-commands/create-custom-index.adoc
index b4dc08566f0a..85c56721b7d4 100644
--- a/doc/modules/cassandra/pages/reference/cql-commands/create-custom-index.adoc
+++ b/doc/modules/cassandra/pages/reference/cql-commands/create-custom-index.adoc
@@ -54,6 +54,8 @@ index_name::
Optional identifier for index.
If no name is specified, the default used is `\_\_idx`.
Enclose in quotes to use special characters or to preserve capitalization.
++
+include::cassandra:partial$index-naming.adoc[]
column_name::
The name of the table column on which the SAI index is being defined.
diff --git a/doc/modules/cassandra/pages/reference/cql-commands/create-index.adoc b/doc/modules/cassandra/pages/reference/cql-commands/create-index.adoc
index ae95c33a1d50..5b191709e5bf 100644
--- a/doc/modules/cassandra/pages/reference/cql-commands/create-index.adoc
+++ b/doc/modules/cassandra/pages/reference/cql-commands/create-index.adoc
@@ -65,7 +65,7 @@ SAI returns `InvalidRequestException` if you try to define an index on a column
== Optional parameters
-[cols="1,3"]
+[cols="1,3"]
|===
| Parameter | Description
@@ -75,6 +75,8 @@ SAI returns `InvalidRequestException` if you try to define an index on a column
Enclose in quotes to use special characters or preserve capitalization.
If no name is specified, {product} names the index as `\_\_idx`.
+include::cassandra:partial$index-naming.adoc[]
+
| keyspace_name
| Name of the keyspace that contains the table to index.
If no name is specified, the current keyspace is used.
@@ -248,7 +250,7 @@ Assume a cyclist table contains this map data where `nation` is the map key and
----
To index map keys, use the `KEYS` keyword and map name in nested parentheses in the CREATE INDEX statement.
-To run a `SELECT` query on the table, use xref:cassandra:reference/cql-commands/select.adoc#filtering-on-collections[CONTAINS KEY] in `WHERE` clauses.
+To run a `SELECT` query on the table, use xref:cassandra:developing/cql/dml.adoc#allow-filtering[CONTAINS KEY] in `WHERE` clauses.
This query returns cyclist teams that have an entry for the year 2015.
[tabs]
diff --git a/doc/modules/cassandra/pages/reference/cql-commands/create-table-examples.adoc b/doc/modules/cassandra/pages/reference/cql-commands/create-table-examples.adoc
index d3487202b56e..b037b68e2ae0 100644
--- a/doc/modules/cassandra/pages/reference/cql-commands/create-table-examples.adoc
+++ b/doc/modules/cassandra/pages/reference/cql-commands/create-table-examples.adoc
@@ -46,9 +46,9 @@ Create the `race_winners` table that has a frozen user-defined type (UDT):
include::cassandra:example$CQL/race_winners-table.cql[tag=usetype]
----
-See xref:developing/user-defined-type-create.adoc[Create a user-defined type] for information on Create UDTs.
+See xref:cassandra:developing/cql/types.adoc#udts[Create a user-defined type] for information on Create UDTs.
UDTs can be created unfrozen if only non-collection fields are used in the user-defined type creation.
-If the table is created with an unfrozen UDT, then xref:developing/inserting/insert-user-defined-type.adoc[individual field values can be updated and deleted].
+If the table is created with an unfrozen UDT, then xref:cassandra:developing/cql/dml.adoc#update-statement[individual field values can be updated and deleted].
== Create a table with a CDC log
diff --git a/doc/modules/cassandra/pages/reference/cql-commands/create-table.adoc b/doc/modules/cassandra/pages/reference/cql-commands/create-table.adoc
index 69cb7cc7431a..5b21af328c4e 100644
--- a/doc/modules/cassandra/pages/reference/cql-commands/create-table.adoc
+++ b/doc/modules/cassandra/pages/reference/cql-commands/create-table.adoc
@@ -67,6 +67,7 @@ include::cassandra:partial$compress-subproperties.adoc[]
include::cassandra:partial$compact-subproperties.adoc[]
+[[table_options]]
== Optional parameters
// Table Keywords
@@ -88,7 +89,7 @@ If the column already contains data, it is indexed during the execution of this
After an index has been created, it is automatically updated when data in the column changes.
Indexing with the `CREATE INDEX` command can impact performance.
-Before creating an index, be aware of when and xref:cassandra:developing/indexing/2i/2i-when-to-use.adoc#when-no-index[when not to create an index].
+Before creating an index, be aware of when and xref:cassandra:developing/cql/indexing/2i/2i-when-to-use.adoc#when-no-index[when not to create an index].
*Restriction:* Indexing counter columns is not supported.
diff --git a/doc/modules/cassandra/pages/reference/index.adoc b/doc/modules/cassandra/pages/reference/index.adoc
index 8e09e8f8d804..642b9e081058 100644
--- a/doc/modules/cassandra/pages/reference/index.adoc
+++ b/doc/modules/cassandra/pages/reference/index.adoc
@@ -1,7 +1,7 @@
= Reference
* xref:reference/cql-commands/commands-toc.adoc[CQL commands]
-* xref:developing/cql/cql_singlefile.html[CQL specification]
+* xref:cassandra:developing/cql/cql_singlefile.adoc[CQL specification]
* xref:reference/java17.adoc[Java 17]
* xref:reference/native-protocol.adoc[Native Protocol specification]
* xref:reference/sai-virtual-table-indexes.adoc[SAI virtual table]
diff --git a/doc/modules/cassandra/pages/reference/static.adoc b/doc/modules/cassandra/pages/reference/static.adoc
index d27adc76ea5f..96a9f7bf13ef 100644
--- a/doc/modules/cassandra/pages/reference/static.adoc
+++ b/doc/modules/cassandra/pages/reference/static.adoc
@@ -48,7 +48,7 @@ The table that does not have clustering columns has a one-row partition in which
* A column designated to be the partition key cannot be static.
====
-You can do xref:developing/batch/batch-good-example.adoc[batch conditional updates to a static column].
+You can do xref:cassandra:developing/cql/batch/batch-good-example.adoc[batch conditional updates to a static column].
Use the `DISTINCT` keyword to select static columns.
In this case, the database retrieves only the beginning (static column) of the partition.
diff --git a/doc/modules/cassandra/pages/troubleshooting/finding_nodes.adoc b/doc/modules/cassandra/pages/troubleshooting/finding_nodes.adoc
index d2e9a9b10a93..a9b9b3484eec 100644
--- a/doc/modules/cassandra/pages/troubleshooting/finding_nodes.adoc
+++ b/doc/modules/cassandra/pages/troubleshooting/finding_nodes.adoc
@@ -128,6 +128,6 @@ exhaust significant CPU capacitity with a "single" query.
Once you have narrowed down the problem as much as possible (datacenter,
rack , node), login to one of the nodes using SSH and proceed to debug
-using xref:reading_logs.adoc[`logs`], xref:use_nodetooladoc[`nodetool`], and
-xref:use_tools.adoc[`os tools`].
+using xref:cassandra:troubleshooting/reading_logs.adoc[`logs`], xref:cassandra:troubleshooting/use_nodetool.adoc[`nodetool`], and
+xref:cassandra:troubleshooting/use_tools.adoc[`os tools`].
If you are not able to login you may still have access to `logs` and `nodetool` remotely.
diff --git a/doc/modules/cassandra/pages/troubleshooting/reading_logs.adoc b/doc/modules/cassandra/pages/troubleshooting/reading_logs.adoc
index 3f2f1a8ddcb4..d1a3a93cd55f 100644
--- a/doc/modules/cassandra/pages/troubleshooting/reading_logs.adoc
+++ b/doc/modules/cassandra/pages/troubleshooting/reading_logs.adoc
@@ -244,4 +244,4 @@ index b2c5b10..71b0a49 100644
Note that if you want more information than this tool provides, there
are other live capture options available such as
-xref:cassandra:developing/cql/troubleshooting/use_tools.adoc#packet-capture[`packet-capture`].
+xref:cassandra:troubleshooting/use_tools.adoc#packet-capture[`packet-capture`].
diff --git a/doc/modules/cassandra/pages/troubleshooting/use_tools.adoc b/doc/modules/cassandra/pages/troubleshooting/use_tools.adoc
index ed72f5433d72..5b883cfd3ae6 100644
--- a/doc/modules/cassandra/pages/troubleshooting/use_tools.adoc
+++ b/doc/modules/cassandra/pages/troubleshooting/use_tools.adoc
@@ -208,7 +208,7 @@ when it syncs the commit log. This typically enters into the very high
percentiles of write latency.
Note that to get detailed latency breakdowns you will need a more
-advanced tool such as xref:use_tools.adoc#bcc-tools[`bcc-tools`].
+advanced tool such as xref:cassandra:troubleshooting/use_tools.adoc#use-bcc-tools[`bcc-tools`].
=== OS page Cache Usage
@@ -232,8 +232,8 @@ Cassandra performance can suffer significantly. This is why Cassandra
starts with a reasonably small amount of memory reserved for the heap.
If you suspect that you are missing the OS page cache frequently you can
-use advanced tools like xref:use_tools.adoc#use-bcc-tools[cachestat] or
-xref:use_tools.adoc#use-vmtouch[vmtouch] to dive deeper.
+use advanced tools like xref:cassandra:troubleshooting/use_tools.adoc#use-bcc-tools[cachestat] or
+xref:cassandra:troubleshooting/use_tools.adoc#use-vmtouch[vmtouch] to dive deeper.
=== Network Latency and Reliability
@@ -483,7 +483,7 @@ $ ./vmtouch /var/lib/cassandra/data/
In this case almost the entire dataset is hot in OS page Cache.
Generally speaking the percentage doesn't really matter unless reads are
-missing the cache (per e.g. xref:cassandra:developing/cql/troubleshooting/use_tools.adoc#use-bcc-tools[cachestat] in which case
+missing the cache (per e.g. xref:cassandra:troubleshooting/use_tools.adoc#use-bcc-tools[cachestat] in which case
having additional memory may help read performance.
=== CPU Flamegraphs
@@ -544,6 +544,7 @@ $ cat cassandra_stacks | ./stackcollapse-perf.pl | grep -v cpu_idle | \
The resulting SVG is searchable, zoomable, and generally easy to
introspect using a browser.
+[[packet-capture]]
=== Packet Capture
Sometimes you have to understand what queries a Cassandra node is
diff --git a/doc/modules/cassandra/pages/vector-search/concepts.adoc b/doc/modules/cassandra/pages/vector-search/concepts.adoc
index 05c97e4c1d02..ba194779043f 100644
--- a/doc/modules/cassandra/pages/vector-search/concepts.adoc
+++ b/doc/modules/cassandra/pages/vector-search/concepts.adoc
@@ -4,7 +4,7 @@
Vector Search is a new feature added to {cass-50}.
It is a powerful technique for finding relevant content within large datasets and is particularly useful for AI applications.
-Vector Search also makes use of xref:cassandra:developing/cql/indexing/sai/overview.adoc[Storage-Attached Indexes(SAI)], leveraging the new modularity of the latter feature.
+Vector Search also makes use of xref:cassandra:developing/cql/indexing/sai/sai-overview.adoc[Storage-Attached Indexes(SAI)], leveraging the new modularity of the latter feature.
Vector Search is the first instance of validating the extensibility of SAI.
Data stored in a database is useful, but the context of that data is critical to applications.
diff --git a/doc/modules/cassandra/partials/compact-subproperties.adoc b/doc/modules/cassandra/partials/compact-subproperties.adoc
index 1cee84b7d296..1ca766211385 100644
--- a/doc/modules/cassandra/partials/compact-subproperties.adoc
+++ b/doc/modules/cassandra/partials/compact-subproperties.adoc
@@ -108,7 +108,7 @@ Default: `32`
The compaction class `SizeTieredCompactionStrategy` (STCS) triggers a minor compaction when table meets the `min_threshold`.
Minor compactions do not involve all the tables in a keyspace.
See
-xref:operating/compaction/stcs.adoc#stcs_options[SizeTieredCompactionStrategy (STCS)].
+xref:cassandra:managing/operating/compaction/stcs.adoc#stcs_options[SizeTieredCompactionStrategy (STCS)].
[NOTE]
====
@@ -151,7 +151,7 @@ Default: `50` (MB)
[NOTE]
====
The `cold_reads_to_omit` property for
-xref:operating/compaction/stcs.adoc#stcs_options[SizeTieredCompactionStrategy (STCS)] is no longer supported.
+xref:cassandra:managing/operating/compaction/stcs.adoc#stcs_options[SizeTieredCompactionStrategy (STCS)] is no longer supported.
====
[[TWCS]]
@@ -162,7 +162,7 @@ TWCS creates a new time window within each successive time period.
During the active time window, TWCS compacts all SSTables flushed from memory into larger SSTables using STCS.
At the end of the time period, all of these SSTables are compacted into a single SSTable.
Then the next time window starts and the process repeats.
-See xref:operating/compaction/twcs.adoc#twcs_options[TimeWindowCompactionStrategy (TWCS)].
+See xref:cassandra:managing/operating/compaction/twcs.adoc#twcs_options[TimeWindowCompactionStrategy (TWCS)].
[NOTE]
====
@@ -228,7 +228,7 @@ Default: `160`
====
The default value, 160 MB, may be inefficient and negatively impact database indexing and the queries that rely on indexes.
For example, consider the benefit of using higher values for sstable_size_in_mb in tables that use (SAI) indexes.
-For related information, see xref:developing/cql/indexing/sai/configuring.adoc#saiConfigure__saiCompactionStrategies[Compaction strategies].
+For related information, see xref:cassandra:developing/cql/indexing/sai/operations/configuring.adoc#saiConfigure__saiCompactionStrategies[Compaction strategies].
====
==== DateTieredCompactionStrategy (deprecated)
diff --git a/doc/modules/cassandra/partials/index-naming.adoc b/doc/modules/cassandra/partials/index-naming.adoc
new file mode 100644
index 000000000000..9e7977346387
--- /dev/null
+++ b/doc/modules/cassandra/partials/index-naming.adoc
@@ -0,0 +1,6 @@
+[WARNING]
+====
+Index names are unique per keyspace.
+You cannot use the same index name for two different indexes within a keyspace, regardless of which
+table they are on.
+====
diff --git a/doc/modules/cassandra/partials/primary-key-column.adoc b/doc/modules/cassandra/partials/primary-key-column.adoc
index 9abe065f6418..d9f49bb08092 100644
--- a/doc/modules/cassandra/partials/primary-key-column.adoc
+++ b/doc/modules/cassandra/partials/primary-key-column.adoc
@@ -17,7 +17,7 @@ Use a unique name for each column in a table.
To preserve case or use special characters, enclose the name in double-quotes.
cql_type_definition ::
Defines the type of data allowed in the column.
-See xref:reference:data-types.adoc[CQL data type] or a xref:reference:user-defined-type.adoc[user-defined type].
+See xref:cassandra:developing/cql/types.adoc#native-types[CQL data type] or a xref:cassandra:developing/cql/types.adoc#udts[user-defined type].
*STATIC* ::
Optional, the column has a single value.
*PRIMARY KEY* ::
diff --git a/doc/modules/cassandra/partials/table-column-definitions.adoc b/doc/modules/cassandra/partials/table-column-definitions.adoc
index 2a5e3e40df5a..bb981226aadb 100644
--- a/doc/modules/cassandra/partials/table-column-definitions.adoc
+++ b/doc/modules/cassandra/partials/table-column-definitions.adoc
@@ -17,7 +17,7 @@ Use a unique name for each column in a table.
To preserve case or use special characters, enclose the name in double-quotes.
cql_type_definition ::
Defines the type of data allowed in the column.
-See xref:reference:data-types.adoc[CQL data type] or a xref:reference:user-defined-type.adoc[user-defined type].
+See xref:cassandra:developing/cql/types.adoc#native-types[CQL data type] or a xref:cassandra:developing/cql/types.adoc#udts[user-defined type].
*STATIC* ::
Optional, the column has a single value.
*PRIMARY KEY* ::
diff --git a/doc/modules/cassandra/partials/table-properties.adoc b/doc/modules/cassandra/partials/table-properties.adoc
index 9aa6f16d3da5..389dbe15bbbf 100644
--- a/doc/modules/cassandra/partials/table-properties.adoc
+++ b/doc/modules/cassandra/partials/table-properties.adoc
@@ -96,8 +96,8 @@ However, if you lower the `gc_grace_seconds` value, consider its interaction wit
* *hint replays*: When a node goes down and then comes back up, other nodes replay the write operations (called xref:managing/operating/hints.adoc[hints]) that are queued for that node while it was unresponsive.
The database does not replay hints older than gc_grace_seconds after creation.
-The xref:managing/configuration/configuration/cass_yaml_file.adoc#max_hint_window[max_hint_window] setting in the
-xref:managing/configuration/configuration/cass_yaml_file.adoc[cassandra.yaml] file sets the time limit (3 hours by default) for collecting hints for the unresponsive node.
+The xref:cassandra:managing/configuration/cass_yaml_file.adoc#max_hint_window[max_hint_window] setting in the
+xref:cassandra:managing/configuration/cass_yaml_file.adoc[cassandra.yaml] file sets the time limit (3 hours by default) for collecting hints for the unresponsive node.
* *batch replays*: Like hint queues, xref:developing/cql/ddl.adoc#batch_statement[batch operations] store database mutations that are replayed in sequence.
As with hints, the database does not replay a batched mutation older than gc_grace_seconds after creation.
diff --git a/doc/modules/cassandra/partials/vector-search/vector_functions.adoc b/doc/modules/cassandra/partials/vector-search/vector_functions.adoc
index daa4b2b8ce22..e73fc628b466 100644
--- a/doc/modules/cassandra/partials/vector-search/vector_functions.adoc
+++ b/doc/modules/cassandra/partials/vector-search/vector_functions.adoc
@@ -30,12 +30,38 @@ Examples:
Examples:
-`similarity_dot_product([0.1, 0.2], null)` -> `null`
+`similarity_dot_product([0.447214, 0.894427], null)` -> `null`
-`similarity_dot_product([0.1, 0.2], [0.1, 0.2])` -> `0.525`
+`similarity_dot_product([0.447214, 0.894427], [0.447214, 0.894427])` -> `1`
-`similarity_dot_product([0.1, 0.2], [-0.1, -0.2])` -> `0.475`
+`similarity_dot_product([0.447214, 0.894427], [-0.447214, -0.894427])` -> `0`
-`similarity_dot_product([0.1, 0.2], [0.9, 0.8])` -> `0.625`
+`similarity_dot_product([0.447214, 0.894427], [-0.447214, 0.894427])` -> `0.8`
+
+`similarity_dot_product([0.447214, 0.894427], [0.447214, -0.894427])` -> `0.2`
+
+| `random_float_vector(int, float, float)` | Returns a new vector of floats with the specified dimension and where
+all components will be in the specified min-max range.
+
+Examples:
+
+`random_float_vector(2, -1.0, 1.0)` -> `[-0.695395, -0.395755]`
+
+`random_float_vector(2, -1.0, 1.0)` -> `[-0.58795, 0.690014]`
+
+`random_float_vector(2, 0.0, 1.0)` -> `[0.423859, 0.630168]`
+
+`random_float_vector(2, 0.0, 1.0)` -> `[0.468159, 0.283808]`
+
+| `normalize_l2(vector)` | Applies L2 normalization to the input vector.
+The result is a vector with the same direction but with a magnitude of 1.
+
+Examples:
+
+`normalize_l2([0.1])` -> `[1]`
+
+`normalize_l2([-0.7])` -> `[1]`
+
+`normalize_l2([3.0, 4.0])` -> `[0.6, 0.8]`
|===
\ No newline at end of file
diff --git a/doc/native_protocol_v4.1.spec b/doc/native_protocol_v4.1.spec
new file mode 100644
index 000000000000..a10fd2404d8f
--- /dev/null
+++ b/doc/native_protocol_v4.1.spec
@@ -0,0 +1,1212 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+ CQL BINARY PROTOCOL v4.1
+
+
+Table of Contents
+
+ 1. Overview
+ 2. Frame header
+ 2.1. version
+ 2.2. flags
+ 2.3. stream
+ 2.4. opcode
+ 2.5. length
+ 3. Notations
+ 4. Messages
+ 4.1. Requests
+ 4.1.1. STARTUP
+ 4.1.2. AUTH_RESPONSE
+ 4.1.3. OPTIONS
+ 4.1.4. QUERY
+ 4.1.5. PREPARE
+ 4.1.6. EXECUTE
+ 4.1.7. BATCH
+ 4.1.8. REGISTER
+ 4.2. Responses
+ 4.2.1. ERROR
+ 4.2.2. READY
+ 4.2.3. AUTHENTICATE
+ 4.2.4. SUPPORTED
+ 4.2.5. RESULT
+ 4.2.5.1. Void
+ 4.2.5.2. Rows
+ 4.2.5.3. Set_keyspace
+ 4.2.5.4. Prepared
+ 4.2.5.5. Schema_change
+ 4.2.6. EVENT
+ 4.2.7. AUTH_CHALLENGE
+ 4.2.8. AUTH_SUCCESS
+ 5. Compression
+ 6. Data Type Serialization Formats
+ 7. User Defined Type Serialization
+ 8. Result paging
+ 9. Error codes
+ 10. Changes from v4
+
+
+1. Overview
+
+ The CQL binary protocol is a frame based protocol. Frames are defined as:
+
+ 0 8 16 24 32 40
+ +---------+---------+---------+---------+---------+
+ | version | flags | stream | opcode |
+ +---------+---------+---------+---------+---------+
+ | length |
+ +---------+---------+---------+---------+
+ | |
+ . ... body ... .
+ . .
+ . .
+ +----------------------------------------
+
+ The protocol is big-endian (network byte order).
+
+ Each frame contains a fixed size header (9 bytes) followed by a variable size
+ body. The header is described in Section 2. The content of the body depends
+ on the header opcode value (the body can in particular be empty for some
+ opcode values). The list of allowed opcodes is defined in Section 2.4 and the
+ details of each corresponding message are described Section 4.
+
+ The protocol distinguishes two types of frames: requests and responses. Requests
+ are those frames sent by the client to the server. Responses are those frames sent
+ by the server to the client. Note, however, that the protocol supports server pushes
+ (events) so a response does not necessarily come right after a client request.
+
+ Note to client implementors: client libraries should always assume that the
+ body of a given frame may contain more data than what is described in this
+ document. It will however always be safe to ignore the remainder of the frame
+ body in such cases. The reason is that this may enable extending the protocol
+ with optional features without needing to change the protocol version.
+
+
+
+2. Frame header
+
+2.1. version
+
+ The version is a single byte that indicates both the direction of the message
+ (request or response) and the version of the protocol in use. The most
+ significant bit of version is used to define the direction of the message:
+ 0 indicates a request, 1 indicates a response. This can be useful for protocol
+ analyzers to distinguish the nature of the packet from the direction in which
+ it is moving. The rest of that byte is the protocol version (4 for the protocol
+ defined in this document). In other words, for this version of the protocol,
+ version will be one of:
+ 0x04 Request frame for this protocol version
+ 0x84 Response frame for this protocol version
+
+ Please note that while every message ships with the version, only one version
+ of messages is accepted on a given connection. In other words, the first message
+ exchanged (STARTUP) sets the version for the connection for the lifetime of this
+ connection.
+
+ This document describes version 4 of the protocol. For the changes made since
+ version 3, see Section 10.
+
+
+2.2. flags
+
+ Flags applying to this frame. The flags have the following meaning (described
+ by the mask that allows selecting them):
+ 0x01: Compression flag. If set, the frame body is compressed. The actual
+ compression to use should have been set up beforehand through the
+ Startup message (which thus cannot be compressed; Section 4.1.1).
+ 0x02: Tracing flag. For a request frame, this indicates the client requires
+ tracing of the request. Note that only QUERY, PREPARE and EXECUTE queries
+ support tracing. Other requests will simply ignore the tracing flag if
+ set. If a request supports tracing and the tracing flag is set, the response
+ to this request will have the tracing flag set and contain tracing
+ information.
+ If a response frame has the tracing flag set, its body contains
+ a tracing ID. The tracing ID is a [uuid] and is the first thing in
+ the frame body.
+ 0x04: Custom payload flag. For a request or response frame, this indicates
+ that a generic key-value custom payload for a custom QueryHandler
+ implementation is present in the frame. Such a custom payload is simply
+ ignored by the default QueryHandler implementation.
+ Currently, only QUERY, PREPARE, EXECUTE and BATCH requests support
+ payload.
+ Type of custom payload is [bytes map] (see below). If either or both
+ of the tracing and warning flags are set, the custom payload will follow
+ those indicated elements in the frame body. If neither are set, the custom
+ payload will be the first value in the frame body.
+ 0x08: Warning flag. The response contains warnings which were generated by the
+ server to go along with this response.
+ If a response frame has the warning flag set, its body will contain the
+ text of the warnings. The warnings are a [string list] and will be the
+ first value in the frame body if the tracing flag is not set, or directly
+ after the tracing ID if it is.
+
+ The rest of flags is currently unused and ignored.
+
+2.3. stream
+
+ A frame has a stream id (a [short] value). When sending request messages, this
+ stream id must be set by the client to a non-negative value (negative stream id
+ are reserved for streams initiated by the server; currently all EVENT messages
+ (section 4.2.6) have a streamId of -1). If a client sends a request message
+ with the stream id X, it is guaranteed that the stream id of the response to
+ that message will be X.
+
+ This helps to enable the asynchronous nature of the protocol. If a client
+ sends multiple messages simultaneously (without waiting for responses), there
+ is no guarantee on the order of the responses. For instance, if the client
+ writes REQ_1, REQ_2, REQ_3 on the wire (in that order), the server might
+ respond to REQ_3 (or REQ_2) first. Assigning different stream ids to these 3
+ requests allows the client to distinguish to which request a received answer
+ responds to. As there can only be 32768 different simultaneous streams, it is up
+ to the client to reuse stream id.
+
+ Note that clients are free to use the protocol synchronously (i.e. wait for
+ the response to REQ_N before sending REQ_N+1). In that case, the stream id
+ can be safely set to 0. Clients should also feel free to use only a subset of
+ the 32768 maximum possible stream ids if it is simpler for its implementation.
+
+2.4. opcode
+
+ An integer byte that distinguishes the actual message:
+ 0x00 ERROR
+ 0x01 STARTUP
+ 0x02 READY
+ 0x03 AUTHENTICATE
+ 0x05 OPTIONS
+ 0x06 SUPPORTED
+ 0x07 QUERY
+ 0x08 RESULT
+ 0x09 PREPARE
+ 0x0A EXECUTE
+ 0x0B REGISTER
+ 0x0C EVENT
+ 0x0D BATCH
+ 0x0E AUTH_CHALLENGE
+ 0x0F AUTH_RESPONSE
+ 0x10 AUTH_SUCCESS
+
+ Messages are described in Section 4.
+
+ (Note that there is no 0x04 message in this version of the protocol)
+
+
+2.5. length
+
+ A 4 byte integer representing the length of the body of the frame (note:
+ currently a frame is limited to 256MB in length).
+
+
+3. Notations
+
+ To describe the layout of the frame body for the messages in Section 4, we
+ define the following:
+
+ [int] A 4 bytes integer
+ [long] A 8 bytes integer
+ [short] A 2 bytes unsigned integer
+ [string] A [short] n, followed by n bytes representing an UTF-8
+ string.
+ [long string] An [int] n, followed by n bytes representing an UTF-8 string.
+ [uuid] A 16 bytes long uuid.
+ [string list] A [short] n, followed by n [string].
+ [bytes] A [int] n, followed by n bytes if n >= 0. If n < 0,
+ no byte should follow and the value represented is `null`.
+ [value] A [int] n, followed by n bytes if n >= 0.
+ If n == -1 no byte should follow and the value represented is `null`.
+ If n == -2 no byte should follow and the value represented is
+ `not set` not resulting in any change to the existing value.
+ n < -2 is an invalid value and results in an error.
+ [short bytes] A [short] n, followed by n bytes if n >= 0.
+
+ [option] A pair of where is a [short] representing
+ the option id and depends on that option (and can be
+ of size 0). The supported id (and the corresponding )
+ will be described when this is used.
+ [option list] A [short] n, followed by n [option].
+ [inet] An address (ip and port) to a node. It consists of one
+ [byte] n, that represents the address size, followed by n
+ [byte] representing the IP address (in practice n can only be
+ either 4 (IPv4) or 16 (IPv6)), following by one [int]
+ representing the port.
+ [consistency] A consistency level specification. This is a [short]
+ representing a consistency level with the following
+ correspondance:
+ 0x0000 ANY
+ 0x0001 ONE
+ 0x0002 TWO
+ 0x0003 THREE
+ 0x0004 QUORUM
+ 0x0005 ALL
+ 0x0006 LOCAL_QUORUM
+ 0x0007 EACH_QUORUM
+ 0x0008 SERIAL
+ 0x0009 LOCAL_SERIAL
+ 0x000A LOCAL_ONE
+
+ [string map] A [short] n, followed by n pair where and
+ are [string].
+ [string multimap] A [short] n, followed by n pair where is a
+ [string] and is a [string list].
+ [bytes map] A [short] n, followed by n pair where is a
+ [string] and is a [bytes].
+
+
+4. Messages
+
+ Dependant on the flags specified in the header, the layout of the message body must be:
+ [][][]
+ where:
+ - is a UUID tracing ID, present if this is a request message and the Tracing flag is set.
+ - is a string list of warnings (if this is a request message and the Warning flag is set.
+ - is bytes map for the serialised custom payload present if this is one of the message types
+ which support custom payloads (QUERY, PREPARE, EXECUTE and BATCH) and the Custom payload flag is set.
+ - as defined below through sections 4 and 5.
+
+4.1. Requests
+
+ Note that outside of their normal responses (described below), all requests
+ can get an ERROR message (Section 4.2.1) as response.
+
+4.1.1. STARTUP
+
+ Initialize the connection. The server will respond by either a READY message
+ (in which case the connection is ready for queries) or an AUTHENTICATE message
+ (in which case credentials will need to be provided using AUTH_RESPONSE).
+
+ This must be the first message of the connection, except for OPTIONS that can
+ be sent before to find out the options supported by the server. Once the
+ connection has been initialized, a client should not send any more STARTUP
+ messages.
+
+ The body is a [string map] of options. Possible options are:
+ - "CQL_VERSION": the version of CQL to use. This option is mandatory and
+ currently the only version supported is "3.0.0". Note that this is
+ different from the protocol version.
+ - "COMPRESSION": the compression algorithm to use for frames (See section 5).
+ This is optional; if not specified no compression will be used.
+ - "NO_COMPACT": whether or not connection has to be established in compatibility
+ mode. This mode will make all Thrift and Compact Tables to be exposed as if
+ they were CQL Tables. This is optional; if not specified, the option will
+ not be used.
+ - "THROW_ON_OVERLOAD": In case of server overloaded with too many requests, by default the server puts
+ back pressure on the client connection. Instead, the server can send an OverloadedException error message back to
+ the client if this option is set to true.
+ - "PAGE_UNIT": a list of supported page units.
+
+
+4.1.2. AUTH_RESPONSE
+
+ Answers a server authentication challenge.
+
+ Authentication in the protocol is SASL based. The server sends authentication
+ challenges (a bytes token) to which the client answers with this message. Those
+ exchanges continue until the server accepts the authentication by sending a
+ AUTH_SUCCESS message after a client AUTH_RESPONSE. Note that the exchange
+ begins with the client sending an initial AUTH_RESPONSE in response to a
+ server AUTHENTICATE request.
+
+ The body of this message is a single [bytes] token. The details of what this
+ token contains (and when it can be null/empty, if ever) depends on the actual
+ authenticator used.
+
+ The response to a AUTH_RESPONSE is either a follow-up AUTH_CHALLENGE message,
+ an AUTH_SUCCESS message or an ERROR message.
+
+
+4.1.3. OPTIONS
+
+ Asks the server to return which STARTUP options are supported. The body of an
+ OPTIONS message should be empty and the server will respond with a SUPPORTED
+ message.
+
+
+4.1.4. QUERY
+
+ Performs a CQL query. The body of the message must be:
+
+ where is a [long string] representing the query and
+ must be
+ [[name_1]...[name_n]][][][][]
+ where:
+ - is the [consistency] level for the operation.
+ - is a [byte] whose bits define the options for this query and
+ in particular influence what the remainder of the message contains.
+ A flag is set if the bit corresponding to its `mask` is set. Supported
+ flags are, given their mask:
+ 0x00000001: Values. If set, a [short] followed by [value]
+ values are provided. Those values are used for bound variables in
+ the query. Optionally, if the 0x40 flag is present, each value
+ will be preceded by a [string] name, representing the name of
+ the marker the value must be bound to.
+ 0x00000002: Skip_metadata. If set, the Result Set returned as a response
+ to the query (if any) will have the NO_METADATA flag (see
+ Section 4.2.5.2).
+ 0x00000004: Page_size. If set, is an [int]
+ controlling the desired page size of the result (in CQL3 rows or bytes).
+ See the section on paging (Section 8) for more details.
+ 0x00000008: With_paging_state. If set, should be present.
+ is a [bytes] value that should have been returned
+ in a result set (Section 4.2.5.2). The query will be
+ executed but starting from a given paging state. This is also to
+ continue paging on a different node than the one where it
+ started (See Section 8 for more details).
+ 0x00000010: With serial consistency. If set, should be
+ present. is the [consistency] level for the
+ serial phase of conditional updates. That consitency can only be
+ either SERIAL or LOCAL_SERIAL and if not present, it defaults to
+ SERIAL. This option will be ignored for anything else other than a
+ conditional update/insert.
+ 0x00000020: With default timestamp. If set, should be present.
+ is a [long] representing the default timestamp for the query
+ in microseconds (negative values are forbidden). This will
+ replace the server side assigned timestamp as default timestamp.
+ Note that a timestamp in the query itself will still override
+ this timestamp. This is entirely optional.
+ 0x00000040: With names for values. This only makes sense if the 0x01 flag is set and
+ is ignored otherwise. If present, the values from the 0x01 flag will
+ be preceded by a name (see above). Note that this is only useful for
+ QUERY requests where named bind markers are used; for EXECUTE statements,
+ since the names for the expected values was returned during preparation,
+ a client can always provide values in the right order without any names
+ and using this flag, while supported, is almost surely inefficient.
+ 0x40000000: When set, the is provided in bytes rather than in rows.
+
+
+ Note that the consistency is ignored by some queries (USE, CREATE, ALTER,
+ TRUNCATE, ...).
+
+ The server will respond to a QUERY message with a RESULT message, the content
+ of which depends on the query.
+
+
+4.1.5. PREPARE
+
+ Prepare a query for later execution (through EXECUTE). The body consists of
+ the CQL query to prepare as a [long string].
+
+ The server will respond with a RESULT message with a `prepared` kind (0x0004,
+ see Section 4.2.5).
+
+
+4.1.6. EXECUTE
+
+ Executes a prepared query. The body of the message must be:
+
+ where is the prepared query ID. It's the [short bytes] returned as a
+ response to a PREPARE message. As for , it has the exact
+ same definition as in QUERY (see Section 4.1.4).
+
+ The response from the server will be a RESULT message.
+
+
+4.1.7. BATCH
+
+ Allows executing a list of queries (prepared or not) as a batch (note that
+ only DML statements are accepted in a batch). The body of the message must
+ be:
+ ...[][]
+ where:
+ - is a [byte] indicating the type of batch to use:
+ - If == 0, the batch will be "logged". This is equivalent to a
+ normal CQL3 batch statement.
+ - If == 1, the batch will be "unlogged".
+ - If == 2, the batch will be a "counter" batch (and non-counter
+ statements will be rejected).
+ - is a [byte] whose bits define the options for this query and
+ in particular influence what the remainder of the message contains. It is similar
+ to the from QUERY and EXECUTE methods, except that the 4 rightmost
+ bits must always be 0 as their corresponding options do not make sense for
+ Batch. A flag is set if the bit corresponding to its `mask` is set. Supported
+ flags are, given their mask:
+ 0x10: With serial consistency. If set, should be
+ present. is the [consistency] level for the
+ serial phase of conditional updates. That consistency can only be
+ either SERIAL or LOCAL_SERIAL and if not present, it defaults to
+ SERIAL. This option will be ignored for anything else other than a
+ conditional update/insert.
+ 0x20: With default timestamp. If set, should be present.
+ is a [long] representing the default timestamp for the query
+ in microseconds. This will replace the server side assigned
+ timestamp as default timestamp. Note that a timestamp in the query itself
+ will still override this timestamp. This is entirely optional.
+ 0x40: With names for values. If set, then all values for all must be
+ preceded by a [string] that have the same meaning as in QUERY
+ requests [IMPORTANT NOTE: this feature does not work and should not be
+ used. It is specified in a way that makes it impossible for the server
+ to implement. This will be fixed in a future version of the native
+ protocol. See https://issues.apache.org/jira/browse/CASSANDRA-10246 for
+ more details].
+ - is a [short] indicating the number of following queries.
+ - ... are the queries to execute. A must be of the
+ form:
+ []...[]
+ where:
+ - is a [byte] indicating whether the following query is a prepared
+ one or not. value must be either 0 or 1.
+ - depends on the value of . If == 0, it should be
+ a [long string] query string (as in QUERY, the query string might contain
+ bind markers). Otherwise (that is, if == 1), it should be a
+ [short bytes] representing a prepared query ID.
+ - is a [short] indicating the number (possibly 0) of following values.
+ - is the optional name of the following . It must be present
+ if and only if the 0x40 flag is provided for the batch.
+ - is the [value] to use for bound variable i (of bound variable
+ if the 0x40 flag is used).
+ - is the [consistency] level for the operation.
+ - is only present if the 0x10 flag is set. In that case,
+ is the [consistency] level for the serial phase of
+ conditional updates. That consitency can only be either SERIAL or
+ LOCAL_SERIAL and if not present will defaults to SERIAL. This option will
+ be ignored for anything else other than a conditional update/insert.
+
+ The server will respond with a RESULT message.
+
+
+4.1.8. REGISTER
+
+ Register this connection to receive some types of events. The body of the
+ message is a [string list] representing the event types to register for. See
+ section 4.2.6 for the list of valid event types.
+
+ The response to a REGISTER message will be a READY message.
+
+ Please note that if a client driver maintains multiple connections to a
+ Cassandra node and/or connections to multiple nodes, it is advised to
+ dedicate a handful of connections to receive events, but to *not* register
+ for events on all connections, as this would only result in receiving
+ multiple times the same event messages, wasting bandwidth.
+
+
+4.2. Responses
+
+ This section describes the content of the frame body for the different
+ responses. Please note that to make room for future evolution, clients should
+ support extra informations (that they should simply discard) to the one
+ described in this document at the end of the frame body.
+
+4.2.1. ERROR
+
+ Indicates an error processing a request. The body of the message will be an
+ error code ([int]) followed by a [string] error message. Then, depending on
+ the exception, more content may follow. The error codes are defined in
+ Section 9, along with their additional content if any.
+
+
+4.2.2. READY
+
+ Indicates that the server is ready to process queries. This message will be
+ sent by the server either after a STARTUP message if no authentication is
+ required (if authentication is required, the server indicates readiness by
+ sending a AUTH_RESPONSE message).
+
+ The body of a READY message is empty.
+
+
+4.2.3. AUTHENTICATE
+
+ Indicates that the server requires authentication, and which authentication
+ mechanism to use.
+
+ The authentication is SASL based and thus consists of a number of server
+ challenges (AUTH_CHALLENGE, Section 4.2.7) followed by client responses
+ (AUTH_RESPONSE, Section 4.1.2). The initial exchange is however boostrapped
+ by an initial client response. The details of that exchange (including how
+ many challenge-response pairs are required) are specific to the authenticator
+ in use. The exchange ends when the server sends an AUTH_SUCCESS message or
+ an ERROR message.
+
+ This message will be sent following a STARTUP message if authentication is
+ required and must be answered by a AUTH_RESPONSE message from the client.
+
+ The body consists of a single [string] indicating the full class name of the
+ IAuthenticator in use.
+
+
+4.2.4. SUPPORTED
+
+ Indicates which startup options are supported by the server. This message
+ comes as a response to an OPTIONS message.
+
+ The body of a SUPPORTED message is a [string multimap]. This multimap gives
+ for each of the supported STARTUP options, the list of supported values.
+
+
+4.2.5. RESULT
+
+ The result to a query (QUERY, PREPARE, EXECUTE or BATCH messages).
+
+ The first element of the body of a RESULT message is an [int] representing the
+ `kind` of result. The rest of the body depends on the kind. The kind can be
+ one of:
+ 0x0001 Void: for results carrying no information.
+ 0x0002 Rows: for results to select queries, returning a set of rows.
+ 0x0003 Set_keyspace: the result to a `use` query.
+ 0x0004 Prepared: result to a PREPARE message.
+ 0x0005 Schema_change: the result to a schema altering query.
+
+ The body for each kind (after the [int] kind) is defined below.
+
+
+4.2.5.1. Void
+
+ The rest of the body for a Void result is empty. It indicates that a query was
+ successful without providing more information.
+
+
+4.2.5.2. Rows
+
+ Indicates a set of rows. The rest of the body of a Rows result is:
+
+ where:
+ - is composed of:
+ [][?...]
+ where:
+ - is an [int]. The bits of provides information on the
+ formatting of the remaining information. A flag is set if the bit
+ corresponding to its `mask` is set. Supported flags are, given their
+ mask:
+ 0x0001 Global_tables_spec: if set, only one table spec (keyspace
+ and table name) is provided as . If not
+ set, is not present.
+ 0x0002 Has_more_pages: indicates whether this is not the last
+ page of results and more should be retrieved. If set, the
+ will be present. The is a
+ [bytes] value that should be used in QUERY/EXECUTE to
+ continue paging and retrieve the remainder of the result for
+ this query (See Section 8 for more details).
+ 0x0004 No_metadata: if set, the is only composed of
+ these , the and optionally the
+ (depending on the Has_more_pages flag) but
+ no other information (so no nor ).
+ This will only ever be the case if this was requested
+ during the query (see QUERY and RESULT messages).
+ - is an [int] representing the number of columns selected
+ by the query that produced this result. It defines the number of
+ elements in and the number of elements for each row in .
+ - is present if the Global_tables_spec is set in
+ . It is composed of two [string] representing the
+ (unique) keyspace name and table name the columns belong to.
+ - specifies the columns returned in the query. There are
+ such column specifications that are composed of:
+ ()?
+ The initial and are two [string] and are only present
+ if the Global_tables_spec flag is not set. The is a
+ [string] and is an [option] that corresponds to the description
+ (what this description is depends a bit on the context: in results to
+ selects, this will be either the user chosen alias or the selection used
+ (often a colum name, but it can be a function call too). In results to
+ a PREPARE, this will be either the name of the corresponding bind variable
+ or the column name for the variable if it is "anonymous") and type of
+ the corresponding result. The option for is either a native
+ type (see below), in which case the option has no value, or a
+ 'custom' type, in which case the value is a [string] representing
+ the fully qualified class name of the type represented. Valid option
+ ids are:
+ 0x0000 Custom: the value is a [string], see above.
+ 0x0001 Ascii
+ 0x0002 Bigint
+ 0x0003 Blob
+ 0x0004 Boolean
+ 0x0005 Counter
+ 0x0006 Decimal
+ 0x0007 Double
+ 0x0008 Float
+ 0x0009 Int
+ 0x000B Timestamp
+ 0x000C Uuid
+ 0x000D Varchar
+ 0x000E Varint
+ 0x000F Timeuuid
+ 0x0010 Inet
+ 0x0011 Date
+ 0x0012 Time
+ 0x0013 Smallint
+ 0x0014 Tinyint
+ 0x0020 List: the value is an [option], representing the type
+ of the elements of the list.
+ 0x0021 Map: the value is two [option], representing the types of the
+ keys and values of the map
+ 0x0022 Set: the value is an [option], representing the type
+ of the elements of the set
+ 0x0030 UDT: the value is ...
+ where:
+ - is a [string] representing the keyspace name this
+ UDT is part of.
+ - is a [string] representing the UDT name.
+ - is a [short] representing the number of fields of
+ the UDT, and thus the number of pairs
+ following
+ - is a [string] representing the name of the
+ i_th field of the UDT.
+ - is an [option] representing the type of the
+ i_th field of the UDT.
+ 0x0031 Tuple: the value is ... where is a [short]
+ representing the number of values in the type, and
+ are [option] representing the type of the i_th component
+ of the tuple
+
+ - is an [int] representing the number of rows present in this
+ result. Those rows are serialized in the part.
+ - is composed of ... where m is .
+ Each is composed of ... where n is
+ and where is a [bytes] representing the value
+ returned for the jth column of the ith row. In other words,
+ is composed of ( * ) [bytes].
+
+
+4.2.5.3. Set_keyspace
+
+ The result to a `use` query. The body (after the kind [int]) is a single
+ [string] indicating the name of the keyspace that has been set.
+
+
+4.2.5.4. Prepared
+
+ The result to a PREPARE message. The body of a Prepared result is:
+
+ where:
+ - is [short bytes] representing the prepared query ID.
+ - is composed of:
+ [...][