diff --git a/docs/source/index.rst b/docs/source/index.rst index 5e8b0a7e..8f653d8a 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -17,6 +17,7 @@ Contents :maxdepth: 1 self + auto_examples/plot_beginners_guide auto_examples/plot_how_to_use_skada quickstart all diff --git a/examples/plot_beginners_guide.py b/examples/plot_beginners_guide.py new file mode 100644 index 00000000..e68865c3 --- /dev/null +++ b/examples/plot_beginners_guide.py @@ -0,0 +1,575 @@ +""" +Beginners Guide +=============== + +This is an introduction page to SKADA: SciKit Adaptation for beginners. +SKADA is an open-source library focusing on domain adaptation methods that goes +hand in hand with scikit-learn. + +In this page, we will present the key concepts and methods for starting to use SKADA +and how to use them. + +* :ref:`Shifted dataset creation`: + * :ref:`Covariate shift` + * :ref:`Target shift` + * :ref:`Conditional shift` + * :ref:`Subspace shift` + +* :ref:`Methods`: + * :ref:`Reweighting` + * :ref:`Mapping` + * :ref:`Subspace` + +* :ref:`Summary` + +For better readability, only the use of SKADA is provided and the plotting code +with matplotlib is hidden (but is available in the source file of the example). +""" + +# Author: Maxence Barneche +# +# License: BSD 3-Clause +# sphinx_gallery_thumbnail_number = 13 + +# %% +# Necessary imports + +# sphinx_gallery_start_ignore +import matplotlib.pyplot as plt +import numpy as np + +# sphinx_gallery_end_ignore +from sklearn.linear_model import LogisticRegression +from sklearn.neighbors import KernelDensity +from sklearn.svm import SVC + +import skada +from skada.datasets import make_shifted_datasets +from skada.utils import extract_source_indices + +# sphinx_gallery_start_ignore +fig_size = (8, 4) + + +def print_scores_as_table(scores): + max_len = max(len(k) for k in scores.keys()) + for k, v in scores.items(): + print(f"{k}{' '*(max_len - len(k))} | ", end="") + print(f"{v*100}{' '*(6-len(str(v*100)))}%") + + +def decision_borders_plot(model): + # Create a meshgrid + x_min, x_max = -2.5, 4.5 + y_min, y_max = -2.5, 4.5 + xx, yy = np.meshgrid( + np.linspace(x_min, x_max, num=100), np.linspace(y_min, y_max, num=100) + ) + + # Predict on every point of the meshgrid + Z = model.predict(np.c_[xx.ravel(), yy.ravel()]) + Z = Z.reshape(xx.shape) + + # Plot decision borders + plt.contourf(xx, yy, Z, alpha=0.4, cmap="tab10", vmax=9) + + +def plot_full_data(X, y, title=None, size=25, marker="o", subspace=False): + plt.scatter( + X[:, 0], X[:, 1], s=size, c=y, cmap="tab10", vmax=9, marker=marker, alpha=0.8 + ) + plt.xticks([]) + plt.yticks([]) + if title is not None: + plt.title(title, fontsize=16) + plt.gca().set_aspect("equal") + maxi = 2.5 if subspace else 4.5 + mini = -2.5 + plt.xlim(mini, maxi) + plt.ylim(mini, maxi) + + +def source_target_comparison( + X, + y, + sample_domain, + title, + prediction=False, + model=None, + size=25, + figsize=(8, 4), + subspace=False, +): + Xs, Xt, ys, yt = skada.source_target_split(X, y, sample_domain=sample_domain) + + fig, axes = plt.subplots(1, 2, figsize=figsize) + + plt.sca(axes[0]) + plot_full_data( + Xs, ys, title="Source data", size=size, marker="o", subspace=subspace + ) + if prediction: + decision_borders_plot(model) + + plt.sca(axes[1]) + plot_full_data(Xt, yt, title="Target data", marker="v", subspace=subspace) + if prediction: + decision_borders_plot(model) + + fig.suptitle(title, fontsize=16) + fig.tight_layout() + plt.subplots_adjust(top=0.85) + plt.show() + + +# sphinx_gallery_end_ignore + +# %% +# Creating a shifted dataset +# -------------------------- +# +# Data shift refers to changes in the distribution +# of inputs, targets, or their relationships. +# As a result, a model trained on the source data may perform +# poorly on the shifted target data. +# Methods for handling data shift are discussed +# in the :ref:`methods` section. +# +# DA datasets provided by SKADA are organized as follows: +# +# * :code:`X` is the input data, including the source and the target samples +# * :code:`y` is the output data to be predicted (labels on target samples are not +# used when fitting the DA estimator) +# * :code:`sample_domain` encodes the domain of each sample (integer >=0 for +# source and <0 for target) +# +# To create a shifted dataset, use the method :code:`make_shifted_dataset` +# from the `skada.datasets` module. + +X, y, sample_domain = make_shifted_datasets( + n_samples_source=20, n_samples_target=20, random_state=42 +) + +# %% +# To split the dataset between the source and the target dataset, use +# :code:`source_target_split` from the main module + +Xs, Xt, ys, yt = skada.source_target_split(X, y, sample_domain=sample_domain) +# %% +# .. NOTE:: +# +# For reproducibility, we will use the seed 42 throughout the guide. +# For simplicity, we will use a small (20) number of samples +# in both source and target. +# +# .. NOTE:: +# +# Though SKADA can manage multidomain datasets, we will only generate simple 2D +# DA datasets with a single source domain and a single target domain in this guide. +# +# There are four main types of shift available with SKADA. +# To specify the type of shift, adjust the :code:`shift` argument in +# :code:`make_shifted_datasets`. By default, this argument is set to +# :code:`covariate_shift` +# +# Example of covariate shift +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# +# Covariate shift is characterised by a change of distribution +# in one or more of the independent variables from the input data. + +X, y, sample_domain = make_shifted_datasets( + n_samples_source=20, n_samples_target=20, shift="covariate_shift", random_state=42 +) + +# sphinx_gallery_start_ignore +source_target_comparison(X, y, sample_domain, "Example covariate shift") +# sphinx_gallery_end_ignore + +# %% +# Example of target shift +# ~~~~~~~~~~~~~~~~~~~~~~~~~ +# +# Target shift (or prior probability shift) is characterised by a change in +# target variable distribution while the source data distribution remains the same. + +X, y, sample_domain = make_shifted_datasets( + n_samples_source=20, n_samples_target=20, shift="target_shift", random_state=42 +) + +# sphinx_gallery_start_ignore +source_target_comparison(X, y, sample_domain, "Example target shift") +# sphinx_gallery_end_ignore + +# %% +# Example of conditional shift +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# +# Conditional shift (or concept drift) is characterised by a change in the +# relation between input and output variables. + +X, y, sample_domain = make_shifted_datasets( + n_samples_source=20, n_samples_target=20, shift="conditional_shift", random_state=42 +) + +# sphinx_gallery_start_ignore +source_target_comparison(X, y, sample_domain, "Example conditional shift") +# sphinx_gallery_end_ignore + +# %% +# Example of subspace shift +# ~~~~~~~~~~~~~~~~~~~~~~~~~~~ +# +# Subspace shift is characterised by a change in data distribution where there exists +# a subspace such that the projection of the data on that +# subspace keeps the same distribution + +X, y, sample_domain = make_shifted_datasets( + n_samples_source=20, n_samples_target=20, shift="subspace", random_state=42 +) + +# sphinx_gallery_start_ignore +source_target_comparison(X, y, sample_domain, "Example subspace shift", subspace=True) +# sphinx_gallery_end_ignore + +# %% +# Adaptation methods +# ------------------ +# +# As stated before, a model trained on a source dataset will have a decrease +# in performance when evaluated on a shifted target dataset. Thus, there is a need +# to train the model again to account for the shift in data distribution. +# +# In some cases, the source is the only data fully available, as the target may not +# have been classified yet. A common solution is to adapt the source data. +# +# For every shift, there is a method to adapt the source data to train the model on. +# +# Reweighting methods +# ~~~~~~~~~~~~~~~~~~~ +# +# A common method for dealing with covariate and target shift is reweighting +# the source data. +# +# SKADA handles this with multiple reweighting methods: +# +# * Density Reweighting +# * Gaussian Reweighting +# * Discr. Reweighting +# * KLIEPReweight +# * Nearest Neighbor reweighting +# * Kernel Mean Matching +# +# Each method estimates the weight for the source dataset for an estimator +# to predict labels from the target dataset. +# +# Using a simple :code:`LogisticRegression` classifier, +# we can see that the estimator is not optimal on the target dataset. + +X, y, sample_domain = make_shifted_datasets( + n_samples_source=20, n_samples_target=20, shift="covariate_shift", random_state=42 +) +Xs, Xt, ys, yt = skada.source_target_split(X, y, sample_domain=sample_domain) + +base_classifier = LogisticRegression().set_fit_request(sample_weight=True) +base_classifier.fit(Xs, ys) + +# compute the accuracy of the model +accuracy = base_classifier.score(Xt, yt) + +# sphinx_gallery_start_ignore +source_target_comparison( + X, + y, + sample_domain, + "Predictions without reweighting", + prediction=True, + model=base_classifier, +) +print("Accuracy on target:", accuracy) +# sphinx_gallery_end_ignore + +# %% +# The reweighting is done by transforming the source dataset using an instance of +# the :code:`Adapter` class provided by SKADA. + +# Define the classifier as a domain adaptation (DA) pipeline from the base classifier +adapted_clf = skada.DensityReweight( + base_estimator=base_classifier, weight_estimator=KernelDensity(bandwidth=0.5) +) + +adapted_clf.fit(X, y, sample_domain=sample_domain) + +# To extract the weights, we take the weight estimator from the pipeline +weight_estimator = adapted_clf[0].get_estimator() +idx = extract_source_indices(sample_domain) +# then compute the weights corresponding to the source dataset +weights = weight_estimator.compute_weights(X, sample_domain=sample_domain)[idx] + +# compute the accuracy of the newly obtained model +accuracy = adapted_clf.score(Xt, yt) + +# sphinx_gallery_start_ignore +weights = 15 * weights +source_target_comparison( + X, + y, + sample_domain, + "Prediction on reweighted dataset", + prediction=True, + model=adapted_clf, + size=weights, +) +print("Accuracy on target:", accuracy) +# sphinx_gallery_end_ignore + + +# %% +# Mapping methods +# ~~~~~~~~~~~~~~~ +# +# The traditional way of dealing with conditional and target shift is via +# mapping the source data to the target data. +# +# First, let's look at what happens when we try to fit an estimator on the target + +X, y, sample_domain = make_shifted_datasets( + n_samples_source=20, n_samples_target=20, shift="conditional_shift", random_state=42 +) +Xs, Xt, ys, yt = skada.source_target_split(X, y, sample_domain=sample_domain) + +# setting and fitting the estimator on the source data +base_classifier = SVC() +base_classifier.fit(Xs, ys) + +# accuracy on target data +accuracy = base_classifier.score(Xt, yt) + +# sphinx_gallery_start_ignore +source_target_comparison( + X, y, sample_domain, "Prediction without mapping", True, base_classifier +) +print("Accuracy on target:", accuracy) +# sphinx_gallery_end_ignore + +# %% +# Mapping consist of mapping point from the source data to point from the target data +# using optimal transport methods. This is done automatically +# by the :code:`OTMapping` method. + +adapted_clf = skada.OTMapping(base_classifier) +adapted_clf.fit(X, y, sample_domain=sample_domain) +accuracy = adapted_clf.score(Xt, yt) + +# sphinx_gallery_start_ignore +n_tot_source = Xs.shape[0] +n_tot_target = Xt.shape[0] +adapter = adapted_clf.named_steps["otmappingadapter"].get_estimator() +T = adapter.ot_transport_.coupling_ +T = T / T.max() + +plt.figure(9, figsize=fig_size) +plot_full_data(Xs, ys, "Mapping source (circle) to target (triangle)") +plot_full_data(Xt, yt, marker="v") +for i in range(n_tot_source): + for j in range(n_tot_target): + if T[i, j] > 0: + plt.plot( + [Xs[i, 0], Xt[j, 0]], + [Xs[i, 1], Xt[j, 1]], + "-g", + alpha=T[i, j] * 0.5, + zorder=0, + ) + +plt.tight_layout() +plt.show() + +source_target_comparison( + X, y, sample_domain, "Predictions after mapping", True, adapted_clf +) +print("Accuracy on target:", accuracy) +# sphinx_gallery_end_ignore + +# %% +# Subspace methods +# ~~~~~~~~~~~~~~~~ +# +# The goal of a subspace method is to project data from its original space into a +# lower dimensional subspace. Subspace methods are particularly effective when dealing +# with subspace shift, when the source and target data have the same distribution when +# projected onto a subspace. +# +# There are multiple methods available in SKADA: +# +# * Subspace alignment +# * Transfer Component Analysis +# * Transfer Joint Matching +# * Transfer Subspace Learning +# +# Without domain adaptation, the estimator will have difficulties on the target + +X, y, sample_domain = make_shifted_datasets( + n_samples_source=20, n_samples_target=20, shift="subspace", random_state=42 +) + +Xs, Xt, ys, yt = skada.source_target_split(X, y, sample_domain=sample_domain) + +# setting and fitting the estimator on the source data +base_classifier = SVC() +base_classifier.fit(Xs, ys) + +# accuracy on target data +accuracy = base_classifier.score(Xt, yt) + +# sphinx_gallery_start_ignore +source_target_comparison( + X, y, sample_domain, "Prediction on dataset", True, base_classifier, subspace=True +) +print("Accuracy on target:", accuracy) +# sphinx_gallery_end_ignore + +# %% +# Now, we will illustrate what happens when using the Transfer Subspace Learning method +adapted_clf = skada.TransferSubspaceLearning(base_classifier, n_components=1) +adapted_clf.fit(X, y, sample_domain=sample_domain) +accuracy = adapted_clf.score(Xt, yt) + +# sphinx_gallery_start_ignore +clf = skada.TransferJointMatching(SVC(), n_components=1) +clf.fit(X, y, sample_domain=sample_domain) + +subspace_estimator = clf.steps[-2][1].get_estimator() +Xs_sub = subspace_estimator.transform( + Xs, + # mark all samples as sources + sample_domain=np.ones(Xs.shape[0]), + allow_source=True, +) +Xt_sub = subspace_estimator.transform(Xt) +Xs_sub *= 9 +mean = Xs_sub.mean() +Xs_sub = Xs_sub - mean +Xs_sub = -Xs_sub + mean +Xs_sub = np.c_[Xs_sub, Xs_sub] + +plt.figure(5, figsize=fig_size) +plt.subplot(1, 2, 1) +plot_full_data(Xs, ys, "Source data (projected)", subspace=True) +plot_full_data(Xs_sub, ys, subspace=True) +for i in range(len(Xs)): + plt.plot( + [Xs[i, 0], Xs_sub[i, 0]], + [Xs[i, 1], Xs_sub[i, 1]], + "-g", + alpha=0.2, + zorder=0, + ) + +plt.subplot(1, 2, 2) +Xt_sub *= 9 +mean = Xt_sub.mean() +Xt_sub = Xt_sub - mean +Xt_sub = -Xt_sub + mean +Xt_sub = np.c_[Xt_sub, Xt_sub] +plot_full_data(Xt, yt, "Target data (projected)", marker="v", subspace=True) +plot_full_data(Xt_sub, yt, marker="v", subspace=True) +for i in range(len(Xt)): + plt.plot( + [Xt[i, 0], Xt_sub[i, 0]], + [Xt[i, 1], Xt_sub[i, 1]], + "-g", + alpha=0.2, + zorder=0, + ) +plt.show() + +source_target_comparison( + X, y, sample_domain, "Prediction on dataset", True, adapted_clf, subspace=True +) +print("\nAccuracy on target:", accuracy) +# sphinx_gallery_end_ignore + +# %% +# Results summary +# --------------- +# +# Here is a summary of the different shifts and the domain adaptation methods used. + +# Using everything we have used before + +# sphinx_gallery_start_ignore +plt.figure(13, (12.5, 12.5)) +shift_list = ["covariate_shift", "target_shift", "conditional_shift", "subspace"] +clfs = { + "covariate_shift": LogisticRegression().set_fit_request(sample_weight=True), + "target_shift": LogisticRegression().set_fit_request(sample_weight=True), + "conditional_shift": SVC(), + "subspace": SVC(), +} +shift_acc_before = {} +shift_acc_after = {} + +for indx, shift in enumerate(shift_list): + X, y, sd = make_shifted_datasets(20, 20, shift, random_state=42) + Xs, Xt, ys, yt = skada.source_target_split(X, y, sample_domain=sd) + shift_name = shift.capitalize().replace("_", " ") + subspace = shift == "subspace" + + plt.subplot(4, 4, 1 + indx) + plot_full_data(Xs, ys, shift_name, subspace=subspace) + + plt.subplot(4, 4, 5 + indx) + plot_full_data(Xt, yt, marker="v", subspace=subspace) + + base_clf = clfs[shift] + base_clf.fit(Xs, ys) + shift_acc_before[shift_name] = base_clf.score(Xt, yt) + plt.subplot(4, 4, 9 + indx) + plot_full_data(Xt, yt, marker="v", subspace=subspace) + decision_borders_plot(base_clf) + + if shift == "covariate_shift" or shift == "target_shift": + adapted_clf = skada.DensityReweight( + base_estimator=base_clf, weight_estimator=KernelDensity(bandwidth=0.5) + ) + + elif shift == "conditional_shift": + adapted_clf = skada.OTMapping(base_clf) + + elif subspace: + adapted_clf = skada.TransferSubspaceLearning(base_clf, n_components=1) + + adapted_clf.fit(X, y, sample_domain=sd) + shift_acc_after[shift_name] = adapted_clf.score(Xt, yt) + plt.subplot(4, 4, 13 + indx) + plot_full_data(Xt, yt, marker="v", subspace=subspace) + decision_borders_plot(adapted_clf) + +X, y, sd = make_shifted_datasets(20, 20, random_state=42) +Xs, Xt, ys, yt = skada.source_target_split(X, y, sample_domain=sd) + +plt.subplot(4, 4, 1) +plt.ylabel("Source data", fontsize=16) + +plt.subplot(4, 4, 5) +plt.ylabel("Target data", fontsize=16) + +plt.subplot(4, 4, 9) +plt.ylabel("Train before DA", fontsize=16) + +plt.subplot(4, 4, 13) +plt.ylabel("Train after DA", fontsize=16) + +plt.suptitle("Summary plot", fontsize=22) +plt.tight_layout() + +print("\nAccuracy scores before Domain Adaptation:") +print_scores_as_table(shift_acc_before) +print("\nAfter Domain Adaptation:") +print_scores_as_table(shift_acc_after) +# sphinx_gallery_end_ignore + +# %% +# For information on DA pipelines and the :code:`Adapter` class with SKADA, +# don't hesitate to check out the "How to use SKADA" page and the "Users Guide" page. diff --git a/examples/plot_how_to_use_skada.py b/examples/plot_how_to_use_skada.py index 53ba4d68..e44544bc 100644 --- a/examples/plot_how_to_use_skada.py +++ b/examples/plot_how_to_use_skada.py @@ -97,7 +97,6 @@ # DA estimator in a pipeline # ----------------------------- # - # SKADA estimators can be used as the final estimator of a scikit-learn pipeline. # Again, the only difference is that the :code:`sample_domain` array must be passed # by name during in fit. @@ -121,7 +120,7 @@ # Here is an example with the CORAL and GaussianReweight adapters. # # .. WARNING:: - +# # Note that as illustrated below for reweighting adapters, one needs a # subsequent estimator that takes :code:`sample_weight` as an input parameter. # This can be done using the :code:`set_fit_request` method of the estimator