# Least Angle Regression¶

Least Angle Regression (LARS) is an algorithm used to fit a linear regression model. LARS is simliar to forward stagewise regression but less greedy. Instead of including variables at each step, the estimated parameters are increased in a direction equiangular to each one’s correlations with the residual. LARS can be used to solve LASSO, which is L1-regularized least square regression.

\begin{align}\begin{aligned}\min \|X^T\beta - y\|^2 + \lambda\|\beta\|_{1}]\\\|\beta\|_1 = \sum_i|\beta_i|\end{aligned}\end{align}

where $$X$$ is the feature matrix with explanatory features and $$y$$ is the dependent variable to be predicted. Pre-processing of $$X$$ and $$y$$ are needed to ensure the correctness of this algorithm: $$X$$ needs to be normalized: each feature should have zero-mean and unit-norm, $$y$$ needs to be centered: its mean should be zero.

## Example¶

Imagine we have files with training and test data. We create CDenseFeatures (here 64 bit floats aka RealFeatures) and CRegressionLabels as

features_train = RealFeatures(f_feats_train)
features_test = RealFeatures(f_feats_test)
labels_train = RegressionLabels(f_labels_train)
labels_test = RegressionLabels(f_labels_test)

features_train = RealFeatures(f_feats_train);
features_test = RealFeatures(f_feats_test);
labels_train = RegressionLabels(f_labels_train);
labels_test = RegressionLabels(f_labels_test);

RealFeatures features_train = new RealFeatures(f_feats_train);
RealFeatures features_test = new RealFeatures(f_feats_test);
RegressionLabels labels_train = new RegressionLabels(f_labels_train);
RegressionLabels labels_test = new RegressionLabels(f_labels_test);

features_train = Shogun::RealFeatures.new f_feats_train
features_test = Shogun::RealFeatures.new f_feats_test
labels_train = Shogun::RegressionLabels.new f_labels_train
labels_test = Shogun::RegressionLabels.new f_labels_test

features_train <- RealFeatures(f_feats_train)
features_test <- RealFeatures(f_feats_test)
labels_train <- RegressionLabels(f_labels_train)
labels_test <- RegressionLabels(f_labels_test)

features_train = shogun.RealFeatures(f_feats_train)
features_test = shogun.RealFeatures(f_feats_test)
labels_train = shogun.RegressionLabels(f_labels_train)
labels_test = shogun.RegressionLabels(f_labels_test)

RealFeatures features_train = new RealFeatures(f_feats_train);
RealFeatures features_test = new RealFeatures(f_feats_test);
RegressionLabels labels_train = new RegressionLabels(f_labels_train);
RegressionLabels labels_test = new RegressionLabels(f_labels_test);

auto features_train = some<CDenseFeatures<float64_t>>(f_feats_train);
auto features_test = some<CDenseFeatures<float64_t>>(f_feats_test);
auto labels_train = some<CRegressionLabels>(f_labels_train);
auto labels_test = some<CRegressionLabels>(f_labels_test);


To normalize and center the features, we create an instance of preprocessors CPruneVarSubMean and CNormOne and apply it on the feature matrices.

SubMean = PruneVarSubMean()
Normalize = NormOne()
SubMean.init(features_train)
SubMean.apply_to_feature_matrix(features_train)
SubMean.apply_to_feature_matrix(features_test)
Normalize.init(features_train)
Normalize.apply_to_feature_matrix(features_train)
Normalize.apply_to_feature_matrix(features_test)

SubMean = PruneVarSubMean();
Normalize = NormOne();
SubMean.init(features_train);
SubMean.apply_to_feature_matrix(features_train);
SubMean.apply_to_feature_matrix(features_test);
Normalize.init(features_train);
Normalize.apply_to_feature_matrix(features_train);
Normalize.apply_to_feature_matrix(features_test);

PruneVarSubMean SubMean = new PruneVarSubMean();
NormOne Normalize = new NormOne();
SubMean.init(features_train);
SubMean.apply_to_feature_matrix(features_train);
SubMean.apply_to_feature_matrix(features_test);
Normalize.init(features_train);
Normalize.apply_to_feature_matrix(features_train);
Normalize.apply_to_feature_matrix(features_test);

SubMean = Shogun::PruneVarSubMean.new
Normalize = Shogun::NormOne.new
SubMean.init features_train
SubMean.apply_to_feature_matrix features_train
SubMean.apply_to_feature_matrix features_test
Normalize.init features_train
Normalize.apply_to_feature_matrix features_train
Normalize.apply_to_feature_matrix features_test

SubMean <- PruneVarSubMean()
Normalize <- NormOne()
SubMean$init(features_train) SubMean$apply_to_feature_matrix(features_train)
SubMean$apply_to_feature_matrix(features_test) Normalize$init(features_train)
Normalize$apply_to_feature_matrix(features_train) Normalize$apply_to_feature_matrix(features_test)

SubMean = shogun.PruneVarSubMean()
Normalize = shogun.NormOne()
SubMean:init(features_train)
SubMean:apply_to_feature_matrix(features_train)
SubMean:apply_to_feature_matrix(features_test)
Normalize:init(features_train)
Normalize:apply_to_feature_matrix(features_train)
Normalize:apply_to_feature_matrix(features_test)

PruneVarSubMean SubMean = new PruneVarSubMean();
NormOne Normalize = new NormOne();
SubMean.init(features_train);
SubMean.apply_to_feature_matrix(features_train);
SubMean.apply_to_feature_matrix(features_test);
Normalize.init(features_train);
Normalize.apply_to_feature_matrix(features_train);
Normalize.apply_to_feature_matrix(features_test);

auto SubMean = some<CPruneVarSubMean>();
auto Normalize = some<CNormOne>();
SubMean->init(features_train);
SubMean->apply_to_feature_matrix(features_train);
SubMean->apply_to_feature_matrix(features_test);
Normalize->init(features_train);
Normalize->apply_to_feature_matrix(features_train);
Normalize->apply_to_feature_matrix(features_test);


We create an instance of CLeastAngleRegression by selecting to disable the LASSO solution, setting the penalty $$\lambda$$ for l1 norm and setting training data and labels.

lamda1 = 0.01
lars = LeastAngleRegression(False)
lars.set_features(features_train)
lars.set_labels(labels_train)
lars.set_max_l1_norm(lamda1)

lamda1 = 0.01;
lars = LeastAngleRegression(false);
lars.set_features(features_train);
lars.set_labels(labels_train);
lars.set_max_l1_norm(lamda1);

double lamda1 = 0.01;
LeastAngleRegression lars = new LeastAngleRegression(false);
lars.set_features(features_train);
lars.set_labels(labels_train);
lars.set_max_l1_norm(lamda1);

lamda1 = 0.01
lars = Shogun::LeastAngleRegression.new false
lars.set_features features_train
lars.set_labels labels_train
lars.set_max_l1_norm lamda1

lamda1 <- 0.01
lars <- LeastAngleRegression(FALSE)
lars$set_features(features_train) lars$set_labels(labels_train)
lars$set_max_l1_norm(lamda1)  lamda1 = 0.01 lars = shogun.LeastAngleRegression(False) lars:set_features(features_train) lars:set_labels(labels_train) lars:set_max_l1_norm(lamda1)  double lamda1 = 0.01; LeastAngleRegression lars = new LeastAngleRegression(false); lars.set_features(features_train); lars.set_labels(labels_train); lars.set_max_l1_norm(lamda1);  auto lamda1 = 0.01; auto lars = some<CLeastAngleRegression>(false); lars->set_features(features_train); lars->set_labels(labels_train); lars->set_max_l1_norm(lamda1);  Then we train the regression model and apply it to test data to get the predicted CRegressionLabels . lrr.train() labels_predict = lrr.apply_regression(features_test) bias = lrr.get_bias()  lrr.train(); labels_predict = lrr.apply_regression(features_test); bias = lrr.get_bias();  lrr.train(); RegressionLabels labels_predict = lrr.apply_regression(features_test); double bias = lrr.get_bias();  lrr.train labels_predict = lrr.apply_regression features_test bias = lrr.get_bias  lrr$train()
labels_predict <- lrr$apply_regression(features_test) bias <- lrr$get_bias()

lrr:train()
labels_predict = lrr:apply_regression(features_test)
bias = lrr:get_bias()

lrr.train();
RegressionLabels labels_predict = lrr.apply_regression(features_test);
double bias = lrr.get_bias();

lrr->train();
auto labels_predict = lrr->apply_regression(features_test);
auto bias = lrr->get_bias();


After training, we can extract $${\bf w}$$.

weights = lrr.get_w()

weights = lrr.get_w();

DoubleMatrix weights = lrr.get_w();

weights = lrr.get_w

weights <- lrr$get_w()  weights = lrr:get_w()  double[] weights = lrr.get_w();  auto weights = lrr->get_w();  Finally, we can evaluate the CMeanSquaredError. eval = MeanSquaredError() mse = eval.evaluate(labels_predict, labels_test)  eval = MeanSquaredError(); mse = eval.evaluate(labels_predict, labels_test);  MeanSquaredError eval = new MeanSquaredError(); double mse = eval.evaluate(labels_predict, labels_test);  eval = Shogun::MeanSquaredError.new mse = eval.evaluate labels_predict, labels_test  eval <- MeanSquaredError() mse <- eval$evaluate(labels_predict, labels_test)

eval = shogun.MeanSquaredError()
mse = eval:evaluate(labels_predict, labels_test)

MeanSquaredError eval = new MeanSquaredError();
double mse = eval.evaluate(labels_predict, labels_test);

auto eval = some<CMeanSquaredError>();
auto mse = eval->evaluate(labels_predict, labels_test);