% Name: Audrey Le
% Course: EE 8993-01
% Assignment: #3
% Purpose: the purpose of this assignment is to classify 2-dimensional 
% data using Linear Discriminant Analysis (LDA). Two approaches of LDA
% are used:  class dependent and class independent transformation.
%

% assisgn data points to be classified
%
a = [-1, -1];
b = [0, 0];
c = [0.5, 0.5];
d = [0.5, -0.5];

test_vectors = [a; b; c; d];

% load in the training data sets
%
% train_set_1.data and train_set_2.data were generated using xmgr with the
% desired distributions.  The format of both data sets is in X Y format.
%
load train_set_1.data;
load train_set_2.data;

% check the dimensions of the input data
%
[r1, c1] = size(train_set_1);

[r2, c2] = size(train_set_2);

if r1 ~= r2 
	fprintf('\nWARNING: sizes of data sets are unequal\n');
end

if c1 ~= c2
	fprintf('\nERROR: format of data sets is not same\n');
	exit;
end

% adjust the mean of the training data sets to have a mean of -2,2
% and 2,-2 respectively.
%
train_set_mean = mean(train_set_1);

offset = [-2,2] - train_set_mean;

for i = 1:r1,
%	adjusted_set_1(i,:) = train_set_1(i,:) - offset;
	adjusted_set_1(i,:) = train_set_1(i,:);
	adjusted_set_all(i,:) = adjusted_set_1(i,:);
end

train_set_mean = mean(train_set_2);

offset = [2,-2] - train_set_mean;

for i = 1:r2,
%	adjusted_set_2(i,:) = train_set_2(i,:) - offset;
	adjusted_set_2(i,:) = train_set_2(i,:);
	adjusted_set_all(i+r1,:) = adjusted_set_2(i,:);
end

% find the mean
%
mean_set_1 = mean(adjusted_set_1);
mean_set_2 = mean(adjusted_set_2);
mean_set_all = 0.5 * mean_set_1 + 0.5 * mean_set_2;

% find the Euclidean distance of each test point from each class in the
% original space
%
dist_a_to_set_1 = dist(a, transpose(mean_set_1));
dist_a_to_set_2 = dist(a, transpose(mean_set_2));
dist_b_to_set_1 = dist(b, transpose(mean_set_1));
dist_b_to_set_2 = dist(b, transpose(mean_set_2));
dist_c_to_set_1 = dist(c, transpose(mean_set_1));
dist_c_to_set_2 = dist(c, transpose(mean_set_2));
dist_d_to_set_1 = dist(d, transpose(mean_set_1));
dist_d_to_set_2 = dist(d, transpose(mean_set_2));

% determine the class of each test point
%
fprintf ('\nclassify data using original space:\n');

if (dist_a_to_set_1 < dist_a_to_set_2) 
   fprintf ('a[%.2f, %.2f] belongs to set_1\n', a(1,1), a(1,2));
else
   fprintf ('a[%.2f, %.2f] belongs to set_2\n', a(1,1), a(1,2));
end;

if (dist_b_to_set_1 < dist_b_to_set_2) 
   fprintf ('b[%.2f, %.2f] belongs to set_1\n', b(1,1), b(1,2));
else
   fprintf ('b[%.2f, %.2f] belongs to set_2\n', b(1,1), b(1,2));
end;

if (dist_c_to_set_1 < dist_c_to_set_2) 
   fprintf ('c[%.2f, %.2f] belongs to set_1\n', c(1,1), c(1,2));
else
   fprintf ('c[%.2f, %.2f] belongs to set_2\n', c(1,1), c(1,2));
end;

if (dist_d_to_set_1 < dist_d_to_set_2) 
   fprintf ('d[%.2f, %.2f] belongs to set_1\n', d(1,1), d(1,2));
else
   fprintf ('d[%.2f, %.2f] belongs to set_2\n', d(1,1), d(1,2));
end;

% find the decision region
%

i_1 = 1;
i_2 = 1;

% use arbitrary data points to find the decision region
%
for x = -5:0.1:5
   for y = -5:0.1:5
      dist_1 = dist(mean_set_1, transpose([x,y]));
      dist_2 = dist(mean_set_2, transpose([x,y]));

      % see if the sign of the distance difference has changed. if so
      % then add the point to the plot data and increment the index
      % value
      %
      if (dist_1 < dist_2) 
         orig_sp_plot_value_1(i_1,:) = [x,y];
         i_1 = i_1 + 1;
      elseif (dist_2 < dist_1) 
	 orig_sp_plot_value_2(i_2,:) = [x,y];
         i_2 = i_2 + 1;
      end;
   end
end

% plot the decision region
%
figure;
hold on;
title('Decision Region for the Two-Class Problem in Original Space');
xlabel('X');
ylabel('Y');
plot(adjusted_set_1(:,1), adjusted_set_1(:,2), 'ko');
plot(adjusted_set_2(:,1), adjusted_set_2(:,2), 'k+');
plot(test_vectors(:,1), test_vectors(:,2), 'g*');
plot(orig_sp_plot_value_1(:,1), orig_sp_plot_value_1(:,2), 'r.');
plot(orig_sp_plot_value_2(:,1), orig_sp_plot_value_2(:,2), 'b.');

% find the covariance
%
cov_set_1 = cov(adjusted_set_1);
cov_set_2 = cov(adjusted_set_2);
cov_set_all = cov(adjusted_set_all);

% find the within class scatter
%
sw = 0.5 * cov_set_1 + 0.5 * cov_set_2;

% find the between class scatter
%
set_mean(1,:) = mean_set_1;
set_mean(2,:) = mean_set_2;

sb = 0.5 * cov(set_mean);

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% class independent transform
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

criterion = inv(sw) * sb;

% compute the eigen vectors and eigen values
%
[eig_vect, eig_val] = eig(criterion);

% take the eigen vectors whose eigen values are non-zero to avoid 
% non-redundant set
%
for i=1:c1
        tmp_eig_val(i) = eig_val(i,i);
end;
 
% create the transform. This includes all eigen vectors with
% a corresponding non zero eigen value
%
k = 1;
for i=1:c1       
	if tmp_eig_val(i) ~= 0 
        	transform(:,k) = eig_vect(:,i);
	        k = k + 1;
	end;
end;

% transform the train data
%
trans_set_1 = transpose(transform(:,1)) * transpose(adjusted_set_1);
trans_set_2 = transpose(transform(:,1)) * transpose(adjusted_set_2);

% transform the test points
%
trans_a = transpose(transform(:,1)) * transpose(a);
trans_b = transpose(transform(:,1)) * transpose(b);
trans_c = transpose(transform(:,1)) * transpose(c);
trans_d = transpose(transform(:,1)) * transpose(d);

% find the Euclidean distance of each test point from each class in
% the transformed space
%
mean_trans_set_1 = mean(transpose(trans_set_1));
mean_trans_set_2 = mean(transpose(trans_set_2));

dist_a_to_trans_set_1 = dist(trans_a ,transpose(mean_trans_set_1));
dist_a_to_trans_set_2 = dist(trans_a ,transpose(mean_trans_set_2));
dist_b_to_trans_set_1 = dist(trans_b ,transpose(mean_trans_set_1));
dist_b_to_trans_set_2 = dist(trans_b ,transpose(mean_trans_set_2));
dist_c_to_trans_set_1 = dist(trans_c ,transpose(mean_trans_set_1));
dist_c_to_trans_set_2 = dist(trans_c ,transpose(mean_trans_set_2));
dist_d_to_trans_set_1 = dist(trans_d ,transpose(mean_trans_set_1));
dist_d_to_trans_set_2 = dist(trans_d ,transpose(mean_trans_set_2));

% determine the class of each test point
%
fprintf ('\nclassify data using class independent transform:\n');

if (dist_a_to_trans_set_1 < dist_a_to_trans_set_2) 
   fprintf ('a[%.2f, %.2f] belongs to set_1\n', a(1,1), a(1,2));
else
   fprintf ('a[%.2f, %.2f] belongs to set_2\n', a(1,1), a(1,2));
end;

if (dist_b_to_trans_set_1 < dist_b_to_trans_set_2) 
   fprintf ('b[%.2f, %.2f] belongs to set_1\n', b(1,1), b(1,2));
else
   fprintf ('b[%.2f, %.2f] belongs to set_2\n', b(1,1), b(1,2));
end;

if (dist_c_to_trans_set_1 < dist_c_to_trans_set_2) 
   fprintf ('c[%.2f, %.2f] belongs to set_1\n', c(1,1), c(1,2));
else
   fprintf ('c[%.2f, %.2f] belongs to set_2\n', c(1,1), c(1,2));
end;

if (dist_d_to_trans_set_1 < dist_d_to_trans_set_2) 
   fprintf ('d[%.2f, %.2f] belongs to set_1\n', d(1,1), d(1,2));
else
   fprintf ('d[%.2f, %.2f] belongs to set_2\n', d(1,1), d(1,2));
end;

% find the decision region
%

i_1 = 1;
i_2 = 1;

% use arbitrary data points to find the decision region
%
for x = -5:0.1:5
   for y = -5:0.1:5
      dist_1 = dist(mean_trans_set_1, transpose(transform(:,1)) * transpose([x,y]));
      dist_2 = dist(mean_trans_set_2, transpose(transform(:,1)) * transpose([x,y]));

      % see if the sign of the distance difference has changed. if so
      % then add the point to the plot data and increment the index
      % value
      %
      if (dist_1 < dist_2) 
         class_ind_plot_value_1(i_1,:) = [x,y];
         i_1 = i_1 + 1;
      elseif (dist_2 < dist_1) 
	 class_ind_plot_value_2(i_2,:) = [x,y];
         i_2 = i_2 + 1;
      end;
   end
end

% plot the decision region
%
figure;
hold on;
title('Decision Region for the Two-Class Problem Using Class-Independent Trans.');
xlabel('X');
ylabel('Y');
plot(adjusted_set_1(:,1), adjusted_set_1(:,2), 'ko');
plot(adjusted_set_2(:,1), adjusted_set_2(:,2), 'k+');
plot(test_vectors(:,1), test_vectors(:,2), 'g*');
plot(class_ind_plot_value_1(:,1), class_ind_plot_value_1(:,2), 'r.');
plot(class_ind_plot_value_2(:,1), class_ind_plot_value_2(:,2), 'b.');


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% class dependent transform
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

criterion_1 = inv(cov_set_1) * sb;
criterion_2 = inv(cov_set_2) * sb;

% compute the eigen vectors and eigen values
%
[eig_vect_1, eig_val_1] = eig(criterion_1);
[eig_vect_2, eig_val_2] = eig(criterion_2);

% take the eigen vectors whose eigen values are non-zero to avoid 
% non-redundant set
%
for i=1:c1
        tmp_eig_val_1(i) = eig_val_1(i,i);
        tmp_eig_val_2(i) = eig_val_2(i,i);
end;
 
% create the transform. This includes all eigen vectors with
% a corresponding non zero eigen value
%
k = 1;
l = 1;
for i=1:c1       
	if tmp_eig_val_1(i) ~= 0 
        	transform_1(:,k) = eig_vect_1(:,i);
	        k = k + 1;
	end;

	if tmp_eig_val_2(i) ~= 0 
        	transform_2(:,l) = eig_vect_2(:,i);
	        l = l + 1;
	end;
end;

% transform the train data
%
trans_1_set_1 = transpose(transform_1(:,1)) * transpose(adjusted_set_1);
trans_2_set_2 = transpose(transform_2(:,1)) * transpose(adjusted_set_2);

% transform the test points
%
trans_1_a = transpose(transform_1(:,1)) * transpose(a);
trans_2_a = transpose(transform_2(:,1)) * transpose(a);
trans_1_b = transpose(transform_1(:,1)) * transpose(b);
trans_2_b = transpose(transform_2(:,1)) * transpose(b);
trans_1_c = transpose(transform_1(:,1)) * transpose(c);
trans_2_c = transpose(transform_2(:,1)) * transpose(c);
trans_1_d = transpose(transform_1(:,1)) * transpose(d);
trans_2_d = transpose(transform_2(:,1)) * transpose(d);

% find the Euclidean distance of each test point from each class in
% the transformed space
%
mean_trans_1_set_1 = mean(trans_1_set_1);
mean_trans_2_set_2 = mean(trans_2_set_2);

dist_a_to_trans_1_set_1 = dist(transpose(trans_a), transpose(mean_trans_1_set_1));
dist_a_to_trans_2_set_2 = dist(transpose(trans_a), transpose(mean_trans_2_set_2));
dist_b_to_trans_1_set_1 = dist(transpose(trans_b), transpose(mean_trans_1_set_1));
dist_b_to_trans_2_set_2 = dist(transpose(trans_b), transpose(mean_trans_2_set_2));
dist_c_to_trans_1_set_1 = dist(transpose(trans_c), transpose(mean_trans_1_set_1));
dist_c_to_trans_2_set_2 = dist(transpose(trans_c), transpose(mean_trans_2_set_2));
dist_d_to_trans_1_set_1 = dist(transpose(trans_d), transpose(mean_trans_1_set_1));
dist_d_to_trans_2_set_2 = dist(transpose(trans_d), transpose(mean_trans_2_set_2));

% determine the class of each test point
%
fprintf ('\nclassify data using class dependent transform:\n');

if (dist_a_to_trans_1_set_1 < dist_a_to_trans_2_set_2) 
   fprintf ('a[%.2f, %.2f] belongs to set_1\n', a(1,1), a(1,2));
else
   fprintf ('a[%.2f, %.2f] belongs to set_2\n', a(1,1), a(1,2));
end;

if (dist_b_to_trans_1_set_1 < dist_b_to_trans_2_set_2) 
   fprintf ('b[%.2f, %.2f] belongs to set_1\n', b(1,1), b(1,2));
else
   fprintf ('b[%.2f, %.2f] belongs to set_2\n', b(1,1), b(1,2));
end;

if (dist_c_to_trans_1_set_1 < dist_c_to_trans_2_set_2) 
   fprintf ('c[%.2f, %.2f] belongs to set_1\n', c(1,1), c(1,2));
else
   fprintf ('c[%.2f, %.2f] belongs to set_2\n', c(1,1), c(1,2));
end;

if (dist_d_to_trans_1_set_1 < dist_d_to_trans_2_set_2) 
   fprintf ('d[%.2f, %.2f] belongs to set_1\n', d(1,1), d(1,2));
else
   fprintf ('d[%.2f, %.2f] belongs to set_2\n', d(1,1), d(1,2));
end;

% find the decision region
%

i_1 = 1;
i_2 = 1;

% use arbitrary data points to find the decision region
%
for x = -5:0.1:5
   for y = -5:0.1:5
      dist_1 = dist(mean_trans_1_set_1, transpose(transform_1(:,1)) * transpose([x,y]));
      dist_2 = dist(mean_trans_2_set_2, transpose(transform_2(:,1)) * transpose([x,y]));

      % see if the sign of the distance difference has changed. if so
      % then add the point to the plot data and increment the index
      % value
      %
      if (dist_1 < dist_2) 
         class_dep_plot_value_1(i_1,:) = [x,y];
         i_1 = i_1 + 1;
      elseif (dist_2 < dist_1) 
	 class_dep_plot_value_2(i_2,:) = [x,y];
         i_2 = i_2 + 1;
      end;
   end
end

% plot the decision region
%
figure;
hold on;
title('Decision Region for the Two-Class Problem Using Class-Dependent Trans.');
xlabel('X');
ylabel('Y');
plot(adjusted_set_1(:,1), adjusted_set_1(:,2), 'ko');
plot(adjusted_set_2(:,1), adjusted_set_2(:,2), 'k+');
plot(test_vectors(:,1), test_vectors(:,2), 'g*');
plot(class_dep_plot_value_1(:,1), class_dep_plot_value_1(:,2), 'r.');
plot(class_dep_plot_value_2(:,1), class_dep_plot_value_2(:,2), 'b.');





