% data preparation for utility parameters estimation using parametric version of ccps.

clear;
load 'C:\Users\LoweryR\Documents\My Dropbox\UND_research_Banking\Final_files\data_files\ccpdata_pvar_KLW.mat'

%{ 
    House_index              48245x1             385960  double              
  House_index_next         48245x1             385960  double              
  Senate_index             48245x1             385960  double              
  Senate_index_next        48245x1             385960  double              
  V                           11x11               968  double              
  assGrowth_1y_lag1        48245x1             385960  double              
  assGrowth_1y_lag2        48245x1             385960  double              
  assGrowth_1y_lag3        48245x1             385960  double              
  assGrowth_1y_lag4        48245x1             385960  double              
  asset_growth_1y          48245x1             385960  double              
  asset_growth_2y          48245x1             385960  double              
  assets                   48245x1             385960  double              
  assets_lag1              48245x1             385960  double              
  assets_lag2              48245x1             385960  double              
  assets_lag3              48245x1             385960  double              
  assets_lag4              48245x1             385960  double              
  equity                   48245x1             385960  double              
  equity_a                 48245x1             385960  double              
  equity_a_lag1            48245x1             385960  double              
  equity_a_lag2            48245x1             385960  double              
  equity_a_lag3            48245x1             385960  double              
  equity_a_lag4            48245x1             385960  double              
  estimated_cost           48245x1             385960  double              
  failed                   48245x1             385960  double              
  first_core_dep_a         48245x1             385960  double              
  log_assets               48245x1             385960  double              
  log_assets_lag1          48245x1             385960  double              
  log_assets_lag2          48245x1             385960  double              
  log_assets_lag3          48245x1             385960  double              
  log_assets_lag4          48245x1             385960  double              
  log_county_deposits      48245x1             385960  double              
  net_income               48245x1             385960  double              
  net_income_annual        48245x1             385960  double              
  nonperf_a_lag1           48245x1             385960  double              
  nonperf_a_lag2           48245x1             385960  double              
  nonperf_a_lag3           48245x1             385960  double              
  nonperf_a_lag4           48245x1             385960  double              
  nonperf_loans            48245x1             385960  double              
  nonperf_loans_a          48245x1             385960  double              
  realEst_own_a_lag1       48245x1             385960  double              
  realEst_own_a_lag2       48245x1             385960  double              
  realEst_own_a_lag3       48245x1             385960  double              
  realEst_own_a_lag4       48245x1             385960  double              
  real_estate_owned        48245x1             385960  double              
  real_estate_owned_a      48245x1             385960  double              
  roa_annual               48245x1             385960  double              
  roa_annual_lag1          48245x1             385960  double              
  roa_annual_lag2          48245x1             385960  double              
  roa_annual_lag3          48245x1             385960  double              
  roa_annual_lag4          48245x1             385960  double              
  roa_qtr                  48245x1             385960  double              
  rssd_id                  48245x1             385960  double              
  state                    48245x1             385960  double              
  unemp_nat                48245x1             385960  double              
  unemp_state              48245x1             385960  double              
  unemp_state_lag1         48245x1             385960  double              
  unemp_state_lag2         48245x1             385960  double              
  unemp_state_lag3         48245x1             385960  double              
  unemp_state_lag4         48245x1             385960  double              
  us_equity_a              48245x1             385960  double              
  us_equity_a_lag1         48245x1             385960  double              
  yearq                    48245x1             385960  double    
%}


ALLX = [rssd_id, yearq, failed, state, ... %1 to 4
  equity_a, equity_a_lag1, equity_a_lag2, equity_a_lag3, ... % 5 to 8
  nonperf_loans_a, nonperf_a_lag1, nonperf_a_lag2, nonperf_a_lag3, ... % 9 to 12
  log_assets, log_assets_lag1, log_assets_lag2, log_assets_lag3, ... % 13 to 16
  real_estate_owned_a, realEst_own_a_lag1, realEst_own_a_lag2, realEst_own_a_lag3, ...% 17 to 20
  roa_annual, roa_annual_lag1, roa_annual_lag2, roa_annual_lag3, ...% 21 to 24
  unemp_state, unemp_state_lag1, unemp_state_lag2, unemp_state_lag3,... % 25 to 28
  asset_growth_1y, assGrowth_1y_lag1, assGrowth_1y_lag2, assGrowth_1y_lag3, ... % 29 to 32
  House_index, Senate_index, House_index_next, Senate_index_next... % 33, 34, 35, 36
  assets, assets_lag1, assets_lag2, assets_lag3, ...; % 37 to 40
  first_core_dep_a,log_county_deposits,us_equity_a,us_equity_a_lag1]; % 41, 42, 43, 44

J = sum(isnan(ALLX(:,1:end-4)) + isinf(ALLX(:,1:end-4)),2); % note that you don't want to drop the observation if the last four variables are not available (this is ver.26 not ver.27)
J = logical(J > 0);
JJ = logical(net_income_annual >= 0); % no case
sum(JJ)
JJJ = logical(equity_a > 0.3); % no case
sum(JJJ)
JJJJ = logical(yearq > (1992 - 1960)*4 + (4-1) | yearq < (1985 - 1960)*4 + (4-1)); %only use between (1985Q4~1992Q4) 
% for transitions: "keep if inrange(yearq, yq(1985,4),yq(1992,4))"
% for cost function estimation: from 1985.3 to 1992.4
% after that, data range is 1985.4 to 1992.4
J = logical(J > 0 | JJ > 0  | JJJ > 0 | JJJJ > 0);
sum(J)


Jtemp = logical(J == 1 & ALLX(:,3) == 1); % loosing 190 closed banks since they don't have lags of varibles.
ALLXtemp = ALLX(Jtemp,:);

ALLX(J,:) = [];
ALLX = sortrows(ALLX,[1 2]);
BankID = ALLX(:,1);
yearq = ALLX(:,2);
Bclose = ALLX(:,3);
equity_a_all = ALLX(:,5:8);
npf_a_all = ALLX(:,9:12);
logasset_all = ALLX(:,13:16);
realEst_own_a_all = ALLX(:,17:20);
roa_annual_all = ALLX(:,21:24);
unemp_all = ALLX(:,25:28);%why do we need lags? they used in transition of other bank variables.
assGrowth_1y_all = ALLX(:,29:32);
House = ALLX(:,33);
Senate = ALLX(:,34); 
House_next = ALLX(:,35);
Senate_next = ALLX(:,36); 
asset_all = ALLX(:,37:40);
core_dep_a = ALLX(:,41);
logcounty_dep = ALLX(:,42);
us_equity_a_all = ALLX(:,43:44);

Jclose = logical(Bclose == 1);
Jopen = logical(Bclose == 0);

num_obs = size(ALLX,1);

% loading estimated probabilities
load 'C:\Users\LoweryR\Documents\My Dropbox\UND_research_Banking\Final_files\output_files\logit_predict_KLW.txt';
Eprob_now = logit_predict_KLW; % are they matched?
%{
    %%% If the model with trend is used:
load 'C:\Users\LoweryR\Documents\My Dropbox\UND_research_Banking\Final_files\output_files\logit_trend_predict_KLW.txt';
Eprob_t_now = logit_trend_predict_v26R; % are they matched?
%}
load 'C:\Users\LoweryR\Documents\My Dropbox\UND_research_Banking\Final_files\output_files\ccpnext_param_sim5000_KLW.txt';
%{
IDyearqP = [BankID, yearq, Eprob_Bclose, Eln_prob_Bclose, Emc_next,...
    Enpf_a_next, Eroa_next, Enpf_a_next2, Eroa_next2, Egrow_a_next, Erealest_a_next, Elogasset_next, Elogasset_next2, ...
	Eequity_a_next, Eunemp_next, Eprob_trend_Bclose, Eln_prob_trend_Bclose];
%}
sum(logical(BankID ~= ccpnext_param_sim5000_KLW(:,1)))
sum(logical(yearq ~= ccpnext_param_sim5000_KLW(:,2)))
Eprob_next = ccpnext_param_sim5000_KLW(:,3);
Eln_prob_next = ccpnext_param_sim5000_KLW(:,4);
Emc_next = ccpnext_param_sim5000_KLW(:,5);
Enpf_a_next2 = ccpnext_param_sim5000_KLW(:,8);
Eroa_next2 = ccpnext_param_sim5000_KLW(:,9);
Elogasset_next2 = ccpnext_param_sim5000_KLW(:,13);

%{
    %%% If the model with trend is used:
Eprob_t_next = ccpnext_param_sim5000_KLW(:,end-1);
Eln_prob_t_next = ccpnext_param_sim5000_KLW(:,end);
%}

beta_bankvar = xlsread('C:\Users\LoweryR\Documents\My Dropbox\UND_research_Banking\Final_files\data_files\coeff_resid_KLW.xlsx','trans_beta');%note that matlab does not import Strings. So the first col. in the excel file is not existing.
beta_logasset = beta_bankvar(:,1);
J = isnan(beta_logasset);
beta_logasset(J,:) = [];
beta_equity_a = beta_bankvar(:,2);
J = isnan(beta_equity_a);
beta_equity_a(J,:) = [];
beta_npf_a = beta_bankvar(:,3);
J = isnan(beta_npf_a);
beta_npf_a(J,:) = [];
beta_realest_a = beta_bankvar(:,4);
J = isnan(beta_realest_a);
beta_realest_a(J,:) = [];
beta_roa = beta_bankvar(:,5);
J = isnan(beta_roa);
beta_roa(J,:) = [];
beta_grow_a = beta_bankvar(:,6);
J = isnan(beta_grow_a);
beta_grow_a(J,:) = [];
beta_asset = beta_bankvar(:,7);
J = isnan(beta_asset);
beta_asset(J,:) = [];

Xfortran = [logasset_all,equity_a_all,npf_a_all,...%remember it is for tomorrow's, so we need to use current as the first lag.
realEst_own_a_all,roa_annual_all,assGrowth_1y_all,unemp_all,ones(num_obs,1)];

Elogasset_next = Xfortran*beta_logasset;
Eequity_a_next = Xfortran*beta_equity_a;
Enpf_a_next = Xfortran*beta_npf_a;
Ja = logical(Enpf_a_next < 0);
Enpf_a_next(Ja,:) = 0;
Erealest_a_next = Xfortran*beta_realest_a;
Jb = logical(Erealest_a_next < 0);
Erealest_a_next(Jb,:) = 0;
Eroa_next = Xfortran*beta_roa;
Egrow_a_next = Xfortran*beta_grow_a;
Easset_next = [asset_all,ones(num_obs,1)]*beta_asset;


beta_macrovar = xlsread('C:\Users\LoweryR\Documents\My Dropbox\UND_research_Banking\Final_files\data_files\coeff_resid_KLW.xlsx','unemp_beta_AR4');
beta_unemp = beta_macrovar(:,1);
J = isnan(beta_unemp);
beta_unemp(J,:) = [];


Eunemp_next = beta_unemp(1)*unemp_all(:,1) + beta_unemp(2)*unemp_all(:,2) + beta_unemp(3)*unemp_all(:,3) + beta_unemp(4)*unemp_all(:,4) + beta_unemp(5)*ones(size(unemp_all(:,1)));
beta_mc_sigma = xlsread('C:\Users\LoweryR\Documents\My Dropbox\UND_research_Banking\Final_files\data_files\coeff_resid_KLW.xlsx','cost_beta_v26');
beta_mc = beta_mc_sigma(1:end-1,:); sigma_mc = beta_mc_sigma(end);
%{ 
Monetary Cost by v26.
model	
log_assets	-.0307782
equity_a	-.9750841
nonperf_loans_a	1.614266
real_estate_owned_a	1.377229
roa_annual	-.9223549
unemp_state	.0138776
_cons	.0758327
sigma	
_cons	1.16055
%}

Xmc_now = [logasset_all(:,1), equity_a_all(:,1), npf_a_all(:,1), realEst_own_a_all(:,1), roa_annual_all(:,1), unemp_all(:,1), ones(num_obs,1)];
mc_now_temp = Xmc_now*beta_mc;
mc_now_temp = normcdf(mc_now_temp/sigma_mc,0,1).*mc_now_temp+sigma_mc*normpdf(mc_now_temp/sigma_mc,0,1); % See Wooldridge p674 (Eq.17.14)
Emc_now = mc_now_temp.*asset_all(:,1);

%{
    %%% If the model with trend is used:
save('C:\Users\LoweryR\Documents\My Dropbox\UND_research_Banking\Final_files\output_files\dataforparam_KLW.mat', ...
    'BankID', 'yearq', 'Bclose',...
'Eprob_now','Eprob_t_now','Eln_prob_next','Eprob_next','Eln_prob_t_next','Eprob_t_next','Emc_now','Emc_next',...
'equity_a_all', 'npf_a_all', 'logasset_all', 'realEst_own_a_all', 'roa_annual_all', 'unemp_all',...
'assGrowth_1y_all', 'House', 'Senate', 'House_next', 'Senate_next', 'asset_all',...
'Elogasset_next', 'Elogasset_next2','Eequity_a_next', 'Enpf_a_next', 'Enpf_a_next2', 'Erealest_a_next', 'Eroa_next', 'Eroa_next2', 'Egrow_a_next', 'Easset_next','Eunemp_next');
%}

% Since the model with trend is not utilized in this exercise, we set all variables with treand to be NaN:
Eprob_t_now = NaN(size(BankID));
Eprob_t_next = NaN(size(BankID));
Eln_prob_t_next = NaN(size(BankID));
save('C:\Users\LoweryR\Documents\My Dropbox\UND_research_Banking\Final_files\output_files\dataforparam_KLW.mat', ...
    'BankID', 'yearq', 'Bclose',...
'Eprob_now','Eprob_t_now','Eln_prob_next','Eprob_next','Eln_prob_t_next','Eprob_t_next','Emc_now','Emc_next',...
'equity_a_all', 'npf_a_all', 'logasset_all', 'realEst_own_a_all', 'roa_annual_all', 'unemp_all',...
'assGrowth_1y_all', 'House', 'Senate', 'House_next', 'Senate_next', 'asset_all',...
'Elogasset_next', 'Elogasset_next2','Eequity_a_next', 'Enpf_a_next', 'Enpf_a_next2', 'Erealest_a_next', 'Eroa_next', 'Eroa_next2', 'Egrow_a_next', 'Easset_next','Eunemp_next');

