* Fund sample selection almost exactly follows Jiang Verbeek Wang(2012) which may be considered as an enhancement based on KacperczykSialmZheng(2008RFS)'s selection;

*ods html close; 
*ods preferences;

dm "out;clear;log;clear;";

libname pmf "/home/usc/junbow";

libname IVBProj "/scratch/usc/temp";
* libname EHDCRSP "CRSPQ";		* libname EHDComp "COMPA";		*full CRSP and Compustat;
* libname EHDlink "CRSPA";		*full CRSP and Compustat;



options errors=1 noovp;
options nocenter ps=max ls=78;
options mprint source nodate symbolgen macrogen msglevel=i;

%let serverID = 2;


%let CRSPCutYr = 1952; 

** CRSP Monthly Frequency Data;
***********************************************; * Following CGS (2012); 
***********************************************; * 1. NYSE/AMEX/NASDAQ COMMON stocks only;						 
* - Manipulating CRSP Individual Securities   *; * 2. Penny (<1 USD) stocks are dropped;						 
***********************************************; * 3. Logs of (MkCap,B2M,Turnover) are included; 
***********************************************; * 4. At least 5 Yrs (not consecutive) to compute MkCap and B2M, i.e Returns;
* *;											 * 5. Drop financial companies;
* *;											 * 6. Market Cap-based Screenings (0.1 Million);
*** - Data set is extended to Oct-2012;
* Macros;
%Macro PreProcRawCRSP(dname1=,dname2=,DataCutYr=,outdname=);
* DSEnames;
data CRSPnames (drop=shrcd); set &dname1 
	(keep=permno namedt nameendt shrcd exchcd siccd ticker);   		* ADD more variables, depending on applications;
	where (shrcd = 10 or shrcd = 11) and (exchcd = 1 or exchcd = 2 or exchcd = 3) and nmiss(permno) = 0; *Common shares and NYSE/AMEX/NASDAQ;									
	if not(missing(nameendt)) and year(nameendt) < &DataCutYr then delete;		
	format namedt nameendt date9.; run; 		
proc sort data=CRSPnames; by permno namedt nameendt; run;
* DSF;																* ADD more variables, depending on applications;
data CRSPRaw; set &dname2 (keep=permno date prc vol ret shrout where=(year(date)>=&DataCutYr)); format date date9.; run;
proc sort data=CRSPRaw; by permno date; run;
proc sql; create table &outdname (drop=Npermno namedt nameendt) as select *      
	from CRSPRaw as a, CRSPnames (rename=(permno=Npermno)) as b
	where a.permno = b.Npermno and b.namedt <= a.date <= b.nameendt; quit;  *Assuming no missing of namedt and nameendt (checked OK);
* Sanity check for DSEnames;
data Dsenames_check; set &dname1; if missing(namedt) or missing(nameendt); run;
proc datasets lib=work; delete CRSPRaw CRSPnames Dsenames_check; quit;
%Mend PreProcRawCRSP;
* Macro to selectively run PreProcRawCRSP for laptop and server;
%Macro PreProcRawCRSPd_Sel(OSid=,DataCutYr1=);
%if &OSid = 1 %then %do;
	data Pre_CRSPd; set crspa.dsf (where=(year(date)>=2000)); run; %end; *PARTIAL data only for laptop testing purpose;
%if &OSid > 1 %then %do;
	* Pre-Processing of Daily raw CRSP data;
	%PreProcRawCRSP(dname1=CRSPQ.Dsenames,dname2=CRSPQ.dsf,DataCutYr=&DataCutYr1,outdname=Pre_CRSPd); %end;
%Mend PreProcRawCRSPd_Sel;

* Execution;
%PreProcRawCRSP(dname1=CRSPQ.Msenames,dname2=CRSPQ.msf,DataCutYr=&CRSPCutYr,outdname=Pre_CRSPm); 
data CRSPm_tmp; retain permno ticker; set Pre_CRSPm;
	where prc ne 0 and vol > 0 and shrout > 0;
	/* if siccd >= 6000 and siccd <= 6800 then delete; 	* drop financial firms;   */
	prc = abs(prc); 	  		*CRSP price;
	vol = vol*100; 		  		*share volume in hundreds (Monthly data only); *Total number of shares traded on that month;
	shrout = shrout*1000; 		*shrout in thousands;	
	MkCapm = prc*shrout;		*market capitalization; 
	Turn = vol/shrout;	  		*Turnover;
	LogTurn = log(Turn);		*Log turnover;	* Log(B2M) is implemented below;
	Sizem = log(MkCapm);		*Size proxy;
	Dvol = vol*prc;				*Dollar volume; 
	if prc < 1 then delete;		*dropping penny stocks at a given month;
	label MkCapm = "Market Capitalization from Monthly CRSP"; 		label Sizem = "Size of firm (Log of Monthly MkCap)";
	label Dvol = "Dollar volume";	label prc = "CRSP Price";		label Turn = "Turnover (Vol/Shrout)"; 
	label LogTurn = "Log of Turnover"; run;
proc sort data=CRSPm_tmp; by permno date; run;


*** - Market Cap-based Stock Screening;
%let MkCap_Cut = 6;	*in power;
data CRSPm_tmp; set CRSPm_tmp; if MkCapm >= 10**&MkCap_Cut; run;

*** - CRSP Delisting bias correction (following Shumway,JF,1997);
* Macro;
%Macro CorrectDelBias(dname=,Outdname=,MrgOpt=);
* Loading CRSP delisting files;
data MseDelist; set CRSPQ.MseDelist (keep=permno dlstdt dlstcd DlRet);
	where not(missing(permno)) and not(missing(dlstcd)) and not(missing(DlRet)); 
	if dlstcd in (500,520,580,574,584) or (dlstcd>=551 and dlstcd<=573) then DlRet = - 0.3; 
	format dlstdt date9.; if year(dlstdt)>2011 or abs(DlRet)>1 then delete; run;
proc sort data=MseDelist; by permno dlstdt; run;

* Merging with CRSP msf;
%if &MrgOpt = 0 %then %do;
proc sql; create table &Outdname (drop=Npermno dlstdt dlstcd) as select *
	from &dname as a left join MseDelist (rename=(permno=Npermno)) as b
	on a.permno = b.Npermno	and 
	year(a.date) = year(b.dlstdt) and month(a.date) = month(b.dlstdt); quit;
data &Outdname (drop=DlRet); set &Outdname; if not(missing(DlRet)) then Ret = DlRet; run; %end;
%if &MrgOpt = 1 %then %do;
proc sql; create table &Outdname (drop=Npermno dlstdt dlstcd) as select *
	from &dname as a left join MseDelist (rename=(permno=Npermno)) as b
	on a.permno = b.Npermno; quit;
proc sort data=&Outdname; by permno date; run;
data &Outdname (drop=DlRet); set &Outdname; by permno; if not(missing(DlRet)) and last.permno then Ret = DlRet; run; %end;
proc datasets lib=work; delete MseDelist; quit;
%Mend CorrectDelBias;

* Execution;
%CorrectDelBias(dname=CRSPm_tmp,Outdname=CRSP_FFm_tmp,MrgOpt=1);
/* data CRSP_FFm_tmp; set CRSPm_tmp; run; */

*** - Merging CRSP and Monthly FF Factors;
data FFm; retain dateff; set FF.Factors_monthly; 
	format dateff date9.; if year(dateff)>=&CRSPCutYr; drop date year month; run;
proc sql; create table CRSP_FFm (drop=dateff) as select *
	from CRSP_FFm_tmp as a, FFm as b
	where year(a.date) = year(b.dateff) and month(a.date) = month(b.dateff); quit;
proc sort data=CRSP_FFm; by permno date; run;
proc datasets lib=work; delete CRSP_FFm_tmp; quit;
* Output: CRSP_FFm;


*** - Computing Ratio of CAPITALIZATIONs of the MKT PORTFOLIOS based on July 1962;
** TS starts from Jan. 1952;
** Equally-weighted Mkt Caps across all stocks;
** Used for Amihud illiquidity measure normalization below;
proc sort data=CRSP_FFm out=CRSP_FFm_sort (keep=permno date MkCapm); by date permno; run;
proc means data=CRSP_FFm_sort noprint; by date;
	var MkCapm; output out=Avg_MkCapm (drop=_type_ _freq_) mean= /autoname; run;
data Avg_MkCapm; set Avg_MkCapm;
	if year(date)=1962 and month(date)=7 then do; call symputx('AvgMkCap0',MkCapm_Mean); end; run;
data Avg_MkCapm; set Avg_MkCapm; AvgMkCap0 = &AvgMkCap0;
	MkCapm_Ratio = MkCapm_Mean/&AvgMkCap0; 
	label MkCapm_Mean = "X-sectional average Market Capitalization at a given month";
	label MkCapm_Ratio = "Market Capitalization Ratio of Mkt Portfolios based on Jul 1962";
	label AvgMkCap0 = "Market Capitalization of Mkt Portfolio at July 1962"; run;
proc sort data=Avg_MkCapm; by date; run;
data IVBProj.Avg_MkCapm; set Avg_MkCapm; run;
proc datasets lib=work; delete CRSP_FFm_sort; quit;
* Output: Avg_MkCapm (Starting date is Jan 1952, MkCapm_Ratio is computed based on July 1962);


*** - Six-month Momentum_t: Cumulative returns from t-2 to t-7;
* Mom6NCut: minimum number of non-missing months to be included (<=6);
* Macro;
%Macro GetMoM6(dname=,Mom6NCut=,OutDname=);
* Table looking back 6 months;
data Mom6M; set &dname; M6fdate = intnx('Month',date,-7,'Begin'); M6ldate = intnx('Month',date,-2,'End');
	format M6fdate M6ldate date9.; keep permno date Ret M6fdate M6ldate; run;
data Mom6M_6; set Mom6M; keep permno date Ret; label Ret = "Past return rate for Six-month Momentum Calculation";
	rename permno = permno_6 date = date_6 Ret = Ret_6; run;
proc sql; create table Mom6M_Tab_tmp as select *
	from Mom6M as a, Mom6M_6 as b
	where a.permno = b.permno_6 and a.M6fdate <= b.date_6 <= a.M6ldate; quit;
proc sort data=Mom6M_Tab_tmp; by permno date date_6; run;
* Computing momentums;
proc means data=Mom6M_Tab_tmp (drop=M6fdate M6ldate permno_6 date_6) noprint;
	by permno date; var Ret_6; output out=Mom6M_Tab (drop=_type_) sum = /autoname; run;
data Mom6M_Tab; set Mom6M_Tab;
	label _freq_ = "Number of available (non-missing) past months for Six-month momentum computation";
	label Ret_6_Sum = "Momentum (Return) based on past 6 months"; rename _freq_=Mom6N Ret_6_Sum=Mom6; run;
proc sort data=Mom6M_Tab; by permno date; run;

* Merging back into the original CRSP data;
proc sql; create table &OutDname (drop=permnoN dateN) as select *
	from &dname as a, Mom6M_Tab (rename=(permno=permnoN date=dateN)) as b
	where a.permno = b.permnoN and year(a.date)=year(b.dateN) and month(a.date)=month(b.dateN); quit;
proc sort data=&OutDname (where=(Mom6N>=&Mom6NCut)); by permno date; run;
proc datasets lib=work; delete Mom6M Mom6M_6 Mom6M_Tab_Tmp Mom6M_Tab; quit;
%Mend GetMoM6;

* Execution;
%GetMoM6(dname=CRSP_FFm,Mom6NCut=4,OutDname=CRSP_MoM6);  *at least 4 past months are needed;
* Output: CRSP_MoM6; 


* COMPUSTAT starts from June, 1951;
**********************************************; * CCM products in the CRSP folder;
* - Merging CRSP and Compustat through CCM   *; * Book-to-Market (B2M) ratios are also calculated;
**********************************************; * ACC/FIN items merging TIMING: 6 month dealy - start from July (ie end of June);
* Macro;
%Macro MergeCRSPnCompnBM(CompVars1=,CompDname=,CCMDname=,CRSPDname=,DelayMonth=,DataCutYr=,OutDname=);
/* %let CompVars1=&CompVars;	%let CompDname=COMPA.funda;		%let CCMDname=CRSPA.ccmxpf_linktable;		
%let CRSPDname=CRSP_MoM6;		%let DelayMonth=6;					%let DataCutYr=&CRSPCutYr;
%let OutDname=CRSP_CCM_Comp; 				*/
** - Step 1: Loading Compustat data;
%let CompVars2 = indfmt datafmt popsrc consol; *built-in Compustat variables;
data Comp; set &CompDname (keep=&CompVars1 &CompVars2); 
	where not(missing(gvkey)) and year(datadate)>=&DataCutYr; 
	if (indfmt='INDL' or indfmt='FS') and datafmt='STD' and popsrc='D' and consol='C';
	* Industrial and Finanail Service are included;
	* BookValue computation;								 			*PSTKRV: Preferred stock - Redemption value;
	Pref=coalesce(PSTKRV,PSTKL,PSTK,0.0); 					 			*PSTKL: Preferred stock - liquidating value; 
	if missing(PSTK) then PSTK = 0.0;						 			*PSTK: Preferred/Preference Stocks (Capital) - Total;
	if missing(TXDITC) then TXDITC = 0.0;					 			*TXDITC: Deferred taxes and investment tax credit;
	if SEQ ne . then BookValue = SEQ + (TXDITC - Pref); 	 			*SEQ: Stockholder equity - Total;
	else if CEQ ne . then BookValue = CEQ + PSTK + (TXDITC - Pref);  	*CEQ: Common/Ordinary equity - Total;
	else BookValue = AT - LT + (TXDITC - Pref);						 	*AT: Asset - Total;    *LT: liabilities - Total;
	csho = csho*10**6; 					*# of shares outstanding;
	BookValue = BookValue*10**6;		*Unit conversion;
	BookValue_Naive = bkvlps*csho;  	*Naive book-equity;
	endfyr = datadate;			begfyr = intnx('Month',endfyr,-11,'Begin');
	format datadate date9.; 	format endfyr begfyr yymmddn8.;		
	drop &CompVars2	PSTK PSTKRV PSTKL AT LT TXDITC SEQ CEQ Pref; run;
proc sort data=Comp; by gvkey datadate; run;

** - Step 2: Merging with CCM LinkTable (having CRSP-permnos);
proc sql; create table Comp_CCM (drop=Ngvkey endfyr begfyr) as select *
	from Comp as a, &CCMDname (rename=(gvkey=Ngvkey)) as b
	where a.gvkey = b.Ngvkey and b.usedflag=1 and 
	b.linktype in ("LU","LC","LD","LN","LS","LX") and
	(b.linkdt<=a.endfyr or b.linkdt=.B) and (a.endfyr<=b.linkenddt or b.linkenddt=.E); quit;

** - Step 3: Computing the ANNUAL B2M ratios (Following FF (1992) and BCS (1998)); 
data CRSP_Ann; set &CRSPDname (keep=permno date prc shrout); if month(date) = 12; run; *pick up Dec-ME only;
proc sort data=CRSP_Ann; by permno date; run;
proc sql; create table Comp_CCM_B2M as select *
	from Comp_CCM as a, CRSP_Ann as b
	where a.lpermno = b.permno and year(a.datadate) = year(b.date); quit;
data Comp_CCM_B2M; set Comp_CCM_B2M; where BookValue > 0;
	B2M = BookValue/(prc*shrout); 					logB2M = log(B2M); 
	label B2M = "Book-to-Market Ratio";				label logB2M = "Logarithm of Book-to-Market Ratio";
	label date = "Market Equity (ME) picked-up date for B2M calculation"; 
	drop permno prc shrout linkprim liid linktype lpermco usedflag linkdt linkenddt; rename date=B2Mdate; run;
data Comp_CCM; set Comp_CCM_B2M; where not(missing(B2M)) and not(missing(logB2M)); run;
proc sort data=Comp_CCM; by lpermno B2Mdate; run;
/** Sanity check for B2M computation;
proc sort data=Comp_CCM out=Comp_CCM_Check_tmp; by datadate gvkey; run;
proc means data=Comp_CCM_Check_tmp noprint; by datadate; var B2M; output out=Comp_CCM_Check mean=/autoname; run; */

** - Step 4: Merging MONTHLY CRSP and ANNUAL Compustat;
* Matching Compustat and CRSP, allow 6 months delay (b/c END-of-month consideration);
%let DelayMonth1 = &DelayMonth;				%let DelayMonth2 = %eval(&DelayMonth+11);
proc sql; create table &OutDname (drop=lpermno) as select *
	from &CRSPDname as a, Comp_CCM as b 
	where a.permno = b.lpermno and 
	intck('Month',b.B2Mdate,a.date) between &DelayMonth1 and &DelayMonth2; quit;
	** Should be B2Mdate instead of datadate!!;
proc sort data=&OutDname (drop=BookValue_Naive); by permno date B2Mdate; run;
proc datasets lib=work; delete Comp Comp_CCM CRSP_Ann Comp_CCM_B2M; quit;
%Mend MergeCRSPnCompnBM;

* Execution;
%let CompVars = gvkey datadate fyear tic cusip conm fyr bkvlps csho PSTKRV PSTKL PSTK TXDITC SEQ CEQ AT LT; 
%MergeCRSPnCompnBM(CompVars1=&CompVars,CompDname=COMPA.funda,CCMDname=CRSPA.ccmxpf_linktable,
				   CRSPDname=CRSP_MoM6,DelayMonth=6,DataCutYr=&CRSPCutYr,OutDname=CRSP_CCM_Comp); 

* Sanity check with Duplicates;
proc sort data=CRSP_CCM_Comp dupout=CRSP_CCM_Comp_DupKey NoDupkey; by permno date; run;
proc univariate data=CRSP_CCM_Comp_DupKey noprint; by permno;
	var date; output out=CRSP_CCM_Comp_DupKey_Check N=TS_OBS; run;


** - B2M Winsorization with 0.005 and 0.995 percentiles for a given month;
* 	 type=1: Winsorization, type=2: Trimming;
* Macro;
%Macro GetWinsor(dname=,vname=,byvname=,pctl=,type=);
/* %let dname=CRSP_CCM_Comp;	%let vname=B2M;		%let byvname=date;
%let pctl=0.5 99.5;			%let type=1; 	*/
proc sort data=&dname; by permno &byvname; run;
proc sort data=&dname out=tmp_data; by &byvname &vname; run;
proc univariate data=tmp_data noprint; by &byvname; var &vname;
	output out=pctl_tmp_data PCTLPTS=&pctl PCTLPRE=pctl_ PCTLNAME=L H N=freq; run;
proc sql; create table New_tmp_data as select *
	from tmp_data as a, pctl_tmp_data (rename=(&byvname=N_&byvname)) as b 
	where a.&byvname = b.N_&byvname; quit;
*Type I - Winsorization;
%if &type = 1 %then %do;
data New_tmp_data; set New_tmp_data (drop=N_&byvname);
	if not(missing(&vname)) and freq >= 20 and &vname < pctl_L then &vname=pctl_L;
	if not(missing(&vname)) and freq >= 20 and &vname > pctl_H then &vname=pctl_H; run; %end;
*Type II - Trimming (set to missing);
%if &type = 2 %then %do;
data New_tmp_data; set New_tmp_data (drop=N_&byvname);
	if not(missing(&vname)) and freq >= 20 and &vname < pctl_L then &vname = .;
	if not(missing(&vname)) and freq >= 20 and &vname > pctl_H then &vname = .; run; %end;
proc sort data=New_tmp_data (drop=freq pctl_L pctl_H); by permno &byvname; run;
data tmp_data2; merge &dname (drop=&vname) New_tmp_data; run; 
data &dname; set tmp_data2; run;
proc datasets lib=work; delete tmp_data pctl_tmp_data New_tmp_data tmp_data2; quit; 
%MEND GetWinsor;

* Execution;
%GetWinsor(dname=CRSP_CCM_Comp,vname=B2M,byvname=date,pctl=0.5 99.5,type=1);
* LogB2M is recalculated after winsorization;
data CRSP_CCM_Comp; set CRSP_CCM_Comp; LogB2M = log(B2M); label logB2M = "Logarithm of Book-to-Market Ratio"; run;



*** - 1. Requiring at least 5years (non-consecutive) of Rets following CGS (2012) 
*		 over the WHOLE sample period;
* Macro;
%Macro GetTSCntCut(dname=,vname=,Xvname=,CntCut=,OutDname=); 
/*  %let dname=CRSPm_tmp;		%let vname=Prc;		%let Xvname=permno;
%let CntCut=60;			%let OutDname=CRSPm;  */
/* %let dname=CRSP_Comp_Mon_AmiTab_SP1;	%let vname=Ret;
%let Xvname=permno;		%let CntCut=36;	%let OutDname=CRSP_Comp_Mon_AmiTab;		*/
proc sort data=&dname; by &Xvname date; run;
proc univariate data=&dname noprint; by &Xvname; 
	var &vname; output out=&vname._Cnt (where=(&vname._Cnt>=&CntCut)) N=&vname._Cnt; run;
proc sql; create table &OutDname (drop=N&Xvname) as select *
	from &dname as a, &vname._Cnt (rename=(&Xvname=N&Xvname) drop=&vname._Cnt) as b 
	where a.&Xvname = b.N&Xvname; quit;
proc sort data=&OutDname; by &Xvname date; run;
proc datasets lib=work; delete &dname &vname._Cnt; quit; * special case;
%Mend GetTSCntCut; 

* Execution;
%let TSCnt_Cut = 60;	*Minimum time-series length;
%GetTSCntCut(dname=CRSP_CCM_Comp,vname=Ret,Xvname=permno,CntCut=&TSCnt_Cut,OutDname=CRSP_CCM_Comp_Mon_Tab);
data IVBProj.CRSP_CCM_Comp_Mon_Tab; set CRSP_CCM_Comp_Mon_Tab; run;
* Output: CRSP_CCM_Comp_Mon_Tab;
*** - End of Processing of Individual Monthly data;



***************************;
* - For Testing AP LCAPM  *;
*********************************; * After computing AP monthly Amihud illiquidity measures (Normed and Trunced),
* - CRSP Daily Data Processing  *; * merged into the CRSP MONTHLY data (Processed above);
*********************************; 
** - Analyzing the NYSE and AMEX common shares as in AP (Drop NASDAQ stocks here!!);
**   All other screenings will be AUTOMATICALLY applied while combining with CRSP Monthly;
** - No screenings based on Trading days in a given month YET. Might need to do so later;
%PreProcRawCRSPd_Sel(OSid=&ServerID,DataCutYr1=&CRSPCutYr); 
data CRSPd; retain permno date dyear dmonth dday; set Pre_CRSPd;
	where abs(prc)>0 and vol>0 and shrout>0;
	/* if siccd >= 6000 and siccd <= 6800 then delete; 	* drop financial firms; 	*/
	prc = abs(prc); 	  					*CRSP price;
	*vol = vol*100; 		  				*share volume NOT in hundreds (for CRSP Daily);
	shrout = shrout*1000; 					*shrout in thousands;	
	MkCap = prc*shrout;   					*market capitalization; 
	Dvol = vol*prc;							*Dollar volume;
	Amihud = abs(Ret)/Dvol*10**6;			*Daily Amihud illiquidity measure (Ret/DVol in millions);
	dyear=year(date); dmonth=month(date); dday=day(date); 
	if exchcd in (1,2);						*Keeping NYSE and AMEX Only;
	keep permno date prc vol Ret shrout MkCap Dvol Amihud dyear dmonth dday;
	label MkCap = "Market Capitalization by Daily CRSP"; 
	label DVol = "Dollar Volume"; label date = "date"; 
	label Amihud = "Amihud Daily Illiquidity Measure [Abs(Ret)/DVol in Millions]"; run;

** - Computing Monthly Amihud measures (and also return volatility);
proc sort data=CRSPd; by permno dyear dmonth dday; run;
proc means data=CRSPd noprint; by permno dyear dmonth; 
	var Amihud ret; output out=Amihud_Mon (drop=_type_) mean=Amihud_Mon Ret_Mean std=Amihud_Vol_Mon Ret_Vol_Mon; run;
proc sort data=Amihud_Mon (drop=Ret_Mean); by permno dyear dmonth; run;
data Amihud_Mon (drop=dyear dmonth); retain permno date dyear dmonth; set Amihud_Mon; 
	format date date9.; date = mdy(dmonth,01,dyear); date = intnx('Month',date,0,'End');
	label Amihud_Mon = "Monthly Amihud Illiquidity Measures";
	label Amihud_Vol_Mon = "Monthly Volatility of Amihud Illiquidity Measures";
	label Ret_Vol_Mon = "Monthly Volatility of Daily Returns";
	label _Freq_ = "Number of Days to compute Monthly Amihud measures"; rename _Freq_ = Amihud_Mon_Cnt; run;













** Start the daily data part 
%let serverID = 2;


%let CRSPCutYr = 1952; 
%let DimsonT_Th = 1;



* Macro to Pre-Process the DAILY CRSP dsf and DSEnames files;
* Necessary when using the RAW CRSP data on WRDS server;
* (When WRDS web-queried data are used, No need);
* Input: dname1 (location of Dsename file);
*		 dname2 (location of Dsf file);
* Can be also used for MONTHLY CRSP data;
%Macro PreProcRawCRSP(dname1=,dname2=,DataCutYr=,outdname=);
* DSEnames;
data CRSPnames (drop=shrcd); set &dname1 
	(keep=permno namedt nameendt shrcd exchcd siccd ticker);   		* ADD more variables, depending on applications;
	where (shrcd = 10 or shrcd = 11) and (exchcd = 1 or exchcd = 2 or exchcd = 3) and nmiss(permno) = 0; *Common shares and NYSE/AMEX/NASDAQ;									
	if not(missing(nameendt)) and year(nameendt) < &DataCutYr then delete;		
	format namedt nameendt date9.; run; 		
proc sort data=CRSPnames; by permno namedt nameendt; run;
* DSF;																* ADD more variables, depending on applications;
data CRSPRaw; set &dname2 (keep=permno date prc vol ret shrout where=(year(date)>=&DataCutYr)); format date date9.; run;
proc sort data=CRSPRaw; by permno date; run;
proc sql; create table &outdname (drop=Npermno namedt nameendt) as select *      
	from CRSPRaw as a, CRSPnames (rename=(permno=Npermno)) as b
	where a.permno = b.Npermno and b.namedt <= a.date <= b.nameendt; quit;  *Assuming no missing of namedt and nameendt (checked OK);
* Sanity check for DSEnames;
data Dsenames_check; set &dname1; if missing(namedt) or missing(nameendt); run;
proc datasets lib=work; delete CRSPRaw CRSPnames Dsenames_check; quit;
%Mend PreProcRawCRSP;
* Macro to selectively run PreProcRawCRSP for laptop and server;
%Macro PreProcRawCRSPd_Sel(OSid=,DataCutYr1=);
%if &OSid = 1 %then %do;
	data Pre_CRSPd; set crspa.dsf (where=(year(date)>=2000)); run; %end; *PARTIAL data only for laptop testing purpose;
%if &OSid > 1 %then %do;
	* Pre-Processing of Daily raw CRSP data;
	%PreProcRawCRSP(dname1=CRSPQ.Dsenames,dname2=CRSPQ.dsf,DataCutYr=&DataCutYr1,outdname=Pre_CRSPd); %end;
%Mend PreProcRawCRSPd_Sel;




*********************************; * After computing time-varying betas;
* - CRSP Daily Data Processing  *; * will be merged into the CRSP MONTHLY data for asset pricing tests;
*********************************; 
** - Analyzing the NYSE/AMEX/NASDAQ common shares; 
**   All other screenings will be AUTOMATICALLY applied while combining with CRSP Monthly;
** - Be careful when using CRSP daily from laptop since it only has PARTIAL data;
** - Pre-Processing of Daily raw CRSP data (Selective execution based on OS);
** - No screenings based on Trading days in a given month YET. Might need to do so later;
%PreProcRawCRSPd_Sel(OSid=&ServerID,DataCutYr1=&CRSPCutYr); 
data CRSPd; retain permno date dyear dmonth dday; set Pre_CRSPd;
	where abs(prc)>0 and shrout>0; *and vol>0;
	/* if siccd >= 6000 and siccd <= 6800 then delete; 	* drop financial firms; 	*/
	prc = abs(prc); 	  					*CRSP price;
	*vol = vol*100; 		  				*share volume NOT in hundreds (for CRSP Daily);
	shrout = shrout*1000; 					*shrout in thousands;	
	MkCap = prc*shrout;   					*market capitalization; 
	Dvol = vol*prc;							*Dollar volume;
	* if hexcd in (1,2,3);					*Keeping NYSE/AMEX/NASDAQ if serverid=1;
	if exchcd in (1,2,3);					*Keeping NYSE/AMEX/NASDAQ;
	dyear = year(date); dmonth = month(date); dday = day(date);
	keep permno date prc vol Ret shrout MkCap Dvol dyear dmonth dday;
	label MkCap = "Market Capitalization by Daily CRSP"; 
	label DVol = "Dollar Volume"; label date = "date"; run;
/* data CRSPd; set CRSPd; if dyear>=2001 and dyear<=2003; run; */

** Screening based on the number of daily Rets;
*  For each firm and a given year, at least 100 daily returns should be available (non-consecutive);
%let RetdCntCut = 100; *For shorter than 3Yrs, use 100 days;
proc sort data=CRSPd; by permno dyear dmonth dday; run;
proc univariate data=CRSPd noprint; by permno dyear;
	var Ret; output out=CRSPd_RetCnt (where=(RetdCnt>&RetdCntCut)) N=RetdCnt; run;
data CRSPd_RetCnt; set CRSPd_RetCnt; 
	label RetdCnt = "Number of daily returns available for a given year"; run;
proc sql; create table CRSPd_Scr (drop=Npermno Ndyear) as select *
	from CRSPd as a, CRSPd_RetCnt (rename=(permno=Npermno dyear=Ndyear)) as b
	where a.permno = b.Npermno and a.dyear = b.Ndyear; quit;
proc sort data=CRSPd_Scr; by permno date; run;
proc datasets lib=work; delete CRSPd; quit;


** Combining with Daily FF3 Factors;
data FF3d; set FF.Factors_Daily; format date date9.; if year(date)>=&CRSPCutYr; run;
proc sort data=FF3d; by date; run;
proc sql; create table CRSPd_FF3d (drop=Ndate) as select *
	from CRSPd_Scr (drop=dyear dmonth dday) as a, FF3d (rename=(date=Ndate)) as b
	where year(a.date) = year(b.Ndate) and month(a.date) = month(b.Ndate) 
		and day(a.date) = day(b.Ndate); run;
proc sort data=CRSPd_FF3d; by permno date; run;
data CRSPd_FF3d; retain permno date dyear dmonth dday EvenOddMon; set CRSPd_FF3d;
	dyear = year(date); dmonth = month(date); dday = day(date); EvenOddMon = mod(dmonth,2);
	label EvenOddMon = "Even- and Odd-month Indicator (Even=0 and Odd=1)"; run;
proc datasets lib=work; delete CRSPd_RetCnt CRSPd_Scr FF3d Pre_CRSPd; quit;


** Lead and Lag the factors for Dimson corrections;
*  1. Not considering the begining and end of a month;
proc sort data=CRSPd_FF3d; by permno date; run;
data CRSPd_FF3d; set CRSPd_FF3d; by permno;
	MKTRF_1 = lag(MKTRF); SMB_1 = lag(SMB); HML_1 = lag(HML); 
	if first.permno then do; MKTRF_1 = .; SMB_1 = .; HML_1 = .; end; run;
proc sort data=CRSPd_FF3d; by permno descending date; run;
data CRSPd_FF3d; set CRSPd_FF3d; by permno;
	MKTRF1 = lag(MKTRF); SMB1 = lag(SMB); HML1 = lag(HML); 
	if first.permno then do; MKTRF1 = .; SMB1 = .; HML1 = .; end; run;
proc sort data=CRSPd_FF3d; by permno date; run;

*  2. Dropping the begining and end of a month;
***   To avoid the mixing between even- and odd-month factor realizations;
*     Begining and end of a year are automatically considered;
proc sort data=CRSPd_FF3d; by permno dyear dmonth dday; run;
data CRSPd_FF3d; set CRSPd_FF3d; by permno dyear dmonth; BEMon_Ind = 0;
	if first.dmonth or last.dmonth then BEMon_Ind = 1; run;
data CRSPd_FF3d_Check; retain permno dyear dmonth BEMon_Ind; set CRSPd_FF3d; run;
data CRSPd_FF3d (drop=BEMon_Ind); set CRSPd_FF3d; if BEMon_Ind = 1 then delete; run;
/* data IVBProj.CRSPd_FF3d; set CRSPd_FF3d; run;  */



******************************; * To avoid the memory issues;
** - Splitting the Permnos   *; * For arbitrary size of each dataset; 
******************************; * Choose PermnoStep depending on the size of each dataset;
%Macro GetPermnosSplit(Dname=,PermnoStep=);
%if &ServerID > 1 %then %do;
	data IVBProj.CRSPd_FF3d; set CRSPd_FF3d; run;
%end;
proc sort data=&Dname out=&Dname._tmp; by permno date; run;
data &Dname._tmp; set &Dname._tmp; by permno; 
	if last.permno; call symputx('EndPermno',permno); run;
%let isT = %sysfunc(ceil(%sysevalf((&EndPermno-10000)/&PermnoStep)));
%put "Largest Permno is &EndPermno";
%put  "Total Number of Split Datasets is &isT";
%global TotDNum;	%let TotDNum = &isT;

*** - Loop Starts;
%do is=1 %to &isT;
	*Initial file;
	%if &is = 1 %then %do;
		data &Dname._1; set &Dname; if permno <= 10000+&PermnoStep; run;	
	%end;
	*Iterations for files;
	%if &is > 1 and &is < &isT %then %do;
		data &Dname._&is; set &Dname; 
			if 10000+&PermnoStep*(&is-1) < permno <= 10000+&PermnoStep*&is; run;	
	%end;
	*Last file;
	%if &is = &isT %then %do;
		data &Dname._&is; set &Dname; if 10000+&PermnoStep*(&is-1) < permno; run;	
	%end;
%end;
proc datasets lib=work; delete &Dname._tmp; quit;
%MEnd GetPermnosSplit;
%GetPermnosSplit(Dname=CRSPd_FF3d,PermnoStep=1000);



****************************************************; *Using daily returns and daily factors;
** - Estimation of Betas (with Dimson correction)  *; *Rolling-beta estimation (based on the past returns);
****************************************************; *Within each window (with RBLeng Yrs), Even- and Odd-months are split;
** - Macro for time-varying IVBetas with Dimson corrections;
*    Lead and Lag of regressors are controlled simultaneously as in Dimson (1979);
** - Daily Standard FM-Betas with Dimson corrections are also estimated;
** - Length of rolling window is determined by &RBLeng (in Yrs);
%Macro GetTVIVBeta_Dimson(Dname=,Outdname=,APM=,RBLeng=);
/* %let Dname=CRSPd_FF3d;		%let Outdname=CRSPd_RBetas_FF3;		
%let APM=FF3;				%let RBLeng = 3; *in Yr;		*/
** FF3;
%if %sysfunc(lowcase(&APM)) = %sysfunc(lowcase(FF3)) %then %do;
	%let Reg = MKTRF SMB HML; %let Reg_1 = MKTRF_1 SMB_1 HML_1; %let Reg1 = MKTRF1 SMB1 HML1; %end;
** CAPM;
%if %sysfunc(lowcase(&APM)) = %sysfunc(lowcase(CAP)) %then %do;
	%let Reg = MKTRF; %let Reg_1 = MKTRF_1; %let Reg1 = MKTRF1; %end;

*****************************;
** - Rolling-beta Windows   *;
*****************************;
* Original data;
proc sort data=&Dname; by permno dyear dmonth dday; run;
data &Dname._Mon (drop=dyear dmonth dday); set &Dname (keep=permno date dyear dmonth dday); 
	by permno dyear dmonth; if last.dmonth; run;
* Replicated data;
data &Dname._Rep; set &Dname; DepVarrf = Ret - rf; *Daily excess return; 
	keep permno date EvenOddMon DepVarrf MKTRF SMB HML MKTRF_1 SMB_1 HML_1 MKTRF1 SMB1 HML1; run;
* Combining the above two;
proc sql; create table &Dname._RB_tmp (drop=Npermno) as select *
	from &Dname._Mon as a,
	     &Dname._Rep (rename=(permno=Npermno date=Ndate EvenOddMon=NEvenOddMon)) as b
	where a.permno=b.Npermno and 0<intck('Month',b.Ndate,a.date)<=&RBLeng*12; quit;
proc sort data=&Dname._RB_tmp; by permno date Ndate; run;

** Dropping too short time-series;
proc univariate data=&Dname._RB_tmp noprint; by permno date;	  *roughly &RBLeng Yrs;
	var Ndate; output out=&Dname._RB_tmp_Cnt (where=(Ndate_Cnt>=&RBLeng*100)) N=Ndate_Cnt; run;
																  			  *Non-missing values;
proc sql; create table &Dname._RB_tmp_Scr (drop=Npermno Mdate Ndate_Cnt) as select *
	from &Dname._RB_tmp as a, 
		 &Dname._RB_tmp_Cnt (rename=(permno=Npermno date=Mdate)) as b
	where a.permno = b.Npermno and year(a.date) = year(b.Mdate) 
		  and month(a.date) = month(b.Mdate); quit;
proc sort data=&Dname._RB_tmp_Scr; by permno date Ndate; run;
proc datasets lib=work; delete &Dname._Mon &Dname._Rep &Dname._RB_tmp &Dname._RB_tmp_Cnt; quit;


*******************************;
** - Running TS-regressions   *;
*******************************;
data &Dname._RB_tmp_Scr; retain permno date dyear dmonth Ndate; 
	set &Dname._RB_tmp_Scr; dmonth = month(date); dyear = year(date); run;
** 1. Running FM TS-regressions;
proc sort data=&Dname._RB_tmp_Scr; by permno dyear dmonth Ndate; run;
proc reg data=&Dname._RB_tmp_Scr outest=&Dname._RBFM noprint tableout;
	by permno dyear dmonth; 
	Reg: model DepVarrf = &Reg &Reg_1/ adjrsq b; run;
	/* Reg: model DepVarrf = &Reg &Reg1 &Reg_1/ adjrsq b; run;  */
data &Dname._RBFM_PE (drop=_type_ _model_); set &Dname._RBFM; 
	if _type_='PARMS'; drop _depvar_ _rmse_ DepVarrf _in_ _p_ _edf_ _rsq_ Intercept _adjrsq_; run;
* Screening based on T-stats of lags;
data &Dname._RBFM_T (drop=_type_ _model_ &Reg); set &Dname._RBFM; 
	if _type_='T'; drop _depvar_ _rmse_ DepVarrf _in_ _p_ _edf_ _rsq_ Intercept _adjrsq_; run;
data &Dname._RBFM_T (drop=&Reg_1); set &Dname._RBFM_T;
	* Lags;
	MKTRF_T_1 = 1;	SMB_T_1 = 1;	HML_T_1 = 1;
	if (MKTRF_1) < &DimsonT_Th then MKTRF_T_1 = 0;	
	if (SMB_1) < &DimsonT_Th then SMB_T_1 = 0;
	if (HML_1) < &DimsonT_Th then HML_T_1 = 0;		
	/* Leads;
	MKTRF_T1 = 1;	SMB_T1 = 1;	HML_T1 = 1;
	if abs(MKTRF1) < &DimsonT_Th then MKTRF_T1 = 0;	
	if abs(SMB1) < &DimsonT_Th then SMB_T1 = 0;
	if abs(HML1) < &DimsonT_Th then HML_T1 = 0;		
	*/
run;
proc sql; create table &Dname._RBFM_PET (drop=Npermno Ndyear Ndmonth) as select *
	from &Dname._RBFM_PE as a, 
		 &Dname._RBFM_T (rename=(permno=Npermno dyear=Ndyear dmonth=Ndmonth)) as b
	where a.permno = b.Npermno and a.dyear = b.Ndyear 
	and a.dmonth = b.Ndmonth; quit;
data &Dname._RBFM_PET (drop=MKTRF_T_1 SMB_T_1 HML_T_1); 
	set &Dname._RBFM_PET;
	* Lags;
	MKTRF_1 = MKTRF_1*MKTRF_T_1;		SMB_1 = SMB_1*SMB_T_1;
	HML_1 = HML_1*HML_T_1;				
	/* Leads;
	MKTRF1 = MKTRF1*MKTRF_T1;			SMB1 = SMB1*SMB_T1;
	HML1 = HML1*HML_T1;					*/
run;
* Selected Dimson correction;
data &Dname._RBFM (drop=dyear dmonth); retain permno date; set &Dname._RBFM_PET; format date date9.;
	date = mdy(dmonth,01,dyear); date = intnx('Month',date,0,'End'); run;
proc datasets lib=work; delete &Dname._RBFM_PE &Dname._RBFM_T &Dname._RBFM_PET; quit;


** 2. Running IVTS-regressions;
proc sort data=&Dname._RB_tmp_Scr out=&Dname._RBIV_tmp_Scr; 
	by permno dyear dmonth NEvenOddMon Ndate; run;
proc reg data=&Dname._RBIV_tmp_Scr outest=&Dname._RBIV noprint tableout;
	by permno dyear dmonth NEvenOddMon;
	Reg: model DepVarrf = &Reg &Reg_1 / adjrsq b; run;
	/* Reg: model DepVarrf = &Reg &Reg1 &Reg_1 / adjrsq b; run;  */
data &Dname._RBIV_PE (drop=_type_ _model_); set &Dname._RBIV;
	if _type_='PARMS'; drop _depvar_ _rmse_ DepVarrf _in_ _p_ _edf_ _rsq_ Intercept _adjrsq_; run;
* Screening based on T-stats of lags;
data &Dname._RBIV_T (drop=_type_ _model_  &Reg); set &Dname._RBIV;
	if _type_='T'; drop _depvar_ _rmse_ DepVarrf _in_ _p_ _edf_ _rsq_ Intercept _adjrsq_; run;
data &Dname._RBIV_T (drop=&Reg_1); set &Dname._RBIV_T;
	* Lags;
	MKTRF_T_1 = 1;	SMB_T_1 = 1;	HML_T_1 = 1;	
	if (MKTRF_1) < &DimsonT_Th then MKTRF_T_1 = 0;	
	if (SMB_1) < &DimsonT_Th then SMB_T_1 = 0;
	if (HML_1) < &DimsonT_Th then HML_T_1 = 0;		
	/* Leads;
	MKTRF_T1 = 1;	SMBT_1 = 1;	HML_T1 = 1;
	if abs(MKTRF1) < &DimsonT_Th then MKTRF_T1 = 0;	
	if abs(SMB1) < &DimsonT_Th then SMB_T1 = 0;
	if abs(HML1) < &DimsonT_Th then HML_T1 = 0;		*/
run;
proc sql; create table &Dname._RBIV_PET (drop=Npermno Ndyear Ndmonth MNEvenOddMon) as select *
	from &Dname._RBIV_PE as a,
		 &Dname._RBIV_T (rename=(permno=Npermno dyear=Ndyear dmonth=Ndmonth NEvenOddMon=MNEvenOddMon)) as b
	where a.permno = b.Npermno and a.dyear = b.Ndyear 
	and a.dmonth = b.Ndmonth and a.NEvenOddMon = b.MNEvenOddMon; quit;
data &Dname._RBIV_PET (drop=MKTRF_T_1 SMB_T_1 HML_T_1); 
	set &Dname._RBIV_PET;
	* Lags;
	MKTRF_1 = MKTRF_1*MKTRF_T_1;		SMB_1 = SMB_1*SMB_T_1;
	HML_1 = HML_1*HML_T_1; 			
	/* Leads;
	MKTRF1 = MKTRF1*MKTRF_T1;			SMB1 = SMB1*SMB_T1;
	HML1 = HML1*HML_T1; 			*/
run;
* Selected Dimson correction;
data &Dname._RBIV (drop=dyear dmonth); retain permno date; set &Dname._RBIV_PET; format date date9.;
	date = mdy(dmonth,01,dyear); date = intnx('Month',date,0,'End'); run;
proc datasets lib=work; delete &Dname._RB_tmp_Scr &Dname._RBIV_tmp_Scr; quit;
proc datasets lib=work; delete &Dname._RBIV_PE &Dname._RBIV_T &Dname._RBIV_PET; quit;



*********************; 
** PostProcessings  *;
*********************;
* FF3;
%if %sysfunc(lowcase(&APM)) = %sysfunc(lowcase(FF3)) %then %do;
	* 1. Dimson correction;
	data &Dname._RBIV (keep=permno date NEvenOddMon MKTRF SMB HML EvenOddMon); 
		retain permno date EvenOddMon NEvenOddMon; 
		set &Dname._RBIV; EvenOddMon = mod(month(date),2);
		/*
		MKTRF = MKTRF + MKTRF_1 + MKTRF1;	SMB = SMB + SMB_1 + SMB1;			
		HML = HML + HML_1 + HML1;		
		*/
		MKTRF = MKTRF + MKTRF_1;			SMB = SMB + SMB_1;			
		HML = HML + HML_1;					run; 
	data &Dname._RBFM (keep=permno date MKTRF_FM SMB_FM HML_FM); 
		set &Dname._RBFM; 
		/*
		MKTRF_FM = MKTRF + MKTRF_1 + MKTRF1;	SMB_FM = SMB + SMB_1 + SMB1;			
		HML_FM = HML + HML_1 + HML1;						
		*/
		MKTRF_FM = MKTRF + MKTRF_1;			SMB_FM = SMB + SMB_1;			
		HML_FM = HML + HML_1;				run;			
	* 2. Combining all;
	data &Dname._RBIV_tmp1 (drop=EvenOddMon NEvenOddMon); 
		set &Dname._RBIV; if EvenOddMon = NEvenOddMon; run;
	data &Dname._RBIV_tmp2 (drop=EvenOddMon NEvenOddMon); 
		set &Dname._RBIV; if EvenOddMon = abs(1-NEvenOddMon); 
		rename MKTRF = MKTRF_IV SMB = SMB_IV HML = HML_IV; run;
	proc sql; 
		create table &Dname._RBIV_Tab (drop=Npermno Ndate) as select *
		from &Dname._RBIV_tmp1 as a,		
			 &Dname._RBIV_tmp2 (rename=(permno=Npermno date=Ndate)) as b
		where a.permno = b.Npermno and year(a.date) = year(b.Ndate) and
			  month(a.date) = month(b.Ndate); quit;
	proc sql; 
		create table &Outdname (drop=Npermno Ndate) as select *
		from &Dname._RBIV_Tab as a,
			 &Dname._RBFM (rename=(permno=Npermno date=Ndate)) as b
		where a.permno = b.Npermno and year(a.date) = year(b.Ndate) and
			  month(a.date) = month(b.Ndate); quit;
	proc sort data=&Outdname; by permno date; run;
%end;
* CAPM;
%if %sysfunc(lowcase(&APM)) = %sysfunc(lowcase(CAP)) %then %do;
	* 1. Dimson correction;
	data &Dname._RBIV (keep=permno date NEvenOddMon MKTRF EvenOddMon); 
		retain permno date EvenOddMon NEvenOddMon; 
		set &Dname._RBIV; EvenOddMon = mod(month(date),2);
		MKTRF = MKTRF + MKTRF_1; * + MKTRF1;		run; 
	data &Dname._RBFM (keep=permno date MKTRF_FM); 
		set &Dname._RBFM; 
		MKTRF_FM = MKTRF + MKTRF_1; * + MKTRF1;	run;			
	* 2. Combining all;
	data &Dname._RBIV_tmp1 (drop=EvenOddMon NEvenOddMon); 
		set &Dname._RBIV; if EvenOddMon = NEvenOddMon; run;
	data &Dname._RBIV_tmp2 (drop=EvenOddMon NEvenOddMon); 
		set &Dname._RBIV; if EvenOddMon = abs(1-NEvenOddMon); 
		rename MKTRF = MKTRF_IV; run;
	proc sql; 
		create table &Dname._RBIV_Tab (drop=Npermno Ndate) as select *
		from &Dname._RBIV_tmp1 as a,		
			 &Dname._RBIV_tmp2 (rename=(permno=Npermno date=Ndate)) as b
		where a.permno = b.Npermno and year(a.date) = year(b.Ndate) and
			  month(a.date) = month(b.Ndate); quit;
	proc sql; 
		create table &Outdname (drop=Npermno Ndate) as select *
		from &Dname._RBIV_Tab as a,
			 &Dname._RBFM (rename=(permno=Npermno date=Ndate)) as b
		where a.permno = b.Npermno and year(a.date) = year(b.Ndate) and
			  month(a.date) = month(b.Ndate); quit;
	proc sort data=&Outdname; by permno date; run;			
%end;
%if &ServerID>1 %then %do; data IVBProj.&OutDname; set &OutDname; run; %end;
proc datasets lib=work; delete &Dname._RBIV &Dname._RBFM &Dname._RBIV_tmp1 &Dname._RBIV_tmp2 &Dname._RBIV_Tab; quit;
%Mend GetTVIVBeta_Dimson;

* Macro to run the iterations for GetTVIVBeta_Dimson;
%Macro GetIters_TVIVBeta_Dimson(DnamePre=,OutDnamePre=,APMIter=);
/* %let DnamePre=CRSPd_FF3d;	%let OutDnamePre=CRSPd_RBetas_FF3;   */
%do cntf=1 %to &TotDNum;
	%put "Dealing with &cntf Dataset out of &TotDNum";
	%GetTVIVBeta_Dimson(Dname=&DnamePre._&cntf,OutDname=&OutDnamePre._&cntf,APM=&APMIter,RBLeng=&RBLeng1);
	*%GetTVIVBeta_Dimson(Dname=CRSPd_FF3d_1,OutDname=CRSPd_RBetas_FF3_1,APM=FF3,RBLeng=&RBLeng1);
%end;
%MEnd GetIters_TVIVBeta_Dimson;
* Invoking the Macros;	* w/ Dimson corrections;
%let RBLeng1 = 3; *in Yrs;
* ServerID=1;
/*%GetIters_TVIVBeta_Dimson(DnamePre=CRSPd_FF3d,OutDnamePre=IVBProj.CRSPd_RBetas_FF3,APMIter=FF3); *FF3;
%GetIters_TVIVBeta_Dimson(DnamePre=CRSPd_FF3d,OutDnamePre=IVBProj.CRSPd_RBetas_Cap,APMIter=CAP); *CAPM;*/
* ServerID>1;
%GetIters_TVIVBeta_Dimson(DnamePre=CRSPd_FF3d,OutDnamePre=CRSPd_RBetas_FF3,APMIter=FF3); *FF3;
%GetIters_TVIVBeta_Dimson(DnamePre=CRSPd_FF3d,OutDnamePre=CRSPd_RBetas_Cap,APMIter=CAP); *CAPM;



** - Macro to combine all above;
**   Can be run in Desktop;
%Macro CombineRBetas(DnamePre=,OutDname=);
%do it=1 %to &TotDNum;
	%if &it=1 %then %do;
		data &OutDname; set IVBProj.&DnamePre._&it; run; %end;
	%if &it>1 %then %do;
		proc append base=&OutDname data=IVBProj.&DnamePre._&it; run; %end;
%end;
proc sort data=&OutDname; by permno date; run;
%if &ServerID>1 %then %do; 
	data IVBProj.&OutDname; set &OutDname; run; %end;
%MEnd CombineRBetas;
%CombineRBetas(DnamePre=CRSPd_RBetas_FF3,OutDname=CRSPd_RBetas_FF3);
%CombineRBetas(DnamePre=CRSPd_RBetas_Cap,OutDname=CRSPd_RBetas_Cap);



* The following can be created as a single document;

%let RBLeng1=3;
%let RetdCntCut = 100;


* Loading monthly CRSP data;
data CRSP_Mon_Tab; set IVBProj.CRSP_CCM_Comp_Mon_Tab; 
	keep permno date ret rf Sizem Mom6 B2M logB2M; run;

proc sort data=CRSP_Mon_Tab; by date permno; run;
data CRSP_Mon_Tab_Ch; set CRSP_Mon_Tab; by date; if first.date; run;

************************************************;
** - Combining IVBetas and Monthly CRSP data   *;
************************************************;
* CAPM;
proc sql; 
	create table CRSPm_FinTab_RBd_CAPM (drop=Npermno Ndate) as select *
	from CRSP_Mon_Tab as a,
		 IVBProj.CRSPd_RBetas_Cap (rename=(permno=Npermno date=Ndate)) as b
	where a.permno = b.Npermno and year(a.date) = year(b.Ndate) and
		  month(a.date) = month(b.Ndate); quit;
proc sort data=CRSPm_FinTab_RBd_CAPM; by permno date; run;
data IVBProj.CRSPm_FinTab_RBd_CAPM_&RBLeng1._&RetdCntCut; set CRSPm_FinTab_RBd_CAPM; run;

* FF3;
proc sql; 
	create table CRSPm_FinTab_RBd_FF3 (drop=Npermno Ndate) as select *
	from CRSP_Mon_Tab as a,
		 IVBProj.CRSPd_RBetas_FF3 (rename=(permno=Npermno date=Ndate)) as b
	where a.permno = b.Npermno and year(a.date) = year(b.Ndate) and
		  month(a.date) = month(b.Ndate); quit;
proc sort data=CRSPm_FinTab_RBd_FF3; by permno date; run;
data IVBProj.CRSPm_FinTab_RBd_FF3N_&RBLeng1._&RetdCntCut; set CRSPm_FinTab_RBd_FF3; run;





** Estimate risk premium using IV-approach: 2 stage least square; 
** Fama French three factor model;

proc sort data=CRSPm_FinTab_RBd_FF3; by date permno; run;


proc model data=CRSPm_FinTab_RBd_FF3 NOPRINT;
   exogenous MKTRF_IV
		   SMB_IV HML_IV
           Sizem LogB2M Mom6;
   by date;
   parms a0-a4 b0-b4 c0-c4 gamma0-gamma6;
   SMB = a0+a1*SMB_IV+a2*Sizem+a3*LogB2M+a4*Mom6;
   MKTRF = b0+b1*MKTRF_IV+b2*Sizem+b3*LogB2M+b4*Mom6;
   HML = c0+c1*HML_IV+c2*Sizem+c3*LogB2M+c4*Mom6;
   ret =gamma0+gamma1*MKTRF+gamma2*SMB+gamma3*HML+gamma4*Sizem+gamma5*LogB2M+gamma6*Mom6;
   fit  ret SMB MKTRF HML / 2sls outest=IV_chars_in;
   instruments MKTRF_IV
		   SMB_IV HML_IV
           Sizem LogB2M Mom6;;
run;


/* proc model data=CRSPm_FinTab_RBd_FF3 NOPRINT;
   exogenous MKTRF_IV
		   SMB_IV HML_IV;
   by date;
   parms a0-a1 b0-b1 c0-c1 gamma0-gamma6;
   SMB = a0+a1*SMB_IV;
   MKTRF = b0+b1*MKTRF_IV;
   HML = c0+c1*HML_IV;
   ret =gamma0+gamma1*MKTRF+gamma2*SMB+gamma3*HML+gamma4*Sizem+gamma5*LogB2M+gamma6*Mom6;
   fit  ret  SMB MKTRF HML / 2sls outest=IV_chars_in;
   instruments MKTRF_IV
		   SMB_IV HML_IV
           Sizem LogB2M Mom6;;
run; */




proc sort data= IV_chars_in; by date; run;

proc means data=IV_chars_in n mean stderr t prt;
var gamma0 gamma1 gamma2 gamma3 gamma4 gamma5 gamma6;
output out=Risk_premium (drop=_TYPE_ _FREQ_);
run;