#] #] ********************* #] "$d_Qndfs""fit_linearRegress - notes.txt" www.BillHowell.ca 05Jan2020 initial # view in text editor, using constant-width font (eg courier), tabWidth = 3 see also "SP500 Fibonacci mirror comments.txt" #48************************************************48 #24************************24 # Table of Contents, generate with : # $ grep "^#]" "$d_Qndfs""fit_linearRegress - notes.txt" | sed "s/^#\]/ /" # ********************* "$d_Qndfs""fit_linearRegress - notes.txt" 19Jul2022 minimum-manual auto-file adaptations : TVdata -> linear regression -> gnuplot 18Jul2022 tabTable_readFrm_path changed to pinn_read_tabTable 07Jan2021 "$d_Qndfs""fit_linearRegress.ndf" - MLR isn't working 05Jan2021 Coeff := I innerproduct (innerproduct (transpose xColsRaw) yColRaw) 05Jan2020 try innerproduct 04Jan2021 To-be-quarantined code #24************************24 # Setup, ToDos, #08********08 #] ??Jul2022 #08********08 #] ??Jul2022 #08********08 #] ??Jul2022 #08********08 #] ??Jul2022 #08********08 #] ??Jul2022 #08********08 #] ??Jul2022 #08********08 #] 19Jul2022 link d_Qndfs 'economics, markets/stock index [semi-log, detrend] data processing.ndf' example of use, and of combining [bash, QNial] operators #08********08 #] 19Jul2022 minimum-manual auto-file adaptations : TVdata -> linear regression -> gnuplot #] SPX, HSSSHSC, DAX40, UKX100, NI225, INDIA50 see "$d_bin"'stock index semi-log data processing.sh' +-----+ olde code # pth_typ_exists (link d_webRawe 'economics, markets/SP500/historical DAX Jan1970-Jul2022 [yrFrac, openPrice].txt') "pinn # pth_typ_exists (link d_webRawe 'economics, markets/SP500/historical DAX Jan1970-Jul2022 [yrFrac, openPrice] semiLog regression output.txt') "povr # pinn_read_tabtable (link d_webRawe 'economics, markets/SP500/historical DAX Jan1970-Jul2022 [yrFrac, openPrice].txt') # 18Jul2022 olde code : titlesRows := tableIn|[,0] ; titlesCols := tableIn|[0,] ; % tableRaw := EACH tonumber tableIn ; % revamp xColIndxL yColIndx because of removal of rowColTitles ; % removed for 0-indexing : xColIndx yColIndx := xColIndx yColIndx EACHLEFT - 1 ; #08********08 #] 18Jul2022 tabTable_readFrm_path changed to pinn_read_tabTable # 18Jul2022 modified to incorporate [xOptrPhr, yOptrPhr, 0-indexing of cols] # changed path_backupDatedTo_dir to pinn_backupDatedTo_zArchive changed argOrders : IF (NAND (EACH pth_typ_exists (pinn "pold) (pout "pnew))) many other updates, problematic 08********08 #] 07Jan2021 "$d_Qndfs""fit_linearRegress.ndf" - MLR isn't working 08********08 #] 05Jan2021 Coeff := I innerproduct (innerproduct (transpose xColsRaw) yColRaw) +-----+ # (DfReg DFError DFTotal)(SSReg SSError SSTotal) (MSReg MSError " )(FRatio " " ) hitch [StErrorEst Rsq] # VarNos Coeff SECoeff T hitch ((DfReg DFError DFTotal)(SSReg SSError SSTotal) (MSReg MSError ".. )(FRatio ".. "..) hitch [StErrorEst Rsq] ) mix (EACH phrase ('StndErrOf estimate' 'CorrCoeff^2')) (third A) VarNos gets 0 hitch front first A ; # Change : +.....+ Coeff gets I innerproduct matprod[flip first,second] X Y; +.....+ To : +.....+ Coeff := I innerproduct (innerproduct (transpose xColsRaw) yColRaw) ; +.....+ +-----+ clean up some duplicate ndf files $ diff "$d_Qndfs""matrix operations - symbolic & real-valued.ndf" "$d_Qndfs""matrix operations - symbolic & real-valued 201009.ndf" --suppress-common-lines >> almost all differences are due to adding lines like : IF flag_debug THEN write 'loading txt_rows_to_array' ; ENDIF ; 08********08 #] 05Jan2020 try innerproduct replaces matrix_multiply in invreg IS inverse matrix_multiply[flip,pass] -->[nextv] xcolsraw 2 0 3 6 5 4 4 3 1 1 6 5 innerproduct [pack,pass] xcolsraw -->[nextv] innerproduct [pack,pass] xcolsraw ?first arg not simple in innerproduct -->[nextv] innerproduct [flip,pass] xcolsraw ?first arg not simple in innerproduct -->[nextv] [transpose,pass] xcolsraw +-----------+---+ |2 3 5 4 1 6|2 0| |0 6 4 3 1 5|3 6| | |5 4| | |4 3| | |1 1| | |6 5| +-----------+---+ -->[nextv] innerproduct [transpose,pass] xcolsraw 91. 81. 81. 87. -->[nextv] xcolsrawInverse := inverse innerproduct [transpose,pass] xcolsraw 0.0641593 -0.0597345 -0.0597345 0.0671091 -->[nextv] ccc := innerproduct xcolsraw xcolsrawInverse 0.128319 -0.119469 -0.165929 0.223451 0.0818584 -0.030236 0.0774336 -0.0376106 0.00442478 0.00737463 0.0862832 -0.0228614 -->[nextv] innerproduct xcolsrawInverse xcolsraw ?conform error in innerproduct -->[nextv] innerproduct xcolsrawInverse (transpose xcolsraw) 0.128319 -0.165929 0.0818584 0.0774336 0.00442478 0.0862832 -0.119469 0.223451 -0.030236 -0.0376106 0.00737463 -0.0228614 >> I'm not getting the identity matrix!!?? -->[nextv] fff := innerproduct [transpose,pass] xcolsraw 91. 81. 81. 87. -->[nextv] innerproduct xcolsraw fff 182. 162. 759. 765. 779. 753. 607. 585. 172. 168. 951. 921. -->[nextv] innerproduct fff xcolsrawInverse 1. 0. 8.88178e-16 1. >> BINGO!! -->[nextv] innerproduct xcolsrawInverse fff 1. 8.88178e-16 0. 1. >> bingo again 08********08 #] 04Jan2021 To-be-quarantined code # From NN2 sqr IS prod[pass,pass] - already in setup.ndf as square (changed in this file) matprod IS INNER[sum,times] EACH mix - replace with matrix_multiply from matrix_multiply.ndf (checks data, signals errors) does matrix_multiply fail due to +* rather than sum times? IF flag_debug THEN write 'loading multLinReg_cycle' ; ENDIF ; # multLinReg_cycle IS OP P A - multLinReg_cycle IS OP P A { M gets A@P;A@P gets 1; A|[first P,] gets A|[first P,] div M; I gets tell first shape A except first P; C gets I EACHLEFT pair second P; M gets A#C;A#C gets 0; A|[I,] gets A|[I,] minus (M OUTER times A|[first P,]) } # inverse of a matrix is a standard QNial operator inverse IS OP A { FOR I WITH tell first shape A DO A gets I pair I multLinReg_cycle A ENDFOR } # From NN2 sqr IS prod[pass,pass] - already in setup.ndf as square (changed in this file) matprod IS INNER[sum,times] EACH mix - replace with matrix_multiply from matrix_multiply.ndf (checks data, signals errors) does matrix_multiply fail due to +* rather than sum times? # From NN20 am IS div[sum,tally] - replace by average as per setup.ndf deviates IS minus[pass,average] sumsq IS sum square deviates # Independent variables xdata IS 1 EACHBOTH hitch pack choose[-1 sum front first,second] # Dependent variable ydata IS link choose[-1 sum last first,second] # Inverse of independent variables matrix invreg IS inverse innerproduct [transpose,pass] # Coefficient table multLinReg_coeff IS OP A { mix pack ("VarNos "Coeff "S.E. "T-Value EACHBOTH hitch first A) } IF flag_debug THEN write 'loading multLinReg_analOfVar' ; ENDIF ; # multLinReg_analOfVar IS OP A - analysis of variance table multLinReg_analOfVar IS OP A { mix pack ("Source "Treatments "Error "Total hitch ("D.F "S.S. "M.S. "F-Ratio EACHBOTH hitch second A) ) } IF flag_debug THEN write 'loading multLinReg_statsTbl3' ; ENDIF ; # multLinReg_statsTbl3 IS IS OP A - table of [std error of estimate, R squared] multLinReg_statsTbl3 IS OP A { mix (EACH phrase ('StndErrOf estimate' 'CorrCoeff^2')) (third A) } % ; % calculate multLinReg_[coeff, analOfVar, statsTbl3, results] ; multLinReg_coeff AnalOfVar multLinReg_statsTbl3 fout EACHRIGHT writefile 'multLinReg_stats' (picture transpose mix (link "VarNos VarNos) (link "Coeff Coeff) (link "S.E. S.E.) (link "T-Value Tstat) ) '' ; # enddoc