#] #] ********************* #] "$d_Qmkts"'0_PuetzUWS intnl stocks notes.txt' - #] see "$d_Qmkts"'0_PuetzUWS intnl stocks notes.txt' # www.BillHowell.ca 12Aug2022 initial # view in text editor, using constant-width font (eg courier), tabWidth = 3 # 19Mar2023 I need to rename this file # "$d_mktData"'1_mktData download instructions.txt' - specific to downloads to csv ONLY # monthly update 70 symbols [3M,5Y,All], see "$d_Qmkts"'0_PuetzUWS intnl stocks notes.txt' see also "$d_mkts"'market data/' for work on symbols "$d_mktData"'1_mktData download instructions.txt' "$d_Qmkts"'0_PuetzUWS intnl stocks notes.txt' - 19Mar2023 I need to rename this file Market data downloads REQUIRE : "$d_Qmkts"'market symbol [regressSL, detrend].ndf' "$d_Qmkts"'market symbol header.ndf' #48************************************************48 #24************************24 # Table of Contents, generate with : # $ grep "^#]" "$d_Qmkts"'0_PuetzUWS intnl stocks notes.txt' | sed "s/^#\]/ /" # ********************* "$d_Qmkts"'0_PuetzUWS intnl stocks notes.txt' - +-----+ Market-related [file, dir]s : +-----+ ToDos : +-----+ 06Aug2022 create_pPng pPlt pPng - pPlt curves, text too long, chokes 06Aug2022 Manually fixed pPlt works nicely, insuffient [tSpikeL, fracLLL] depths 05Aug2022 rerun everything with new(old) yDetrend := (Y/yEstTrn - 1)/relStdDevTrn 04Aug2022 detrend graph - problem with scaling, non-detrended index prices 04Aug2022 Puetz intl stock mkt detrend - tie all code together, clean up 02Aug2022 doAll_calc_tSpikeLL_fracLLL 30-31Jul2022 adapt 'stock index [semi-log, detrend] data processing.ndf' 24May2021 search "gnuplot and how do I set dashed lines?" 30Jul2022 upgrade, clean, clear] 'PuetzUWS calc [time, price] fractals.ndf' 28Jul2022 rsdP_find_aboveBelowLimits 22Jul2022 change regression [calcc,output]s 21Jul2022 revamp yDetrend to log (yObsTrans / yEstTrans), add PuetzUWS [time, price] multi-fractals #24************************24 #] +-----+ #] Market-related [file, dir]s : Market data downloads REQUIRE : "$d_Qmkts"'market symbol [regressSL, detrend].ndf' "$d_Qmkts"'market symbol header.ndf' Market data related : "$d_mktData"'[, detrend, digitize graphs, rawCsv, semi-log, SLregress, TV Fibonacci mirror, unixDate, yrFraction]/' "$d_mkts"'Fischer, David 1996 The Great Wave/0_Fischer notes.txt' also, many other [data, analysis] in "$d_[mkts, Qroot]" "$d_SysMaint"'digitizelt graph from image/digitizelt - produce graphical data from image.txt' PuetzUWS [time, price] multifractals "$d_Qmkts"'PuetzUWS calc [time, price] fractals.ndf modelling as continuous systems : "$d_mkts"'PE Schiller forward vs 10yr Tbills/' "$d_PuetzUWS"'0_PuetzUWS PSOmodel notes.txt' #24************************24 #] +-----+ #] ToDos : #] 31Aug2022 need to modify to include [min, max] of yDetrend[Min, Max]L : yrZero_yrP_calc_timeFracLL_priceFracLLL #] 20Mar2023 SP500: manipulate markets, be a hero in your own mind easy tool for inputting assumptions into forecasts (>1 month range) #] 20Mar2023 SP500 forecast wars [Deus-Fomenko, Harry Dent, ??] similarity measure for approximate time #] +-----+ #24************************24 #08********08 #] ??Jun2023 #08********08 #] ??Jun2023 #08********08 #] ??Jun2023 #08********08 #] ??Jun2023 #08********08 #] ??Jun2023 #08********08 #] 18Jun2023 monthly data download (partial only, but improved instructions) see instructions : "$d_mktData"'1_mktData download instructions.txt' #08********08 #] 26Mar2023 pCsv_to_pYrFrac: augment pYrFrac with new data, create pLog to track +----+ 24Mar2023 RECAP- already done : pCsv_to_pYrFrac pCsv with : $ ls -1 "$d_PROJECTS"'economics, markets/mktDat M/yrFrac/' na pCsv := link d_PROJECTS 'economics, markets/mktDat M/rawCsv/rawCsv CA10Y TV.txt' y pCsv := link d_PROJECTS 'economics, markets/mktDat M/rawCsv/rawCsv CUS2 TV.txt' y pCsv := link d_PROJECTS 'economics, markets/mktDat M/rawCsv/rawCsv DAX40 TV.txt' y pCsv := link d_PROJECTS 'economics, markets/mktDat M/rawCsv/rawCsv GOLD TV.txt' y pCsv := link d_PROJECTS 'economics, markets/mktDat M/rawCsv/rawCsv HSSSHSC300 TV.txt' y pCsv := link d_PROJECTS 'economics, markets/mktDat M/rawCsv/rawCsv INDIA50 TV.txt' y pCsv := link d_PROJECTS 'economics, markets/mktDat M/rawCsv/rawCsv NI225 TV.txt' y pCsv := link d_PROJECTS 'economics, markets/mktDat M/rawCsv/rawCsv SHCOMP TV.txt' y pCsv := link d_PROJECTS 'economics, markets/mktDat M/rawCsv/rawCsv TNX TV.txt' y pCsv := link d_PROJECTS 'economics, markets/mktDat M/rawCsv/rawCsv UKX100 TV.txt' Test logfile on new symbol that hasn't been processed : pCsv := link d_PROJECTS 'economics, markets/mktDat M/rawCsv/rawCsv CADUSD TV.txt' pCsv_to_pYrFrac pCsv >> no output? or pTmp1 still not used? y pCsv_to_pYrFrac 1.txt y pCsv_to_pYrFrac mktDat M log.txt NUTS! one chr per output line commented out : % tblWhole_writeTabSep_padd tabWholeNew|[idxLNew,] pYrFrac ; 'plog MinMaxYrFrac.txt' : 230324 15h56m CA10Y "mktDat M" ?address 2022.66 230326 13h31m CADUSD "mktDat M" ?address ?address >> fix this first so I can see what's going on... +-----+ OK - output format is fixed Now the problem is : tabWholeOld - [?slice >> Of course!! it doesn't exist yet!! code mods didn't create pYrFrac, same errors test : qnial> d_mktDat := '/home/bill/PROJECTS/economics, markets/' /home/bill/PROJECTS/economics, markets/ qnial> pYrFrac := link d_mktDat 'mktDat M/yrFrac/yrFrac CADUSD TV.txt' /home/bill/PROJECTS/economics, markets/mktDat M/yrFrac/yrFrac CADUSD TV.txt qnial> pth_typ_exists pYrFrac "pinn l >> WRONG!?!?!? Must FIX pth_typ_exists, see : "$d_Qtest"'fileops/file_ops- development.txt' #08********08 #] 24Mar2023 pCsv_to_pYrFrac - "mktDat M" work with symbols without a viable pYrFrac $ ls -1 "$d_PROJECTS"'economics, markets/mktDat M/rawCsv/' >"$d_temp"'dir mktDat M, rawCsv.txt' $ ls -1 "$d_PROJECTS"'economics, markets/mktDat M/yrFrac/' >"$d_temp"'dir mktDat M, yrFrac.txt' $ diff "$d_temp"'dir mktDat M, rawCsv.txt' "$d_temp"'dir mktDat M, yrFrac.txt' --suppress-common-lines | grep "<" | sed 's/< //' >"$d_temp"'diff mktDat M, rawCsv versus yrFrac.txt' >> wow! a lot (65 files)... save to file I removed dir z_Archive from file #08********08 #] 24Mar2023 pCsv_to_pYrFrac - % read p[Csv, YrFrac] into QNial tabTbl ; Again : pCsv := link d_PROJECTS 'economics, markets/mktDat M/rawCsv/rawCsv DAX40 TV.txt' pCsv_to_pYrFrac pCsv -->[nextv] EACH secFrom01Jan1970_to_yFrac 205056000 476092800 951897600 1656658800 1976.5 1985.09 2000.16 2022.5 >> OK - so what's wrong with tblDatNew|[,0] := EACH secFrom01Jan1970_to_yFrac tblDatNew|[,0] ; >> wrong, all are 1970. !?!?!? >> YIKES! LHS col is choped off, 2nd col as date!? >> OK, makes sens when converting to tblWhole tabWholeNew[count nRows pNew, 0] := 1656658800 12627.66 13062.23 12390.95 12952.17 12627.66 13062.23 12390.95 12952.17 35831904755 78085834834.75 % read pCsv into QNial tabTbl ; IF flag_break THEN BREAK ; ENDIF ; tabWholeNew := pinn_readExecute_tabTbl pTmp1 ; - tblDatNew titLLNew := tblWhole_make_tbl tabWholeNew ; - yrFracNew := tblWholeNew ; % ; % if pCsv has unixDates - convert to yrFrac (eg all TradingView data has unixDates) ; write link 'refer : ' refer ; IF (refer = 'TV') THEN - % tblDatNew|[,0] := EACH secFrom01Jan1970_to_yFrac (EACH execute tblDatNew|[,0]) ; tblDatNew|[,0] := EACH secFrom01Jan1970_to_yFrac tblDatNew|[,0] ; + tblDatNew|[0,0] := 'time' ; ENDIF ; yrMinDatNew := tblDatNew|[1,0] ; yrMaxDatNew := tblDatNew|[(gage shape tblDatNew) - 1,0] ; write '[yMinDatNew, yMaxDatNew] = ' yMinDatNew yMaxDatNew ; % ; % read pYrFrac ; pYrOld := pYrFrac ; tabWholeOld := pinn_readExecute_tabTbl pYrOld ; - tblDatOld titLLOld := tblWhole_make_tbl tabWholeOld ; yrMinDatOld := tabWholeOld|[1,0] ; yrMaxDatOld := tabWholeOld|[(gage shape tabWholeOld) - 1,0] ; write '[yMinDatOld, yMaxDatOld] = ' yMinDatOld yMaxDatOld ; % ; % [23,24]Mar2023 leave out for now? ; } +-----+ yrL := (first (gage shape tabWholeNew)) - (reverse tell 5) tabWholeNew|[,0] OK - seems to work well so far. I don't actually append new data or save files yet, so that's the next step after testing a couple more dataFiles. pCsv_to_pYrFrac pCsv with : $ ls -1 "$d_PROJECTS"'economics, markets/mktDat M/yrFrac/' na pCsv := link d_PROJECTS 'economics, markets/mktDat M/rawCsv/rawCsv CA10Y TV.txt' y pCsv := link d_PROJECTS 'economics, markets/mktDat M/rawCsv/rawCsv CUS2 TV.txt' y pCsv := link d_PROJECTS 'economics, markets/mktDat M/rawCsv/rawCsv DAX40 TV.txt' y pCsv := link d_PROJECTS 'economics, markets/mktDat M/rawCsv/rawCsv GOLD TV.txt' y pCsv := link d_PROJECTS 'economics, markets/mktDat M/rawCsv/rawCsv HSSSHSC300 TV.txt' y pCsv := link d_PROJECTS 'economics, markets/mktDat M/rawCsv/rawCsv INDIA50 TV.txt' y pCsv := link d_PROJECTS 'economics, markets/mktDat M/rawCsv/rawCsv NI225 TV.txt' y pCsv := link d_PROJECTS 'economics, markets/mktDat M/rawCsv/rawCsv SHCOMP TV.txt' y pCsv := link d_PROJECTS 'economics, markets/mktDat M/rawCsv/rawCsv TNX TV.txt' y pCsv := link d_PROJECTS 'economics, markets/mktDat M/rawCsv/rawCsv UKX100 TV.txt' close to end, eg pCsv := link d_PROJECTS 'economics, markets/mktDat M/rawCsv/rawCsv CUS2 TV.txt' idx of tblWholeNew to start append: 196 but end is lines [196, 197], so "false update" same for [GOLD, SHCOMP, TNX, ] No new data for symbol : [DAX40, HSSSHSC300, INDIA50, NI225, UKX100] +-----+ olde code IF flag_break THEN BREAK ; ENDIF ; idxLParen := findall `/ d_Raw ; idxLast := last idxLParen ; idx2nd := last front idxLParen ; idx3rd := last front front idxLParen ; % (last findall `/ (front d_Raw)) + 1 ; d_bas := (tell (idxLast + 1)) choose pCsv ; subDir := (idx3rd + (count (idx2nd - idx3rd - 1))) choose d_raw ; % (tell ((last findall `/ (front d_bas)) + 1)) choose d_bas ; wordL := ary_itm_cut_subL fNam ` ; symbol := second wordL ; refer := first ary_itm_cut_subL (last wordL) `. ; pUnixDate := link d_bas 'unixDate/' 'unixDate ' symbol ' TV.txt' ; pYrFrac := link d_bas 'yrFrac/' 'yrFrac ' symbol ' TV.txt' ; # [23,24]Mar2023 leave out for now? tblWhole_yrFrac := tbl_make_tblWhole tblDat_titLL_make_tbl tblDat (rowTitL colTitL) ; tblWhole_writeTabSep_povr tblWhole_yrFrac pYrFrac ; % ; % read tabTbl [existing, new] into QNial ; % ; % for TradingView? - replace <> with apos ; cmd := link 'cat "' pTmp1 '" | tr \< \d39 | tr \> \d39 >"' pTmp2 '" ' ; write cmd ; host cmd ; % make it easier to avoid "crossover" file mistakes ; host link 'mv "' pTmp2 '" "' pTmp1 '" ' ; } #08********08 #] 23Mar2023 pCsv_to_pYrFrac - continue work $ tail -n +2 "$d_Qndfs"'faults - header.ndf' >> OK test pCsv_to_pYrFrac % apo NaN, it is specific to TradingView after 1st line - no other text after first line? ; cmd := link 'sed "s|NaN|\' chr_apo 'NaN\' chr_apo '|g" >"' pTmp1 '" ' ; write cmd ; host cmd ; >> hangs up on host cmd!?!?!? sed "s|NaN|\'NaN\'|g" >"/media/bill/ramdisk/pCsv_to_pYrFrac 1.txt" ^Cqnial> >> oops, forgot to include input file (was with olde code) Again : pCsv := link d_PROJECTS 'economics, markets/mktDat M/rawCsv/rawCsv DAX40 TV.txt' pCsv_to_pYrFrac pCsv strTst := link 'time open high low close ' chr_apo 'DAX XETR: Open' chr_apo ' ' chr_apo 'DAX XETR: High' chr_apo ' ' chr_apo 'DAX XETR: Low' chr_apo ' ' chr_apo 'DAX XETR: Close' chr_apo ' Volume Volume MA' +--+ time open high low close 'DAX XETR: Open' 'DAX XETR: High' 'DAX XETR: Low' 'DAX XETR: Close' Volume Volume MA +--+ >> strLnk := link ary_itmP_replace_aryOut strTst (chr_apo null) Now try : line1 := link ary_itmP_replace_aryOut line1 (chr_apo null) line1 := link 'time' chr_tab 'open' chr_tab 'high' chr_tab 'low' chr_tab 'close' chr_tab 'DAX XETR: Open' chr_tab 'DAX XETR: High' chr_tab 'DAX XETR: Low' chr_tab 'DAX XETR: Close' chr_tab 'Volume' chr_tab 'Volume MA' time open high low close DAX XETR: Open DAX XETR: High DAX XETR: Low DAX XETR: Close Volume Volume MA qnial> EACH charrep line1 116 105 109 101 9 111 112 101 110 9 104 105 103 104 9 108 111 119 9 99 108 111 115 101 9 68 65 88 32 32 88 69 84 82 58 32 79 112 101 110 9 68 65 88 32 32 88 69 84 82 58 32 72 105 103 104 9 68 65 88 32 32 88 69 84 82 58 32 76 111 119 9 68 65 88 32 32 88 69 84 82 58 32 67 108 111 115 101 9 86 111 108 117 109 101 9 86 111 108 117 109 101 32 77 65 qnial> line1 := link ary_itmP_replace_aryOut line1 (chr_tab (chr_apo chr_tab chr_apo)) time' 'open' 'high' 'low' 'close' 'DAX XETR: Open' 'DAX XETR: High' 'DAX XETR: Low' 'DAX XETR: Close' 'Volume' 'Volume MA qnial> line1 := link chr_apo line1 chr_apo 'time' 'open' 'high' 'low' 'close' 'DAX XETR: Open' 'DAX XETR: High' 'DAX XETR: Low' 'DAX XETR: Close' 'Volume' 'Volume MA' >> Perfect line1 := link ary_itmP_replace_aryOut line1 (chr_apo null) +-----+ olde code % put apos around each tab, at [start, end] of line ; in_titl := o ; FOR chr WITH linOut DO IF (chr = `') THEN IF in_titl THEN in_titl := o ; ELSE in_titl := l ; ENDIF ; chr_old := null ; chr := link chr_tab chr ; ELSEIF (AND (NOT in_titl) (chr ~= chr_tab)) THEN chr := link chr_apo chr ; in_titl := l ; chr_old = null ; ELSEIF (AND (NOT in_titl) (chr = chr_tab)) THEN chr := link chr chr_apo ; in_titl := l ; chr_old := chr_tab ; chr := null ; ELSEIF (AND in_titl (chr = chr_tab)) THEN chr := link chr_apo chr ; in_titl := o ; chr_old := null ; % ELSEIF (AND in_titl (chr ~= chr_tab)) -> no action ; ENDIF ; line1 := link line1 chr ; ENDFOR ; IF ((last line1) ~= chr_apo) THEN line1 := link line1 chr_apo ; ENDIF ; write link 'line1 : ' line1 ; % write EACH charrep line1 ; % in_titl := o ; FOR chr WITH linOut DO IF (chr = `') THEN IF in_titl THEN in_titl := o ; ELSE in_titl := l ; ENDIF ; chr_old := null ; chr := link chr_tab chr ; ELSEIF (AND (NOT in_titl) (chr ~= chr_tab)) THEN chr := link chr_apo chr ; in_titl := l ; chr_old = null ; ELSEIF (AND (NOT in_titl) (chr = chr_tab)) THEN chr := link chr chr_apo ; in_titl := l ; chr_old := chr_tab ; chr := null ; ELSEIF (AND in_titl (chr = chr_tab)) THEN chr := link chr_apo chr ; in_titl := o ; chr_old := null ; % ELSEIF (AND in_titl (chr ~= chr_tab)) -> no action ; ENDIF ; line1 := link line1 chr ; ENDFOR ; IF ((last line1) ~= chr_apo) THEN line1 := link line1 chr_apo ; ENDIF ; #08********08 #] 21Mar2023 revamp pCsv_to_pYrFrac $ find "$d_PROJECTS"'economics, markets/mktDat 1871-/rawCsv/' -type f -name "rawCsv *" | sed "s|\(.*\)\/||g" rawCsv UKblueChip 1829-2018 Campbell, Grossman, Turner.txt rawCsv UK100 1692 to 2018 GFDFinaeon.txt rawCsv USOIL Dec1861-Aug2022 TV.txt rawCsv SPX 1872-Aug2022 TV.txt >> OK $ find "$d_PROJECTS"'economics, markets/mktDat W/rawCsv/' -type f -name "rawCsv *" | sed "s|\(.*\)\/||g" ~ >> OK, there are only 'rawCsvMulti ' files 22Mar2023 +-----+ olde code # 22Mar2023 pCsv := link d_PROJECTS 'economics, markets/mktDat M/rawCsv/rawCsv DAX40 TV.txt' pCsv_to_pYrFrac pCsv qnial> FOR i with tell 5 DO IF i = 3 THEN i := i + 2 ; ENDIF ; write i ; ENDFOR ; 0 1 2 5 4 >> OK, approach works with FOR, i osn't messed up # 22Mar2023 % if pCsvRaw has unixDates - convert to yrFrac (eg all TradingView data has unixDates) ; bolUnix := tbl_keyAxiTit_selAxiTitL_get_itmLL mktTbl (0 symbol) (1 "bolUnixL) ; IF bolUnix THEN tblDat|[,0] := EACH secFrom01Jan1970_to_yFrac (EACH execute tblDat|[,0]) ; ENDIF ; % ; tblWhole_yrFrac := tbl_make_tblWhole tblDat_titLL_make_tbl tblDat (rowTitL colTitL) ; % ??? ; tblWhole_writeTabSep_povr tblWhole_yrFrac pYrFrac ; } # pCsv_to_pYrFrac 'CUS2 May2006-Aug2022 TradingView' # 12Mar2023 pCsv_to_pYrFrac 'SPX 1872-now TradingView' # symbol := 'SPX 1872-now TradingView' # pTmp1 := link d_temp 'pCsv_to_pYrFrac 1.txt' # pCsv := link d_rawCsv 'rawCsv ' symbol '.txt' ; # 22Mar2023 test # pCsv := link d_PROJECTS 'economics, markets/mktDat M/rawCsv/rawCsv CA10Y TV.txt' # pCsv_to_pYrFrac pCsv # olde code % ; % 24Aug2022 do I need pinn_readExecute_tabTbl? nyet, can't until strings identified ; tbl_raw := pinn_readStr_tabTbl pTmp2 ; tblDat titLL := tblWhole_make_tbl tbl_raw ; rowTitL colTitL := titLL ; rowTitL := EACH str_wrapWithApos rowTitL ; #08********08 #] 19Mar2023 draft optr pCsvMultiSym_to_pLYrFrac IS OP ??? much work remains... +-----+ olde code # +--+ # BEFORE running revampALL_CsvToTab, in "rawCsv files" : # clean up titleLine - remove commas within quotes, NaN # 12Mar2023 for "$d_Qmkts"'market symbol [regressSL, detrend].ndf' example : $ fNam='rawCsv SPX 1872-now TradingView.txt' # $ sed 's|\"\([A-Za-z0-9]\+\)\,|"\1|g' "$d_mktData"'rawCsv/'"$fNam' | sed "s|NaN||g" >"$d_temp$fNam" # $ mv "$d_temp$fNam" "$d_mktData"'rawCsv/'"$fNam' #08********08 #] 19Mar2023 PineScript - how do I script-download data? (or don't use TV!) #] OK! can download all symbols on multi-charts #08********08 #] 12Mar2023 pCsv_to_pYrFrac 'yrFrac SPX 1872-now TradingView.txt' (not working) Wow! I completely forgot everything from last Oct2022, no real instructions many LMDE envVars have changed too qnial> pCsv_to_pYrFrac 'SPX 1872-now TradingView' sed 's|\"\([A-Za-z0-9]\+\)\,|"\1|g' "/home/bill/PROJECTS/economics, markets/market data/rawCsv/rawCsv SPX 1872-now TradingView.txt" | sed "s|NaN||g" >"/media/bill/ramdisk/pCsv_to_pYrFrac 1.txt" cat "/media/bill/ramdisk/pCsv_to_pYrFrac 1.txt" | tr \< \d39 | tr \> \d39 >"/media/bill/ramdisk/pCsv_to_pYrFrac 2.txt" >> it runs, but result is completely mangled! : "$d_mktData"'yrFraction/yrFrac SPX 1872-now TradingView.txt' ... continue next Sunday #08********08 #] 08Oct2022 [fix, update] cart [[bolA_ary_get, ary_itm_cut]_subWithNullsL, pinn_read[Execute, Str]_tabTbl, tblWhole_writeTabSep_povr] >> OK, working after fixes, but need formal tests record of work with each optr Change & save pinn := link d_Qmkts 'market tabWhole.dat' qnial> pinn := link d_Qmkts 'market tabWhole.dat' /home/bill/SG6/web/Qnial/MY_NDFS/economics, markets/market tabWhole.dat qnial> tblWhole := pinn_readExecute_tabTbl pinn >> tblWhole looks good from : symbol_xP_SLRegress IS OP symbol xP : mktTbl := tbl_keyAxiTit_selAxiTitL_put_itmLL mktTbl (0 symbol) (1 ("constant "slope "relStdErr "relStdDev "xObsMin "xObsMax "yObsMin "yObsMin "yDetrendMin "yDetrendMax) ) ( constant slope relStdErr relStdDev xObsMin xObsMax yObsMin yObsMin yDetrendMin yDetrendMax) ; # Try : mktTbl2 := tbl_keyAxiTit_selAxiTitL_put_itmLL mktTbl (0 'SHCOMP 2002-Aug2022 TradingView') (1 ("constant "slope "relStdErr "relStdDev "xObsMin "xObsMax "yObsMin "yObsMin "yDetrendMin "yDetrendMax)) ( -99. 0.0999 1.999 0.999 0.099 0.99 "yObsMin 0.909 "yDetrendMin "yDetrendMax) >> didn't work, run break trace tbl_keyAxiTit_selAxiTitL_put_itmLL >> oops, removed `L pre-tab in mktTblWhole "symName_dateRange "bolStockIntl "bolUSMix "constant "slope "relStdErr "relStdDev "normalRate "xObsMin "xObsMax "yObsMin "yObsMin "yDetrendMin "yDetrendMax "bolUnix "xCol "yCol "xTrnPhr "yTrnPhr "xTics "yRange "yTics "yLabel Try again : mktTbl2 := tbl_keyAxiTit_selAxiTitL_put_itmLL mktTbl (0 'SHCOMP 2002-Aug2022 TradingView') (1 ("constant "slope "relStdErr "relStdDev "xObsMin "xObsMax "yObsMin "yObsMin "yDetrendMin "yDetrendMax)) ( -99. 0.0999 1.999 0.999 0.099 0.99 '"yObsMin' 0.909 '"yDetrendMin' '"yDetrendMax') -->[nextv] titLL@selAxi bolStockIntl bolUSMix constantL slopeL relStdErrL relStdDevL normalRate xObsMinL xObsMaxL yObsMinL yObsMinL yDetrendMinL yDetrendMaxL bolUnixL xCol yCol xTrnPhrL yTrnPhrL xTicsL yRangeL yTicsL yLabelL -->[nextv] titLL@selAxi EACHRIGHT ary_itm_get_idxL ("constant "slope "relStdErr) ?aryItmFindIdxL01 ?aryItmFindIdxL01 ?aryItmFindIdxL01 -->[nextv] titLL@selAxi ary_itm_get_idxL "constant ?aryItmFindIdxL01 >> so that optr doesn't work "constant findall titLL@selAxi >> oops, `L still at end of selAxi -->[nextv] keyAxi keyTit selAxi selTitL +-+-------------------------------+-+------------------------------------------------------------------------- |0|SHCOMP 2002-Aug2022 TradingView|1|constant slope relStdErr relStdDev xObsMin xObsMax yObsMin yObsMin yDetre +-+-------------------------------+-+------------------------------------------------------------------------- ~:~ -----------------+ ndMin yDetrendMax| -----------------+ -->[nextv] keyIdx selIdxL +-+----------------------------------------------------------------------------------------------------------- |6|?aryItmFindIdxL01 ?aryItmFindIdxL01 ?aryItmFindIdxL01 ?aryItmFindIdxL01 ?aryItmFindIdxL01 ?aryItmFindIdxL01 +-+----------------------------------------------------------------------------------------------------------- ~:~ ------------------------------------------------------------------------+ ?aryItmFindIdxL01 ?aryItmFindIdxL01 ?aryItmFindIdxL01 ?aryItmFindIdxL01| ------------------------------------------------------------------------+ -->[nextv] titLL@selAxi bolStockIntl bolUSMix constantL slopeL relStdErrL relStdDevL normalRate xObsMinL xObsMaxL yObsMinL yObsMinL yDetrendMinL yDetrendMaxL bolUnixL xCol yCol xTrnPhrL yTrnPhrL xTicsL yRangeL yTicsL yLabelL -->[nextv] titLL@selAxi EACHRIGHT ary_itm_get_idxL ("constant "slope "relStdErr) ?aryItmFindIdxL01 ?aryItmFindIdxL01 ?aryItmFindIdxL01 >> same problem- `L at end of tit #? ?aryItmFindIdxL01 ary_itm_get_idxL item is not in aryTop (= null resFlt) tblWhole OK, mktTbl wrong!! loaddefs link d_Qmkts 'market symbol header.ndf' >> OK? tbl_keyAxiTit_selAxiTitL_put_itmLL change : slc := link (2 dropright slc) ']' ; to : slc := link (2 dropright slc) ']' ; change : ELSE slc := link slc (link ((EACH string tblShpTell@axi) EACHLEFT link ' ')) ', ' ; to : ELSE slc := link slc ', ' ; #08********08 #] 29Sep2022 fix file_ops, market symbol [dataProcess, regress], Puetz detrended specs].ndf (post changes last 2 months) $ qnial ?undefined identifier: STRL := ARY_SPLITTHENREPLACE_SUBP <***> STRNEW ( '[#=; ' ?undefined identifier: LINE := STREMBED_EXECUTE_STROUT <***> LINE PHRASEVALUELIST ; ?undefined identifier: P_TEMP := PINN_PHRVALUEL_EXECUTEEMBEDS <***> PINN PHRASEVALUELIST ; <<< loading ended : file_ops.ndf qnial> lq_fileops >> OK symbol_xP[_, L_plot]SLRegress - both are incomplete!! (marked with ????) ?expecting endfor: STRUWS '.png' ; <***> ???? := TBL_KEYAXITIT_SELAXITITL_GET_ITMLL ?undefined identifier: := TBL_KEYAXITIT_SELAXITITL_PUT_ITMLL TBL <***> KEYAXITIT SELAXITITL ?undefined identifier: SYMBOL ; SYMBOL_XP_SLREGRESS <***> SYMBOL ; } ?undefined identifier: ENDIF ; SYMBOL_XP_SLREGRESS <***> SYMBOL ; ENDFOR <<<<<< loading ended : market symbol [dataProcess, regress].ndf #08********08 #] 13Sep2022 boolean optrs - not quite right yet, no time to resolve now functionally the same (just NOT boolA) retain only get form : boolA_cut_ary = bolA_get_ary boolA_cut_indexA = bolA_get_idxA TWIG-based boolA_get1_ary not useful - just changes hits to solitaries, miss to [null] IF (~= (TWIG (gage shape) boolA ary)) 14Sep2022 +-----+ olde code # loaddefs link d_Qndfs 'boolean.ndf' IF flag_debug THEN write 'loading boolA_cut_ary' ; ENDIF ; #] boolA_cut_ary IS OP boolA ary - extract a list of subArys by using [boolA, slice optr] #] boolA MUST "span remaining axis" - this optr CANNOT just leave non-existant "interior portions" of ary #] handy compliment to subList (standard QNial operator) # 30Sep2021 initial, tests: boolA_cut_list_test # 15Feb2022 is this now useless? perhaps ary optrs do this? # 13Sep2022 useless but intersting - may yield more intuitive ary_cut optr, # 13Sep2022 need to convert to slice optr!!! like x's and O's # 13Sep2022 is boolA_cut_ary a complement to slice? boolA_cut_ary IS OP boolA ary { LOCAL i in_subL n_bool result subL subLL ; % ; n_bool := gage shape boolA ; % ; IF (~= n_bool (gage shape ary)) THEN subL := [null] ; subLL := fault '?boolA_cut_ary error: ~= (EACH (gage shape) boolA ary)' ; ELSE in_subL := o ; subL := null ; subLL := null ; FOR i WITH (tell n_bool) DO IF boolA@i THEN in_subL := l ; subL := link subL ary@i ; ELSE IF in_subL THEN subLL := link subLL [link subL] ; in_subL := o ; subL := [null] ; ELSE in_subL := o ; ENDIF ; ENDIF ; ENDFOR ; ENDIF ; IF (~= [null] subL) THEN link subLL [link subL] ELSE subLL ENDIF } # tests qnial> a := 3 3 reshape tell 10 0 1 2 3 4 5 6 7 8 qnial> boolA := 3 3 reshape l lll lll lll qnial> boolA_cut_ary boolA a +-----------------+ |0 1 2 3 4 5 6 7 8| +-----------------+ qnial> boolA := 3 3 reshape (l o l l o l l o l) lol lol lol qnial> boolA_cut_ary boolA a +-+---+---+-+ |0|2 3|5 6|8| +-+---+---+-+ qnial> boolA := (l o l l o l l o l) lollollol qnial> boolA_cut_ary boolA a ?boolA_cut_ary error: ~= (EACH (gage shape) boolA ary) # loaddefs link d_Qndfs 'boolean.ndf' IF flag_debug THEN write 'loading boolA_get1_ary' ; ENDIF ; #] boolA_get1_ary IS OP boolA ary - extract an ary (array) of subArys from boolA (not useful!?!) #] ary of atoms (eg [strings, integer, real, complex, phrase, boolean]) must be made solitary if needed #] very tricky - this will be hard to use for complex arys, maybe even for simple arys? # 18Oct2021 initial, [incomplete, untested] # boolA_get_ary_test # 30Sep2021 should be able can use boolA_get_ary to knock out [row, col]s of array!!! (tricky) boolA_get1_ary IS OP boolA ary { IF (~= (TWIG (gage shape) boolA ary)) THEN fault '?boolA_get1_ary error: ~= (EACH (gage shape) boolA ary)' ELSE boolA EACHBOTH boolA_get_ary ary ENDIF } # tests qnial> a := 3 3 reshape tell 10 0 1 2 3 4 5 6 7 8 qnial> boolA := 3 3 reshape l lll lll lll qnial> boolA_get1_ary boolA a +---+---+---+ |+-+|+-+|+-+| ||0|||1|||2|| |+-+|+-+|+-+| +---+---+---+ |+-+|+-+|+-+| ||3|||4|||5|| |+-+|+-+|+-+| +---+---+---+ |+-+|+-+|+-+| ||6|||7|||8|| |+-+|+-+|+-+| +---+---+---+ qnial> boolA := 3 3 reshape (l o l l o l l o l) lol lol lol qnial> boolA_get1_ary boolA a +---+--+---+ |+-+|++|+-+| ||0||||||2|| |+-+|++|+-+| +---+--+---+ |+-+|++|+-+| ||3||||||5|| |+-+|++|+-+| +---+--+---+ |+-+|++|+-+| ||6||||||8|| |+-+|++|+-+| +---+--+---+ qnial> boolA := (l o l l o l l o l) lollollol qnial> boolA_get1_ary boolA a ?boolA_get1_ary error: ~= (EACH (gage shape) boolA ary) # loaddefs link d_Qndfs 'boolean.ndf' IF flag_debug THEN write 'loading boolA_cut_indexA' ; ENDIF ; #] boolA_cut_indexA IS OP boolA - boolean # 18Oct2021 is this redundant with respect to boolA_cut_lst?? IF flag_break THEN BREAK ; ENDIF ; boolA_cut_indexA IS OP boolA { LOCAL i indexA last_boolean subList tell_boolean ; tell_boolean := tell (gage shape boolA) ; subList := null ; indexA := null ; last_boolean := null ; FOR i WITH tell_boolean DO IF (= last_boolean boolA@i) THEN subList := link subList tell_boolean@i ; ELSE indexA := append indexA subList ; subList := null ; subList := tell_boolean@i ; ENDIF ; last_boolean := boolA@i ; ENDFOR ; rest indexA } # boolA_cut_indexA_str := link 'IF delta > 0 THEN host (link ' chr_apo 'sleep' chr_apo ' (string delta) ' chr_apo 's' chr_apo ') ; ENDIF ; ' # boolA_cut_indexA_result boolA_cut_indexA_str chrs_QNialSymbols #08********08 #] 11Sep2022 continue revamp tbl optrs to >2D +-----+ olde code # 24************************24 # +-----+ # Simple [array, table, matrix] operators - Howell : IF flag_debug THEN write '+-----+' ; ENDIF ; IF flag_debug THEN write 'Simple [array, table, matrix] operators - Howell : ' ; ENDIF ; # [colTitL, tbl] are [nonlocal, permanent] - # create copies of this optr for [tbl, colTitL][0, 1, 2, 3, ...] # OR reassign [colTitL, tbl], and just use this one optr # oversimplified QNial roster databases, really just a tbl pinn_readExecute_tabTbl IS OP pinn { LOCAL line n_rows n_cols resFlt rowStrL ; rowStrL := null ; IF (NOT (pth_typ_exists pinn "pinn)) THEN resFlt := fault '?pinn_readExecute_tabTbl file unknown error : ' pinn ; ELSE finn := open pinn "r ; WHILE (NOT isfault (line := readfile finn)) DO IF (`# ~= line@0) THEN rowStrL := append rowStrL (ary_itm_cut_subL line chr_tab) ; ENDIF ; ENDWHILE ; close finn ; % ; colTitles := first rowStrL ; colTitleA := 1 (gage shape colTitles) reshape colTitles ; rowStrL := rest rowStrL ; n_rows := gage shape rowStrL ; n_cols := gage shape (first rowStrL) ; rawTbl := n_rows n_cols reshape (link rowStrL) ; exeTbl := EACH str_to_typ rawTbl ; resFlt := aryA_catVertical (colTitleA exeTbl) ; ENDIF ; resFlt } qnial> see "boolL_option_cull_lst booll_option_cull_lst IS OPERATION Booll Option Lst { LOCAL I In_subl N_bool Result Subl Subll ; % ; N_bool := gage shape Booll ; % ; IF ( ~= N_bool ( gage shape Lst ) ) THEN Subl := [ Null ] ; Result := fault '?boolL_option_cull_lst error, ~= (EACH (gage shape) boolCullL lst)' ; ELSE In_subl := o ; Subl := Null ; Subll := Null ; IF ( = "cut Option ) THEN Boolculll := not Booll ; ELSE Boolculll := Booll ; ENDIF ; FOR I WITH ( tell N_bool ) DO IF Boolculll @ I THEN In_subl := l ; Subl := append Subl Lst @ I ; ELSE IF In_subl THEN Subll := append Subll Subl ; In_subl := o ; Subl := Null ; ELSE In_subl := o ; ENDIF ; ENDIF ; ENDFOR ; IF ( ~= Null Subl ) THEN Result := append Subll Subl ; ELSE Result := Subll ; ENDIF ; IF ( = "hold Option ) THEN Result := link Result ; ENDIF ; ENDIF ; Result } #08********08 #] 08Sep2022 continue revamp tbl optrs to >2D +-----+ olde code IF flag_debug THEN write 'loading tbl_titLL_get_sub' ; ENDIF ; # tbl_titLL_get_sub IS OP tbl titLL keyL - returns subTbl from titLL, then with key in axiIdxHyperPlane # assumes that tbl_titLL_check has been run, so data is consistent # titL=null (part of titLL) retains all axiIdxs for that axis # keyL=null simply returns subTbl defined by titLL, if only one key, then use [keyL] in args!! # easy to use with boolean-valued cols that can contain other vals # remember : [row, col]TitL are LISTS, if only one returned it is in form of [[row, col]Tit] # 06Sep2022 initial, 07Sep2022 revamped for >2D # use successive tbl_[col, row]TitL_key_get_subTbl for multi-selections [inefficient, robust, flexible] # ordering of dataCols in subTbl depends on titLL # retains rowTitles (col 0) # example usage : symName_dateRangeL UnixDateToYrfracL := EACH rest cols # tbl_titLL_get_subTbl tbl ("symName_dateRangeL "UnixDateToYrfracL) ; # see tblDat_titLL_make_tbl_testSimple in 'arrays/arrays- testOptrs.ndf' for examples # 07Sep2022 can't use key=null as search term, but can replace with a special code like '~|~' # IF flag_break THEN BREAK ; ENDIF ; tbl_titLL_get_sub IS OP tbl titLL keyL { LOCAL idxL idxLL strIdxLL faultL i subTbl subTblDat tblDat tblDatShp titL titAllL titAllLL subTitLL slcAxiL slcAxiLL valencer bolA bolAxi bolAxiLL ; % ; faultL idxLL subTbl := null null null ; titAllLL tblDat := tbl ; valencer := valence tblDat ; tblDatShp := gage shape tblDat ; % ; % find idxLL of titLL, that is, lists of axis titles to be retained on first pass ; FOR i WITH valencer DO titL titAllL := titLL@i titAllLL@i ; IF (= null titL) THEN idxL := tell tblDatShp@i ; ELSE idxL := titAllL EACHRIGHT ary_itm_find_idxL titL ; % there must be at least one axisIdx with key, if there are any itms=key! ; IF (OR (ary_hasFaults idxL) (= null idxL)) THEN faultL := link faultL (fault '?tblTitKeyGetSub01') ; ELSEIF (issingle idxL) THEN idxL := first idxL ; ENDIF ; ENDIF ; idxLL := link idxLL [idxL] ; ENDFOR ; % ; IF (NOT ary_hasFaults faultL) THEN % create subTbl based on titLL ; subTitLL := idxLL EACHBOTH EACHLEFT pick titAllLL ; subTblDat := tblDat EACHLEFT pick (cart idxLL) ; % ; IF (~= null keyL) THEN % find key in subTbl, reduce subTbl downTo titLL of cells that have key ; % axiIdxL are selected on whether there is a `l for that axiIdx in it's hyperPlane ; bolA := OR (keyL EACHLEFT EACHRIGHT = subTblDat) ; IF (NOR bolA) THEN slcAxiLL := '[' ; FOR i WITH valencer DO bol := bolA@i ; slcAxiL := null ; FOR j WITH (tell tblDatShp@i) DO bolAxi := IF bolAxi THEN slcAxiL := link slcAxiL (string j) ; ENDIF ; ENDFOR ; slcAxiLL := link slcAxiLL slcAxiL ', ' ; ENDIF ; slcAxiLL := (2 dropright slcAxiLL) link ']' ; subTblDat := execute link 'subTblDat|' slcAxiLL ; subTbl := subTitLL subTblDat ; ENDIF ; ENDIF ; % ; IF (ary_hasFaults faultL) THEN faultL ELSE subTbl ENDIF } # fault list : #? ?tblTitKeyGetSub01 tbl_titLL_get_sub titL none in titAllL # ary_itm_find_idxL (tell 20) (3 8 13 17) # (tell 20) EACHRIGHT ary_itm_find_idxL (3 8 13 17) IF flag_debug THEN write 'loading tbl_keyColTit_keyItm_colTitL_get_itmLL' ; ENDIF ; # tbl_keyColTit_keyItm_colTitL_get_itmLL IS OP tbl keyColTit keyItm colTitL - # returns a list of lists of keyItm hits, do NOT (solitary keyItm)! # 14Aug2022 initial # 14Aug2022 upgrade later to use keyItmL rather than just keyItm # [colTitL, tbl] are [nonlocal, permanent] - # create copies of this optr for [tbl, colTitL][0, 1, 2, 3, ...] # OR reassign [colTitL, tbl], and just use this one optr # oversimplified QNial roster databases, really just a tbl # IF flag_break THEN BREAK ; ENDIF ; tbl_keyTitLL_keyItm_titLL_get_itmLL IS OP tbl keyTitLL keyItm titLL { LOCAL itmLL rowTitAllL colTitAllL tblDat titLL keyIdx colIdxL rowIdxL ; % ; tblDat titLL := tbl ; rowTitAllL colTitAllL := titLL ; rowTitL colTitL := keyTitLL ; keyIdx := first ary_itm_find_idxL colTitAllL keyColTit ; itmLL := null ; IF (ary_hasFaults keyIdx) THEN itmLL := link itmLL (fault '?tblKeyColGetItm01') ; ENDIF ; % ; colIdxL := link (colTitAllL ary_itm_find_idxL colTitL) ; rowIdxL := link (rowTitAllL ary_itm_find_idxL colTitL) ; IF (OR (ary_hasFaults colIdxL) (OR (null EACHRIGHT = colIdxL))) THEN itmLL := link itmLL (fault '?tblKeyColGetItm02') ; ENDIF ; % ; % augment rowIdxL by one as n_tblCols includes colTitL ; rowIdxL := 1 + ary_itm_find_idxL tblDat|[,keyIdx] keyItm ; IF (OR (ary_hasFaults rowIdxL) (OR (= null rowIdxL))) THEN itmLL := link itmLL (fault '?tblKeyColGetItm03') ; ENDIF ; % ; IF (NOT ary_hasFaults itmLL) THEN itmLL := tbl|[rowIdxL, colIdxL] ; ENDIF ; % ; itmLL } # faultL #? ?tblKeyColGetItm01 tbl_keyColTit_keyItm_colTitL_get_itmLL keyColTit doesn"t exist #? ?tblKeyColGetItm02 tbl_keyColTit_keyItm_colTitL_get_itmLL colTitL not found #? ?tblKeyColGetItm03 tbl_keyColTit_keyItm_colTitL_get_itmLL rowVal not found % for each axis, retain keyIdxL ; % repeat bolAdrL := null ; FOR adr WITH keyAdrL DO bolAdrL := link bolAdrL (EACH OR (adr EACHBOTH EACHLEFT in keyTitIdxLL)) ; ENDFOR ; keyAdrL := bolAdrL subList keyAdrL ; keyIdxL := slcKeyAdrL := faultL := null ; IF (ary_hasFaults keyIdxL) THEN faultL := link faultL (fault '?tblKeyTitItmGetSub01') ; ELSE % reduce subTbl downTo hyperPlanes that have key ; bolA := OR (keyL EACHLEFT EACHRIGHT = tblDat) ; IF (NOR bolA) THEN slcAxiLL := '[' ; FOR i WITH valencer DO bol := bolA@i ; slcAxiL := null ; FOR j WITH (tell tblDatShp@i) DO bolAxi := IF bolAxi THEN slcAxiL := link slcAxiL (string j) ; ENDIF ; ENDFOR ; slcAxiLL := link slcAxiLL slcAxiL ', ' ; ENDIF ; IF flag_debug THEN write 'loading tbl_keyTitLL_keyItmL_get_idxLL' ; ENDIF ; #] tbl_keyTitLL_keyItmL_get_idxLL IS OP tbl keyTitLL keyItmL - #] returns an idxLL of keyItmL hits #] assumes that args are consistent - use _chk to ensure this!! #] (keyTitL = null) returns ALL idxs of key in tbl, keyTit (for any axis) ignores that axis #] make sure that single-arg [keyTitLL, keyItmL] are solitaries (protect [str, ary, list, etc] titles) # 14Aug2022 initial # 14Aug2022 upgrade later to use keyItmL rather than just keyItm # [colTitL, tbl] are [nonlocal, permanent] - # create copies of this optr for [tbl, colTitL][0, 1, 2, 3, ...] # OR reassign [colTitL, tbl], and just use this one optr # oversimplified QNial roster databases, really just a tbl # IF flag_break THEN BREAK ; ENDIF ; tbl_keyTitLL_keyItmL_get_idxLL IS OP tbl keyTitLL keyItmL { LOCAL itmLL rowTitAllL colTitAllL tblDat titLL keyIdx colIdxL rowIdxL ; % ; tblDat titLL := tbl ; rowTitAllL colTitAllL := titLL ; rowTitL colTitL := keyTitLL ; keyIdx := first ary_itm_find_idxL colTitAllL keyColTit ; itmLL := null ; IF (ary_hasFaults keyIdx) THEN itmLL := link itmLL (fault '?tblKeyColGetItm01') ; ENDIF ; % ; colIdxL := link (colTitAllL ary_itm_find_idxL colTitL) ; rowIdxL := link (rowTitAllL ary_itm_find_idxL colTitL) ; IF (OR (ary_hasFaults colIdxL) (OR (null EACHRIGHT = colIdxL))) THEN itmLL := link itmLL (fault '?tblKeyColGetItm02') ; ENDIF ; % ; % augment rowIdxL by one as n_tblCols includes colTitL ; rowIdxL := 1 + ary_itm_find_idxL tblDat|[,keyIdx] keyItm ; IF (OR (ary_hasFaults rowIdxL) (OR (= null rowIdxL))) THEN itmLL := link itmLL (fault '?tblKeyColGetItm03') ; ENDIF ; % ; IF (NOT ary_hasFaults itmLL) THEN itmLL := tbl|[rowIdxL, colIdxL] ; ENDIF ; % ; itmLL . IF (~= null keyL) THEN % find key in subTbl, reduce subTbl downTo titLL of cells that have key ; % axiIdxL are selected on whether there is a `l for that axiIdx in it's hyperPlane ; bolA := OR (keyL EACHLEFT EACHRIGHT = subTblDat) ; IF (NOR bolA) THEN slcAxiLL := '[' ; FOR i WITH valencer DO bol := bolA@i ; slcAxiL := null ; FOR j WITH (tell tblDatShp@i) DO bolAxi := IF bolAxi THEN slcAxiL := link slcAxiL (string j) ; ENDIF ; ENDFOR ; slcAxiLL := link slcAxiLL slcAxiL ', ' ; ENDIF ; slcAxiLL := (2 dropright slcAxiLL) link ']' ; subTblDat := execute link 'subTblDat|' slcAxiLL ; subTbl := subTitLL subTblDat ; ENDIF ; } # faultL #? ?tblKeyColGetItm01 tbl_keyColTit_keyItm_colTitL_get_itmLL keyColTit doesn"t exist #? ?tblKeyColGetItm02 tbl_keyColTit_keyItm_colTitL_get_itmLL colTitL not found #? ?tblKeyColGetItm03 tbl_keyColTit_keyItm_colTitL_get_itmLL rowVal not found IF flag_debug THEN write 'loading tbl_keyTitLL_keyItmL_get_sub' ; ENDIF ; #] tbl_keyTitLL_keyItmL_get_sub IS OP tbl keyTitLL keyItmL - #] returns a subTbl showing only hyperPlanes for keyItmL hits #] assumes that args are consistent - use _chk to ensure this!! #] make sure that single-arg [keyTitLL, keyItmL] are solitaries (protect [str, ary, list, etc] titles) # 08Sep2022 initial from 2 other optrs # easy to use with boolean-valued cols that can contain other vals # [axi, idx, adr] = [axis, index, address], see 'QNial userWkspSymbol header.ndf' for description # IF flag_break THEN BREAK ; ENDIF ; tbl_keyTitLL_keyItmL_get_sub IS OP tbl keyTitLL keyItmL { LOCAL adr bolAdrL faultL itmLL slc subTbl titLL keyAdrL keyIdx keyIdxLL keyTitAdrLL keyTitIdxLL tblDat tblDatShp ; % ; tblDat titLL := tbl ; keyAdrL := tblDat EACHRIGHT ary_itm_find_idxL keyItmL ; % ; faultL := null ; IF (ary_hasFaults keyAdrL) THEN faultL := link faultL (fault '?tblKeyTitItmGetSub01') ; ELSE % should return a list of intL - only one match for each keyTit ; keyTitAdrLL := titLL EACHBOTH EACHRIGHT ary_itm_find_idxL keyTitLL ; % eg ((0 1 2) (45 99 2) (45 1 2)) -> ((0 45) (1 99) 2), note adr -> idx change ; keyTitIdxLL := adrLL_make_idxLL keyTitAdrLL ; % keyNoTitIdxLL := ??? ; % ; % select ONLY keyAdrL that are related to keyTitLL ; bolAdrL := null ; FOR adr WITH keyAdrL DO bolAdrL := link bolAdrL (EACH OR (adr EACHBOTH EACHLEFT in keyTitIdxLL)) ; ENDFOR ; keyAdrL := bolAdrL subList keyAdrL ; % ; % hard part - select keyTitLL hyperPlanes for retention ; keyIdxLL := adrLL_make_idxLL keyAdrL ; slc := ary_idxLL_make_slc tblDat keyIdxLL ; tblDat := execute link 'tblDat|' slc ; titLL := keyIdxLL EACHBOTH EACHLEFT pick titLL ; % 08Sep2022 ??? ; subTbl := tblDat titLL ; ENDIF ; % ; IF (ary_hasFaults faultL) THEN faultL ELSE subTbl ENDIF } # faultL #? ?tblKeyTitItmGetSub01 tbl_keyColTit_keyItm_colTitL_get_itmLL key not found (ary_hasFaults keyAdrL) # [colTitL, tbl] are [nonlocal, permanent] - # create copies of this optr for [tbl, colTitL][0, 1, 2, 3, ...] # OR reassign [colTitL, tbl], and just use this one optr # oversimplified QNial roster databases, really just a tbl #08********08 #] 07Sep2022 revamp tbl optrs to >2D +-----+ olde code IF flag_debug THEN write 'loading tbl_rowTitL_key_get_subTbl' ; ENDIF ; # tbl_rowTitL_key_get_subTbl IS OP tbl rowTitL key - returns subTbl of cols with key in rowTitL (OR) # if key = null, just selects rowTitL, all cols included # remember : [row, col}TitL are LISTS, if only one returned it is in form of [[row, col]Tit] # can't use characters as titles! use one-char phrases instead # easy to use with boolean-valued cols that can contain other vals # use successive tbl_[col, row]TitL_key_get_subTbl for multi-selections [inefficient, robust, flexible] # ordering of dataRows in subTbl depends on rowTitL # retains colTitles (row 0) # 06Sep2022 initial # IF flag_break THEN BREAK ; ENDIF ; tbl_rowTitL_key_get_subTbl IS OP tbl rowTitL key { LOCAL subTbl rowTitAllL colTitAllL tblDat rowIdxL ; % ; rowTitAllL colTitAllL tblDat := tbl ; rowIdxL := rowTitAllL EACHRIGHT ary_itm_find_idxL rowTitL ; IF (issingle rowIdxL) THEN rowIdxL := first rowIdxL ; ENDIF ; subTbl := null ; IF (ary_hasFaults rowIdxL) THEN subTbl := link subTbl (fault '?tblRowtitkeyGetSub1') ; ELSE IF (= null key) THEN colIdxL := tell (gage shape colTitAllL) ; ELSE colBolL := EACH OR (cols (key EACHRIGHT = tblDat|[rowIdxL, ])) ; colIdxL := colBolL subList subList (tell (gage shape colTitAllL)) ; ENDIF ; subTbl := rowTitAllL colTitAllL|[colIdxL] tblDat|[, colIdxL] ; ENDIF ; % ; subTbl } # fault list : #? ?tblRowtitkeyGetSub1 tbl_rowTitL_key_get_subTbl rolTitL none in colTitAllL was row version IF flag_debug THEN write 'loading tbl_titLL_key_get_sub' ; ENDIF ; # tbl_titLL_key_get_sub IS OP tbl colIdx key - returns a subTbl of key-selected rows # as [row, col]TitL will be more common, this simply calls tbl_[row, col]TitL_key_get_subTbl # retains colTitles (row 0) # 12Oct2021 initial # should generalise to regExpr or something ... tbl_titLL_key_get_sub IS OP tbl colIdxL key { LOCAL rowTitAllL colTitAllL tblDat colTitL ; rowTitAllL colTitAllL tblDat := tbl ; colTitL := colIdxL EACHLEFT pick colTitAllL ; tbl_titLL_key_get_sub tbl colTitL key } # testTbl := 5 4 reshape 11 12 13 14 21 "hit 23 24 31 "hit 33 34 41 42 43 44 51 "hit 53 54 # tbl_titLL_key_get_sub testTbl 1 "hit # tbl_titLL_key_get_sub testTbl 1 42 IF flag_debug THEN write 'loading tbl_get_colTitL' ; ENDIF ; # tbl_get_colTitL IS 1 pick - trivial expression that "hides" structure, allows flexibility # 05Sep2022 initial tbl_get_colTitL IS 1 pick # tbl_get_colTitL mktTblTst # loaddefs link d_Qtest 'arrays/arrays- testOptrs.ndf' IF flag_debug THEN write 'loading tbl_rowTitL_key_get_subTbl_testSimple' ; ENDIF ; # tbl_rowTitL_key_get_subTbl_testSimple := - returns subTbl of cols with key in rowTitL (OR) # if key = null, just selects rowTitL # 06Sep2022 initial # tstOptr "tbl_rowTitL_key_get_subTbl_testSimple tbl_rowTitL_key_get_subTbl_testSimple := "tstSimple 3 "d_arrayTests ( ) #08********08 #] 06Sep2022 [fix, consolidate] tbl optrs 'QNial test optrs.ndf' hasn't been working for a couple of months just test some examples from 'arrays/arrays- testOptrs.ndf' without going the formal way +-----+ olde code IF flag_debug THEN write 'loading tbl_colTitL_get_subTbl' ; ENDIF ; # tbl_colTitL_get_subTbl IS OP tbl colTitL - returns subTbl of colTitL specified cols # ordering of dataCols in subTbl depends on colTitL # retains rowTitles (col 0) # 27Aug2022 initial # example usage : symName_dateRangeL UnixDateToYrfracL := EACH rest cols # tbl_colTitL_get_subTbl tbl ("symName_dateRangeL "UnixDateToYrfracL) ; # IF flag_break THEN BREAK ; ENDIF ; tbl_colTitL_get_subTbl IS OP tbl colTitL { LOCAL rowTitAllL colTitAllL tblDat colIdx subTbl rowIdxL ; % ; rowTitAllL colTitAllL tblDat := tbl ; colIdxL := link (colTitAllL EACHRIGHT ary_itm_find_idxL colTitL) ; subTbl := null ; IF (ary_hasFaults colIdxL) THEN subTbl := link subTbl (fault '?tblColtitGetSub1') ; ELSE IF (= null colIdxL) THEN subTbl := link subTbl (fault '?tblColtitGetSub2') ; ELSE subTbl := rowTitAllL colTitAllL|[colIdxL] tblDat|[, colIdxL] ; ENDIF ; ENDIF ; % ; subTbl } # fault list : #? ?tblColtitGetSub1 tbl_colTitL_val_get_subTbl colTitL faults #? ?tblColtitGetSub2 tbl_colTitL_val_get_subTbl colTitL none in colTitAllL (= null colIdxL) # ary_itm_find_idxL (tell 20) (3 8 13 17) # (tell 20) EACHRIGHT ary_itm_find_idxL (3 8 13 17) IF flag_debug THEN write 'loading tbl_rowTitL_get_subTbl' ; ENDIF ; # tbl_rowTitL_get_subTbl IS OP tbl rowTitL - returns subTbl of rowTitL specified rows # ordering of dataRows in subTbl depends on rowTitL # retains colTitles (row 0) # 27Aug2022 initial # IF flag_break THEN BREAK ; ENDIF ; tbl_rowTitL_get_subTbl IS OP tbl rowTitL { LOCAL subTbl rowTitAllL colTitAllL tblDat rowIdxL ; % ; rowTitAllL colTitAllL tblDat := tbl ; rowIdxL := link (rowTitAllL EACHRIGHT ary_itm_find_idxL rowTitL) ; subTbl := null ; IF (ary_hasFaults rowIdxL) THEN subTbl := link subTbl (fault '?tblRowtitGetSub1') ; ELSE IF (= null rowIdxL) THEN subTbl := link subTbl (fault '?tblRowtitGetSub2') ; ELSE subTbl := rowTitAllL|[rowIdxL] colTitAllL tblDat|[rowIdxL, ] ; ENDIF ; ENDIF ; % ; subTbl } # fault list : #? ?tblRowtitGetSub1 tbl_rowTitL_get_subTbl rolTitL faults #? ?tblRowtitGetSub2 tbl_rowTitL_get_subTbl rolTitL none in rowTitAllL (= null rowIdxL) # loaddefs link d_Qtest 'arrays/arrays- testOptrs.ndf' IF flag_debug THEN write 'loading tbl_colTitL_get_subTbl_testSpecial' ; ENDIF ; # tbl_colTitL_get_subTbl_testSpecial IS - returns subTbl of specified cols (can reorder cols too) # 30Aug2022 initial # tstOptr "tbl_colTitL_get_subTbl_testSpecial tbl_colTitL_get_subTbl_testSpecial IS { LOCAL calcL resStdL optr plog colTitL ; NONLOCAL mktTblTst mktTblcolTitLTst ; resStdL := mktTblColTitTst ; colTitL := "constantL "slopeL "relStdErrL "relStdDevL ; calcL := tbl_colTitL_get_subTbl mktTblTst colTitL ; % ; optr := "tbl_colTitL_get_subTbl_testSpecial ; tstSpecial optr calcL resStdL 1 } # loaddefs link d_Qtest 'arrays/arrays- testOptrs.ndf' IF flag_debug THEN write 'loading tbl_rowTitL_get_subTbl_testSpecial' ; ENDIF ; # tbl_rowTitL_get_subTbl_testSpecial IS - returns subTbl of specified rows (can reorder rows too) # 30Aug2022 initial # tstOptr "tbl_rowTitL_get_subTbl_testSpecial tbl_rowTitL_get_subTbl_testSpecial IS { LOCAL calcL resStdL optr rowTitL ; NONLOCAL mktTblTst mktTblRowTitTst ; rowTitL := 'SP500 1872-1940 TV,yahoo,article' 'GOLD Jan1834-Aug2022 TradingView' ; calcL := tbl_rowTitL_get_subTbl mktTblTst rowTitL ; resStdL := mktTblRowTitTst ; % ; optr := "tbl_rowTitL_get_subTbl_testSpecial ; tstSpecial optr calcL resStdL 1 } IF flag_debug THEN write 'loading tbl_bolColTit_get_subTbl' ; ENDIF ; # tbl_bolColTit_get_subTbl IS OP tbl bolColTit - returns subTbl of bolColTit-selected rows # [row, col] titles must be unique! # retains colTitles (row 0) # 26Aug2022 initial # assumes [row, col]TitL members are unique tbl_bolColTit_get_subTbl IS OP tbl bolColTit { LOCAL subTbl rowTitAllL colTitAllL tblDat colIdx rowIdxL ; % ; rowTitAllL colTitAllL tblDat := tbl ; colIdx := ary_itm_find_idxL colTitAllL bolColTit ; subTbl := null ; IF (ary_hasFaults colIdx) THEN subTbl := link subTbl colIdx (fault '?tblBolColGetsub1') ; ELSE rowIdxL := tblDat|[, colIdx] sublist (tell first (gage shape tblDat)) ; IF (ary_hasFaults rowIdxL) THEN subTbl := link subTbl rowIdxL (fault '?tblBolColGetsub2') ; ELSE subTbl := rowTitAllL|[rowIdxL] colTitAllL tblDat|[rowIdxL,] ; ENDIF ; ENDIF ; % ; subTbl } # fault list #? ?tblBolColGetsub1 tbl_bolColTit_get_subTbl bolColTit not found (ary_hasFaults colIdx) #? ?tblBolColGetsub2 tbl_bolColTit_get_subTbl rowIdxL problem (ary_hasFaults rowIdxL) IF flag_debug THEN write 'loading tbl_bolRowTit_get_subTbl' ; ENDIF ; # tbl_bolRowTit_get_subTbl IS OP tbl bolRowTit - returns subTbl of bolRowTit-selected cols # bolRowTit must be unique! # 27Aug2022 initial # retains rowTitles (col 0) tbl_bolRowTit_get_subTbl IS OP tbl bolRowTit { LOCAL subTbl rowTitAllL colTitAllL tblDat rowIdx colIdxL ; % ; rowTitAllL colTitAllL tblDat := tbl ; rowIdx := ary_itm_find_idxL rowTitAllL bolRowTit ; subTbl := null ; IF flag_break THEN BREAK ; ENDIF ; IF (ary_hasFaults rowIdx) THEN subTbl := link subTbl rowIdx (fault '?tblBolColGetsub1') ; ELSE colIdxL := tblDat|[rowIdx, ] sublist (tell second (gage shape tblDat)) ; IF (ary_hasFaults colIdxL) THEN subTbl := link subTbl colIdxL (fault '?tblBolRowGetsub2') ; ELSE subTbl := rowTitAllL colTitAllL|[colIdxL] tblDat|[,colIdxL] ; ENDIF ; ENDIF ; % ; subTbl } # fault list #? ?tblBolRowGetsub1 tbl_bolRowTit_get_subTbl bolRowTit not found (ary_hasFaults rowIdx) #? ?tblBolRowGetsub2 tbl_bolRowTit_get_subTbl colIdxL problem (ary_hasFaults colIdxL) # loaddefs link d_Qtest 'arrays/arrays- testOptrs.ndf' IF flag_debug THEN write 'loading tbl_bolColTit_get_subTbl_testSpecial' ; ENDIF ; # tbl_bolColTit_get_subTbl_testSpecial IS - returns subTbl of bolColTit-selected rows # 30Aug2022 initial # tstOptr "tbl_bolColTit_get_subTbl_testSpecial tbl_bolColTit_get_subTbl_testSpecial IS { LOCAL calcL resStdL optr ; NONLOCAL mktTblTst mktTblBolColTitTst ; calcL := tbl_bolColTit_get_subTbl mktTblTst "bolUSMix ; resStdL := mktTblBolColTitTst ; % ; optr := "tbl_bolColTit_get_subTbl_testSpecial ; tstSpecial optr calcL resStdL 1 } # loaddefs link d_Qtest 'arrays/arrays- testOptrs.ndf' IF flag_debug THEN write 'loading tbl_bolRowTit_get_subTbl_testSpecial' ; ENDIF ; # tbl_bolRowTit_get_subTbl_testSpecial IS - returns subTbl of bolRowTit-selected Rows # 30Aug2022 initial # tstOptr "tbl_bolRowTit_get_subTbl_testSpecial tbl_bolRowTit_get_subTbl_testSpecial IS { LOCAL calcL resStdL optr ; NONLOCAL mktTblTrp mktTblBolRowTitTrp ; calcL := tbl_bolRowTit_get_subTbl mktTblTrp "bolStockIntl ; resStdL := mktTblBolRowTitTrp ; % ; optr := "tbl_bolRowTit_get_subTbl_testSpecial ; tstSpecial optr calcL resStdL 1 } #08********08 #] 01Sep2022 redefine tbl = rowTitL colTitL tblDat, revamp code ary_allItmsUniqueBol +-----+ olde code # loaddefs link d_Qtest 'arrays/arrays- testOptrs.ndf' IF flag_debug THEN write 'loading tbl_colTitL_add_testSpecial' ; ENDIF ; # tbl_colTitL_add_testSpecial IS # 11Nov2020 initial, 27Aug2022 need fault check # tbl_colTitL_add_testSpecial tbl_colTitL_add_testSpecial IS { LOCAL calcL resStdL optr ; NONLOCAL mktColTitLTst mktDataTst mktTblNoRowTitsTst ; calcL := tbl_colTitL_add mktDataTst (rest mktColTitLTst) ; resStdL := mktTblNoRowTitsTst ; % ; optr := "tbl_colTitL_add_testSpecial ; tstSpecial optr calcL resStdL 1 } # loaddefs link d_Qtest 'arrays/arrays- testOptrs.ndf' IF flag_debug THEN write 'loading tbl_rowTitL_add_testSpecial' ; ENDIF ; # tbl_rowTitL_add_testSpecial IS # 30Aug2022 initial - stupid circular test! but it does test several optrs tbl_rowTitL_add_testSpecial IS { LOCAL calcL resStdL optr ; NONLOCAL mktColTitLTst mktDataTst ; calcL := tbl_rowTitL_add mktDataTst (rest mktRowTitLTst) ; resStdL := mktTblNoColTitsTst ; % ; optr := "tbl_rowTitL_add_testSpecial ; tstSpecial optr calcL resStdL 1 } IF flag_debug THEN write 'loading tbl_colTitL_remove_testSpecial' ; ENDIF ; # loaddefs link d_Qtest 'arrays/arrays- testOptrs.ndf' # tbl_colTitL_remove_testSpecial IS - remove column titles from tbl # 30Aug2022 initial # also for test, see link d_Qtest 'PuetzUWS- tests.ndf' tbl_colTitL_remove_testSpecial IS { LOCAL calcL resStdL optr ; NONLOCAL mktTblNoColTitsTst mktTblTst ; calcL := tbl_colTitL_remove mktTblTst ; resStdL := mktTblNoColTitsTst ; % ; optr := "tbl_colTitL_remove_testSpecial ; tstSpecial optr calcL resStdL 1 } # loaddefs link d_Qtest 'arrays/arrays- testOptrs.ndf' IF flag_debug THEN write 'loading tbl_rowTitL_remove_testSpecial' ; ENDIF ; # tbl_rowTitL_remove_testSpecial IS - remove row titles from tbl # 30Aug2022 initial tbl_rowTitL_remove_testSpecial IS { LOCAL calcL resStdL optr ; NONLOCAL mktTblNoRowTitsTst mktTblTst ; calcL := tbl_rowTitL_remove mktTblTst ; resStdL := mktTblNoRowTitsTst ; % ; optr := "tbl_rowTitL_remove_testSpecial ; tstSpecial optr calcL resStdL 1 } # loaddefs link d_Qtest 'arrays/arrays- testOptrs.ndf' IF flag_debug THEN write 'loading tbl_rowColTitLP_remove_testSpecial' ; ENDIF ; # tbl_rowColTitLP_remove_testSpecial IS - remove both [row, column] titles from tbl # this is the SAME as extracting the "data" part of the tbl # 30Aug2022 initial tbl_rowColTitLP_remove_testSpecial IS { LOCAL calcL resStdL optr ; NONLOCAL mktDataTst mktTblTst ; calcL := tbl_rowColTitLP_remove mktTblTst ; resStdL := mktDataTst ; % ; optr := "tbl_rowColTitLP_remove_testSpecial ; tstSpecial optr calcL resStdL 1 } IF flag_debug THEN write 'loading tbl_colTitL_remove ' ; ENDIF ; # tbl_colTitL_remove IS OP tbl - remove column titles from tbl # ?date? initial # for test, see link d_Qtest 'PuetzUWS- tests.ndf' tbl_colTitL_remove IS OP tbl { tbl|[rest tell (first (gage shape tbl)), ] } # loaddefs link d_Qndfs 'array, tbl.ndf' IF flag_debug THEN write 'loading tbl_rowTitL_remove ' ; ENDIF ; # tbl_rowTitL_remove IS OP tbl - remove row titles from tbl # ?date? initial tbl_rowTitL_remove IS OP tbl { tbl|[, rest tell (second (gage shape tbl))] } IF flag_debug THEN write 'loading tbl_rowColTitLP_remove ' ; ENDIF ; # tbl_rowColTitLP_remove IS OP tbl - remove both [row, column] titles from tbl # this is the SAME as extracting the "data" part of the tbl # ?date? initial tbl_rowColTitLP_remove IS OP tbl { LOCAL n_rows n_cols ; n_rows n_cols := gage shape tbl ; tbl|[rest tell n_rows, rest tell n_cols] } IF flag_debug THEN write 'loading tbl_get_rowTitL_colTitL_data' ; ENDIF ; # tbl_get_rowTitL_colTitL_data IS OP tbl - split tbl into 3 parts # this makes it easy to use [row, col] titles to select from data part, & visa versa # 26Aug2022 initial tbl_get_rowTitL_colTitL_data IS OP tbl { LOCAL n_rows n_cols rowTitAllL colTitAllL data ; n_rows n_cols := gage shape tbl ; rowTitAllL := tbl|[,0] ; colTitAllL := tbl|[0,] ; data := tbl|[rest tell n_rows, rest tell n_cols] ; rowTitAllL colTitAllL data } # loaddefs link d_Qtest 'arrays/arrays- testOptrs.ndf' IF flag_debug THEN write 'loading tbl_get_rowTitL_colTitL_data_testSpecial' ; ENDIF ; # tbl_get_rowTitL_colTitL_data_testSpecial IS - split tbl into 3 parts # this makes it easy to use [row, col] titles to select from data part, & visa versa # 30Aug2022 initial tbl_get_rowTitL_colTitL_data_testSpecial IS { LOCAL calcL resStdL optr rowTitL colTitL ; NONLOCAL mktRowTitLTst mktColTitLTst mktDataTst mktTblTst ; rowTitL colTitL tbl := tbl_get_rowTitL_colTitL_data mktTblTst ; calcL := rowTitL colTitL tbl ; resStdL := mktRowTitLTst mktColTitLTst mktDataTst ; % ; optr := "tbl_get_rowTitL_colTitL_data_testSpecial ; tstSpecial optr calcL resStdL 1 } "tbl_colTitL_add_testSpecial "tbl_rowTitL_add_testSpecial "tbl_colTitL_remove_testSpecial "tbl_rowTitL_remove_testSpecial "tbl_rowColTitLP_remove_testSpecial "tbl_get_rowTitL_colTitL_data_testSpecial #08********08 #] 24Aug2022 fix 'market symbol [dataProcess, regress].ndf' +-----+ 26Aug2022 continue... moved symbol_xPL_plotSLRegress & included it in symbol_xP_SLRegress loading symbol_xPL_plotSLRegress ?undefined identifier: DOALL_CALC_TIMEFRACLL_PRICEFRACLLL ; DOALL_CREATE_PSUBS <***> SYMNEW >> DOALL_CREATE_PSUBS is for detrend graph ONLY, should be symbol_xPL_plotSLRegress revamp symbol_xPL_plotSLRegress IS OP symbol yrPL stop at revampint pRegressionALL_find_statsMinMaxALL (tired) +-----+ 27Aug2022 continue... 31Aug2022 hacking fixes for [tbl, Puetz]s - renaming optrs, change args +-----+ olde code FOR yrP WITH yrPL DO yrStrt yrEnd := yrP ; oldformat := setformat '%7.2f' ; strLo strHi := EACH string yrP ; setformat oldformat ; symNew := link preName ' ' symShort ' ' strLo '-' strHi ; pPlt := link d_work symNew ' ' strUWS '.plt' ; pPng := link d_work symNew ' ' strUWS '.png' ; % ASSUMES that symbol_xP_SLRegress is calling symbol_xPL_plotSLRegress ; % OR that symbol_xP_SLRegress has run a series yrPL ; symbol_xP_SLRegress symbol ; % refresh timeFracLL priceFracLLL ; timeFracLL priceFracLLL := doAll_calc_timeFracLL_priceFracLLL ; doAll_create_pSubs symNew timeFracLL priceFracLLL ; host link 'gnuPlot "' pPlt '" ' ; ENDFOR ; resFltL # olde code LOCAL cmdDetrend '|;s||' strDateRange % cmdSLRegressPlot... debug takeout ; FOR yrP WITH yrPL DO yrStrt yrEnd := yrP ; strLo strHi := EACH string yrP ; strDateRange := link strLo '-' strHi ; pPlt := link d_work pltPreName ' ' strDateRange ' ' strUWS '.plt' ; pPng := link d_work pltPreName ' ' strDateRange ' ' strUWS '.plt' ; # 14Aug2022 USELESS! gnulot does this automatically, just pick the xRange! # NYET! NOW CHANGED!!! # then run fit_SLRgress_TradingView via # revampALL_CsvToYrFrac_unixDate_SLregress as shown below pPlt := link d_work pltPreName ' ' symbol ' ' strUWS '.plt' ; pPng := link d_work pltPreName ' ' symbol ' ' strUWS '.png' ; # symbol_regressPlot_pPng symbol ; revampONE_CsvToYrFrac_unixDate_SLregress # % crunch it ; revampALL_regressPlot ; doAll_calc_timeFracLL_priceFracLLL ; doAll_create_pSubs timeFracLL priceFracLLL ; pTmplt_pSubs_create_pPlt ; host link 'gnuPlot "' pPlt '" ' ; # # 27Aug2022 toying with another defn - but forgot that this is for a SIGLE data series IF flag_debug THEN write 'loading pRegressionALL_find_rsdMinMax' ; ENDIF ; # pRegressionALL_find_statsMinMaxALL IS OP tbl keyColPhr - for ACTIVE symName_dateRangeL # active : bolIncludeL = l # returns : constant slope relStdDevObs relStdDevTrn # xObsMin xObsMax yObsMin yObsMax yDetrendMin yDetrendMax # IF flag_break THEN BREAK ; ENDIF ; pRegressionALL_find_statsMinMaxALL IS OP tbl bolIncludePhr { LOCAL xMin xMax statsTmp symDate ; NONLOCAL ???? ; % ; xMin xMax := tbl_keyColPhr_keyItm_colPhrL_get_itmLL tbl keyColPhr l ("xStrtL "xEndL) ; xMin := min ; xMax := max ; shapeStats := 11 ; statsMinALL := shapeStats reshape 1e10 ; statsMinALL@0 := 'statsMinALL' ; statsMaxALL := shapeStats reshape -1e10 ; statsMaxALL@0 := 'statsMaxALL' ; FOR symDate WITH symName_dateRangeL DO statsTmp := pStatLog_symDate_find_symDateStats symDate ; % write statsTmp ; FOR i WITH (rest tell shapeStats) DO IF (statsTmp@i < statsMinALL@i) THEN statsMinALL@i := statsTmp@i ; ENDIF ; IF (statsTmp@i > statsMaxALL@i) THEN statsMaxALL@i := statsTmp@i ; ENDIF ; ENDFOR ; ENDFOR ; % ; statsMinALL statsMaxALL } IF flag_debug THEN write 'loading tbl_to_colTits_tblData' ; ENDIF ; #] tbl_to_colTits_tblData IS OP tbl - res ipsa loqitor # 13Aug2020 initial tbl_to_colTits_tblData IS OP tbl { LOCAL colTits n_cols n_rows subCols subTemp ; n_rows n_cols := gage shape tbl ; colTits := tbl|[0,] ; tblData := tbl|[rest tell n_rows,] ; colTits tblData } # olde code symName_dateRangeL bolUnixL constantL slopeL relStdErrL relStdDevL xObsMinL xObsMaxL yObsMinL yObsMinL yDetrendMinL yDetrendMaxL xTrnPhrL yTrnPhrL xTicsL yRangeL yTicsL yLabelL := cols tbl_remove_colTitles mktSymTbl ; colTitleL := mktSymTbl|[0,] ; IF flag_debug THEN write 'loading pRegressionALL_find_statsMinMaxALL' ; ENDIF ; # pRegressionALL_find_statsMinMaxALL IS OP symName_dateRangeL - for all symName_dateRangeL # returns : constant slope relStdDevObs relStdDevTrn # xObsMin xObsMax yObsMin yObsMax yDetrendMin yDetrendMax # IF flag_break THEN BREAK ; ENDIF ; pRegressionALL_find_statsMinMaxALL IS OP symName_dateRangeL { LOCAL statsMinALL statsMaxALL statsTmp symDate ; NONLOCAL pStatLog ; % ; shapeStats := 11 ; statsMinALL := shapeStats reshape 1e10 ; statsMinALL@0 := 'statsMinALL' ; statsMaxALL := shapeStats reshape -1e10 ; statsMaxALL@0 := 'statsMaxALL' ; FOR symDate WITH symName_dateRangeL DO statsTmp := pStatLog_symDate_find_symDateStats symDate ; % write statsTmp ; FOR i WITH (rest tell shapeStats) DO IF (statsTmp@i < statsMinALL@i) THEN statsMinALL@i := statsTmp@i ; ENDIF ; IF (statsTmp@i > statsMaxALL@i) THEN statsMaxALL@i := statsTmp@i ; ENDIF ; ENDFOR ; ENDFOR ; % ; statsMinALL statsMaxALL } # test # pRegressionALL_find_statsMinMaxALL #08********08 #] 06Aug2022 create_pPng pPlt pPng - pPlt curves, text too long, chokes gnuplot error msgs : +-----+ "/home/bill/SG6/web/economics, markets/SP500/PuetzUWS intnl stocks/detrend StockMkt Indices 1871-2022 PuetzUWS2011.plt", line 140: warning: truncating variable name that is too long "/home/bill/SG6/web/economics, markets/SP500/PuetzUWS intnl stocks/detrend StockMkt Indices 1871-2022 PuetzUWS2011.plt", line 140: invalid command +-----+ copy curves to p_temp, then run? wait - gnuplot was always handling this easily up until right now. What gives? I put an empty line between '# PuetzUWS curvePlots (auto-added by QNial)' 'plot \' >> works much better, but still error : qnial> create_pPng pPlt pPng "/home/bill/SG6/web/economics, markets/SP500/PuetzUWS intnl stocks/detrend StockMkt Indices 1871-2022 PuetzUWS2011.plt", line 0: Cannot open script file '/home/bill/SG6/web/economics, markets/SP500/PuetzUWS intnl stocks/detrend StockMkt Indices 1871 qnial> pTmplt_pSubs_create_pPlt qnial> create_pPng pPlt pPng "/home/bill/SG6/web/economics, markets/SP500/PuetzUWS intnl stocks/detrend StockMkt Indices 1871-2022 PuetzUWS2011.plt", line 0: Cannot open script file '/home/bill/SG6/web/economics, markets/SP500/PuetzUWS intnl stocks/detrend StockMkt Indices 1871 07Aug2022 Hah, hilarious - a problem was the limited shape of (upped below) : lwSpikeL := count 20 ; lwPriceL := reverse count 20 ; Redo stuff qnial> tSpikeLL_create_pSpikes tSpikeLL qnial> doAll_calc_tSpikeLL_fracLLL qnial> doAll_create_pSubs tSpikeLL fracLLL qnial> pTmplt_pSubs_create_pPlt #> +--+ #> warning!! pTmplt_pSubs_create_pPlt crashes, no pPlt #> before running create_pPng manually, edit pPlt : #> insert pCurves after #> insert 'plot \' just before curves, empty line after targetInsetPoint #> ', \' at end of the last curve must be manually removed!! #> +--+ >> done qnial> create_pPng pPlt pPng OK, it works EXCEPT the tSpikes, priceFracs - whole graph is grey! (too thick, too many?) #08********08 #] 06Aug2022 Manually fixed pPlt works nicely, insuffient [tSpikeL, fracLLL] depths +-----+ Redundant code (dangerous!) : 'stock index semi-log [regression, plot]s.ndf' pPlt_gnuplot_pPng change : +.....+ cmd := link 'cat "' d_work 'semi-log 0_gnuplot template.txt" | sed "s||' symbol '|;s||' (string constant) '|;s||' (string slope) '|;s||' (string relStdErr) '|;s||' (string relStdDev) '|" >"' d_work 'semi-log ' symbol '.plt"' ; % write cmdGnuplotTemplateInsertVariables ; host cmdGnuplotTemplateInsertVariables ; +.....+ To : +.....+ % write cmdGnuplotTemplateInsertVariables ; host cmdGnuplotTemplateInsertVariables ; +.....+ 'gnuplot template, subFiles [create, insert].ndf' pTmplt_pSubs_create_pPlt change : +.....+ cmd := link 'cat "' pTmplt '" | sed ' chr_apo 's||' terminal '|;s||' datafile_separator '|;s||' title '|;s|<ylabel>|' ylabel '|;s|<strUWS>|' strUWS '|;s|<yrStrt>|' (string yrStrt) '|;s|<yrEnd>|' (string yrEnd) '|;s|<xtics>|' (string xtics) '|;s|<yrange>|' yrange '|;s|<ytics>|' ytics '|;s|<strDateRange>|' strDateRange '|;s|<pPng>|' pPng '|;s|<colYr_colyDetr>|' colYr_colyDetr '|' chr_apo ' >"' pWrdChg '" ' ; % write cmd ; host cmd ; +.....+ To : +.....+ % write cmd ; host cmd ; +.....+ bye, qnial I changed 'QNial setup.ndf', now loaddefs : '220730 Puetz2011 detrended international indexes, gnuplot specs.ndf' What gives? many variables not assigned in '220730 Puetz2011 detrended international indexes, gnuplot specs.ndf' yrStrt yrEnd yrP yrZero xtics ytics yRange colYr_colyDetr cmdGnuplotTemplateInsertVariables qnial> yrStrt yrEnd yrP yrZero xtics ytics yRange ?no_value ?no_value ?no_value ?no_value ?no_value ?no_value ?no_value qnial> colYr_colyDetr ?undefined identifier: COLYR_COLYDETR <***> qnial> cmdGnuplotTemplateInsertVariables ?no_value >> WEIRD!!?? I was missig end apo : ytics := '-3, -2, -1, 0.0, 1, 2, 3, 4' ; >> OK now pTmplt_pSubs_create_pPlt link 'grep ' chr_apo ', \\' chr_apo ' "' pPlt '" ' ary_sub_get_slcLL 'test string, \' ', \' >> no workee lastCurveLine := last host_result link 'grep ' chr_apo ', \\' chr_apo ' "' pPlt '" ' >> nothing qnial> a := link 'grep ' chr_apo 'Puetz' chr_apo ' "' pPlt '" ' grep 'Puetz' "/home/bill/SG6/web/economics, markets/SP500/PuetzUWS intnl stocks/detrend StockMkt Indices 1871-2022 PuetzUWS2011.plt" qnial> host_result a +----------------------------------- |# '220730 Puetz2011 detrended international indexes, gnuplot specs.ndf'|# like '# PuetzUWS vertical timeSp +------------------------------------------------------------------------- ... >> works well qnial> lastCurveLine := last host_result link 'grep ' chr_apo ', ' chr_apo ' "' pPlt '" ' set label 16 "{/courier=8 UKX100 Apr1988-Jul2022 TradingView -25.5175 0.014559 1.22857 0.201597 }" at graph 0.01, 0.1 textcolor "grey70" >> that was easy, but the backslash is a killer qnial> host_result link 'grep ' chr_apo ', \\\' chr_apo ' "' pPlt '" ' +------------------------+ |grep: Trailing backslash| +------------------------+ >> hmm, maybe this is the line continuation expectation with bash? Use QNial, search for all lines with ', \' end a := pinn_read_strL pPlt >> pCurves missing - fix later, for now copy-paste into pPlt b := a EACHLEFT ary stuff to work! >> do manually +-----+ insuffient [tSpikeLL, fracLLL] depths 'PuetzUWS calc [time, price] fractals.ndf' rsdP_calc_idxAboveLBelowL maxPriceFracDepth replaced use of `3 for depth create maxPriceFracDepth to be defined in pDefn : '220730 Puetz2011 detrended international indexes, gnuplot specs.ndf' doAll_calc_tSpikeLL_fracLLL in 'gnuplot template, subFiles [create, insert].ndf' qnial> doAll_calc_tSpikeLL_fracLLL +------------------------------------------------------------------------------------------------------------- |+-----------------------------------------------------------------------+-----------------------+-------+---- ||1872.73 1891.73 1910.74 1929.75 1948.76 1967.77 1986.77 2005.78 2024.79|1872.73 1929.75 1986.77|1929.75|1929 |+-----------------------------------------------------------------------+-----------------------+-------+---- | | +------------------------------------------------------------------------------------------------------------- ------------+--------------------------------------------------------------------+ ---+-------+|+----------------------------------------------------+-------------+| .75|1929.75|||+-+----+----------+--------------------------------+|+-+---------+|| ---+-------+|||0|0 9.|0 3. 6. 9.|0 1. 2. 3. 4. 5. 6. 7. 8. 9. 11.|||0|0 -9. -3.||| ||+-+----+----------+--------------------------------+|+-+---------+|| |+----------------------------------------------------+-------------+| ------------+--------------------------------------------------------------------+ >> doesn't look deeper? looks the same as before? doAll_calc_tSpikeLL_fracLLL >> still no change set maxPriceFracDepth := 5 OK - that works, changed also in pDefn doAll_create_pSubs pTmplt_pSubs_create_pPlt >> WHAT? [tSpikeLL fracLLL] didn't change? >> actually, fracLLL seems OK when entered into command line, but not in : 'fracLLL_create_pPrices.txt' even though it was regenerated - must be another `3 somewhere? +--+ strUWS_yrZero_yrP_calc_tSpikeLL missing maxPriceFracDepth >> I hope this is fixed! >> nyet - I've simply addeed "upper tPeriods, this is more complex idxUWSL := (idx_UWSbase - 2) + tell (maxPriceFracDepth + 2) ; To idxUWSL := (idx_UWSbase - 2) - tell (maxPriceFracDepth + 2) ; strUWS_yrZero_yrP_calc_tSpikeLL strUWS yrZero yrP >> now it lists ALL tObsL!?!?! flag_break it idxUWSL := (idx_UWSbase + 2) - tell (maxPriceFracDepth + 2) ; >> too many sublevels! maxPriceFracDepth := 4 >> that looks good! +--+ qnial> doAll_calc_tSpikeLL_fracLLL >> oops, fracLLL is mixed with yRange or something >> no, OK as fracLLL has [above, below] still has old upper limit 0 3. 6. 9. >> nyet - problem with fracLLL calc qnial> rsdP_calc_idxAboveLBelowL rsdP +-----------+---+ |25 24 23 22|4 5| +-----------+---+ qnial> fracLLL +------------------------------------------------------------- ||0|0 9.|0 3. 6. 9.|0 1. 2. 3. 4. 5. 6. 7. 8. 9.|0 0.333333 0.666666 0.999999 1.33333 1.66667 2. 2.33333 2.666 |+-+----+----------+---------------------------- ~|~ -------------------------------------------------------------------------------------------------------------- -------------------------------------------------------------------------------------------------------------- 66 3. 3.33333 3.66666 4. 4.33333 4.66666 5. 5.33333 5.66666 5.99999 6.33333 6.66666 6.99999 7.33333 7.66666 7. -------------------------------------------------------------------------------------~|~ --------------------------------------+-------------+ -------------------------------------+|+-+---------+| 99999 8.33332 8.66666 8.99999 9.66666|||0|0 -9. -3.|| -------------------------------------+|+-+---------+| --------------------------------------+-------------+ >> why the 0 9.? qnial> rsdPuetzL -729. -243. -81. -27. -9. -3. -1. -0.333333 -0.111111 -0.037037 -0.0123457 -0.00411523 -0.00137174 -0.000457247 0. 0.000457247 0.00137174 0.00411523 0.0123457 0.037037 0.111111 0.333333 1. 3. 9. 81. 27. 243. 729. >> OK, it's doing what it is supposed to do Problem : (rsdBelowIdxL EACHLEFT <= (shapeRsdPuetzL - rsdAboveIdxL@2 - 1)) sublist rsdBelowIdxL ; ([0] (0 9.) (0 3. 6. 9.) (0 1. 2. 3. 4. 5. 6. 7. 8. 9.) (0 0.333333 0.666666 0.999999 1.33333 1.66667 2. 2.33333 2.66666 3. 3.33333 3.66666 4. 4.33333 4.66666 5. 5.33333 5.66666 5.99999 6.33333 6.66666 6.99999 7.33333 7.66666 7.99999 8.33332 8.66666 8.99999 9.66666)) ([0] (0 -9. -3.)) >> find out differences in each subList a := rest ([0] (0 9.) (0 3. 6. 9.) (0 1. 2. 3. 4. 5. 6. 7. 8. 9.) (0 0.333333 0.666666 0.999999 1.33333 1.66667 2. 2.33333 2.66666 3. 3.33333 3.66666 4. 4.33333 4.66666 5. 5.33333 5.66666 5.99999 6.33333 6.66666 6.99999 7.33333 7.66666 7.99999 8.33332 8.66666 8.99999 9.66666)) EACH (- [front, rest]) a qnial> EACH (- [front, rest]) a ([-9.] (-3. -3. -3.|-1. -1. -1. -1. -1. -1. -1. -1. -1.|-0.333333 -0.333333 -0.333333 -0.333331 -0.33334 -0.33333 -0.33333 -0.33333 -0.33334 -0.33333 -0.33333 -0.33334 -0.33333 -0.33333 -0.33334 -0.33333 -0.33333 -0.33333 -0.33334 -0.33333 -0.33333 -0.33334 -0.33333 -0.33333 -0.33333 -0.33334 -0.33333 -0.66667)) >> very problematic -0.66667 in last subList! Look at idxLs -->[nextv] shapeRsdPuetzL 29 >> ?Can't be - RsdPuetzL only has 25 itms?!?!?! >> must be reassigned elsewhere? >> oops - I didn't update idxs in definition in header $ find "$d_Qmkts" -maxdepth 4 -type f -name "*" | grep --invert-match "z_Old\|z_Archive" | tr \\n \\0 | xargs -0 -IFILE grep -i --with-filename --line-number "RsdPuetzL" "FILE" >> header file doesn't appear in find, why? - oops, capitalised, I added -i above >> OK, many times more hits ... rsdPuetzL -->[nextv] rsdBelowIdxL := rsdMinIdx + (tell maxPriceFracDepth) 4 5 6 7 -->[nextv] rsdAboveIdxL := rsdMaxIdx - (tell maxPriceFracDepth) 25 24 23 22 -->[nextv] shapeRsdPuetzL EACHRIGHT - rsdBelowIdxL 25 24 23 22 Try (EACH OR ((shapeRsdPuetzL EACHRIGHT - rsdBelowIdxL) EACHLEFT EACHRIGHT <= rsdAboveIdxL)) sublist rsdBelowIdxL missing one subL for rsdAboveIdxL???? >> NO - it's OK, rsdBelowIdxL returns are order : rsdAboveIdxL rsdBelowIdxL As midIdx is included, then set maxPriceFracDepth := 5? OK - but now too many rsdBelowIdxL I need to subtract 1 from maxPriceFracDepth? rsdP_calc_idxAboveLBelowL rsdP to (EACH OR ((shapeRsdPuetzL EACHRIGHT - rsdBelowIdxL) EACHLEFT EACHRIGHT >= rsdAboveIdxL)) sublist rsdBelowIdxL ; doAll_calc_tSpikeLL_fracLLL >> OK tSpikeLL is to deep many changes to get right strUWS_yrZero_yrP_calc_tSpikeLL strUWS yrZero yrP create variable maxTimeFracDepth, define in pDefn . Try again doAll_calc_tSpikeLL_fracLLL >> looks good! lq_Puetz - pGnuplot ont loaddefed? #08********08 #] 05Aug2022 rerun everything with new(old) yDetrend := (Y/yEstTrn - 1)/relStdDevTrn +-----+ DON'T redo : revampALL >> many errors # singleLinearRgression [estimate, residual, detrend]s ?shape >> must be tabTbl titles? >> tab[Partial, Sep] not regenerated, of course I am missing a step : overall "Normal actions" change : +.....+ download all market symbol data of interest revampCsvToTab IS - converts TradingView rawCsv dataSets to tab-separated manually add titleApos to pTabPartial !!?!!!! revampALL IS - from pUnixDate_to_pYrfrac to Plt_gnuplot_pPng +.....+ To : +.....+ download all market symbol data of interest revampCsvToTab IS - converts TradingView rawCsv dataSets to tab-separated manually add titleApos to pTabPartial !!?!!!! revampALL IS - from pUnixDate_to_pYrfrac to Plt_gnuplot_pPng +.....+ +-----+ # debug test : qnial> symName_dateRange := 'DAX40 Jan1970-Jul2022 TradingView' qnial> fit_SLRgress_TradingView >> result : # paramSLRgressL : # |symName |xTrnPhr|xUnTrnPhr|yTrnPhr|yUnTrnPhr|slope |constant| # +---------------------------------+-------+---------+-------+---------+---------# |DAX40 Jan1970-Jul2022 TradingView|pass |?noexpr |log |?noexpr |0.0325753 | -61.6481| # +---------------------------------+-------+---------+-------+---------+--------- >> unTransforms not working 'fit_linearRegress.ndf' SLRgress_model_test (perfect yesterday) : qnial> SLRgress_model_test >> perfect today as well, maybe symName changes? try : qnial> EACH trnOp_reverse "pass "log ?noexpr ?noexpr >> what???? trnOp globals not initiated? qnial> trnOpUnOpLL +------------------------------------------------+ |pass log exp sin cos pass unLog ln arcSin arcCos|pass unLog ln arcSin arcCos pass log exp sin cos| +------------------------------------------------ qnial> paramSLRgressL +-------------------+----+----+---+-----+-+-+ |SLRgress_model_test|pass|pass|log|unLog|2|5| +-------------------+----+----+---+-----+-+-+ >> OK, >> not paramL_setGlobal_paramSLRgressL as NOTHING calls it? >> how did paramSLRgressL get set? not in : '220730 Puetz2011 detrended international indexes, gnuplot specs.ndf' 'stock index semi-log [regression, plot]s.ndf' 'PuetzUWS calc [time, price] fractals.ndf' 'gnuplot template, subFiles [create, insert].ndf' $ find "$d_Qndfs" -type f -name "*.ndf" | grep --invert-match "z_Old" | grep --invert-match "z_Archive" | sort | tr \\n \\0 | xargs -0 -IFILE grep --with-filename --line-number 'paramSLRgressL' "FILE" | sed "s|$d_Qndfs||;s|:.*||" | sort -u fit_linearRegress.ndf >> wow! only appears in that one ndf file!!?? (no .txt) paramL_setGlobal_paramSLRgressL IS OP paramShortL { LOCAL paramShortL xTrnPhr xUnTrnPhr yTrnPhr yUnTrnPhr slope constant ; NONLOCAL paramSLRgressL ; IF (~= 5 (gage shape paramShortL)) THEN paramSLRgressL := fault '?psgSLRgress01' ; >> if ever called, this would set it $ find "$d_Qndfs" -type f -name "*.ndf" | grep --invert-match "z_Old" | grep --invert-match "z_Archive" | sort | tr \\n \\0 | xargs -0 -IFILE grep --with-filename --line-number 'paramL_setGlobal_paramSLRgressL' "FILE" | sed "s|$d_Qndfs||;s|:.*||" | sort -u fit_linearRegress.ndf >> still only the one ndf file! Nuts, fit_SLRgress : % [set, log] paramSLRgressL ; xUnTrnPhr yUnTrnPhr := EACH trnOp_reverse xTrnPhr yTrnPhr ; paramSLRgressL := symName xTrnPhr xUnTrnPhr yTrnPhr yUnTrnPhr slope constant ; 'stock index semi-log [regression, plot]s.ndf' fit_SLRgress_TradingView qnial> EACH trnOp_reverse "pass "log ?noexpr ?noexpr qnial> resFltL := ary_itm_find_idxL "pass (first trnOpUnOpLL) ?itmNotInAry >> idiot - args are reversed!!! qnial> ary_itm_find_idxL (first trnOpUnOpLL) "pass 0 5 >> works, but all trn must be unique! >> I had put them all in (dumb) NYET! not so dumb : I had only repeated "pass, now fixed +-----+ bye, $ qnial # debug test : qnial> symName_dateRange := 'DAX40 Jan1970-Jul2022 TradingView' qnial> fit_SLRgress_TradingView >> semi-log DAX still gives : # singleLinearRgression [estimate, residual, detrend]s ?shape >> hilarious - same problem Try again : qnial> symName_dateRange := 'DAX40 Jan1970-Jul2022 TradingView' qnial> fit_SLRgress_TradingView >> same ?shape problem Sheesh - looks like 'stock index semi-log [regression, plot]s.ndf' fit_SLRgress_TradingView change : +.....+ constant slope relStdErr relStdDev xObsMin xObsMax yObsMin yObsMin yDetrendMin yDetrendMax := fit_SLRgress symName_dateRange yrFrac openPrice "pass "log povr ; +.....+ To : +.....+ fit_SLRgress_TradingView symName constant slope relStdErr relStdDev xObsMin xObsMax yObsMin yObsMin yDetrendMin yDetrendMax := fit_SLRgress symName_dateRange yrFrac openPrice "pass "log povr ; +.....+ >> still get # singleLinearRgression [estimate, residual, detrend]s ?shape mismatch of #args fit_SLRgress IS OP symName xObs yObs xTrnPhr yTrnPhr povr constant slope relStdDevObs relStdDevTrn xObsMin xObsMax yObsMin yObsMax yDetrendMin yDetrendMax versus fit_SLRgress_TradingView symName constant slope relStdErr relStdDev xObsMin xObsMax yObsMin yObsMin yDetrendMin yDetrendMax := fit_SLRgress symName_dateRange yrFrac openPrice "pass "log povr ; >> take out symName!! pStatLog last entry 220805 14h20m14s results are in transformed coordinates (eg log) symName_dateRange constant slope relStdErr relStdDev xObsMin xObsMax yObsMin yObsMin yDetrendMin yDetrendMax DAX40 Jan1970-Jul2022 TradingView -61.6481 0.0325753 0.254876 0.0351671 ?no_value ?no_value ?no_value ?no_value ?no_value ?no_value >> not from the one I just did... redo fit_SLRgress_TradingView with flag_break qnial> fit_SLRgress_TradingView -->[nextv] yrFrac ?slice -->[nextv] tbl_raw ?slice >> interesting... -->[nextv] symName_dateRange ?no_value >> what? should work? # symName_dateRange := 'DAX40 Jan1970-Jul2022 TradingView' # pinn := link d_work 'yrFrac ' symName_dateRange '.txt' -->[nextv] tbl_remove_colTitles pinn_read_tabTable pinn ?slice >> could trailing \t be the problem? or exp format numbers like 7.56285e+10? (share volumes) -->[nextv] pinn / h o m e / b i l l / S G 6 / w e b / e c o n o m i c s , m a r k e t s / S P 5 0 0 / P u e t z U W S i n t n l s t o c k s / y r F r a c ?no_value . t x t >> Oh, that's the problem revampALL IS { LOCAL cmd ; NONLOCAL d_work symName_dateRange symName_dateRangeL UnixDateToYrfracL ; FOR symName_dateRange WITH symName_dateRangeL DO >> this defines symName_dateRange for perhaps the first time my checks di assign symName_dateRange redo - but also change fit_SLRgress_TradingView to take arg symName_dateRange >> probably only called by revampALL, as revampONE takes that arg qnial> fit_SLRgress_TradingView 'DAX40 Jan1970-Jul2022 TradingView' -->[nextv] -61.6481 0.0325753 0.254876 0.0351671 1970. 2022.5 378.2 15958.4 ?no_value ?no_value >> yDetrendMin yDetrendMax don't work in fit_SLRgress yDetrend := (Y / yEstTrn - 1) / relStdDevTrn ; put in flag_break loading fit_SLRgress ?expecting end of block: ; >> oops OK, rerun : 'semi-log DAX40 Jan1970-Jul2022 TradingView regression fit.txt' >> seems OK I converted many optrs in 'stock index semi-log [regression, plot]s.ndf' to taking symbol arg - more clear, can't lose track of symbol, run tests easily, reduce side-effects qnial> symbol := 'DAX40 Jan1970-Jul2022 TradingView' DAX40 Jan1970-Jul2022 TradingView qnial> fit_SLRgress_TradingView 'DAX40 Jan1970-Jul2022 TradingView' qnial> pPlt_gnuplot_pPng symbol >> OK, famous Now pPlt missing curves : OK paramTbl_create_pCurves.txt OK pTmplt_insert_pCurves.txt >> but it wasn't inserted into detrend StockMkt Indices 1871-2022 PuetzUWS2011.plt >> oops, it froze last time that I tried... qnial> pTmplt_pSubs_create_pPlt ^C >> froze again, what happend? pTmplt_pSubs_create_pPlt : pTmplt_add_pSub pCurvesTmp pPlotCmd pPlt insertLine ; OK insertLine := '# PuetzUWS curvePlots (auto-added by QNial)' ; OK pCurvesTmp OK pPlotCmd gnuplot template, plot command.txt OK pPlt detrend StockMkt Indices 1871-2022 PuetzUWS2011.plt '220730 Puetz2011 detrended international indexes, gnuplot specs.ndf' pltPreName := link 'detrend StockMkt Indices' ; qnial> pltPreName detrend StockMkt Indices >> OK pPlt := link d_work pltPreName ' ' strDateRange ' ' strUWS '.plt' ; qnial> pPlt /home/bill/SG6/web/economics, markets/SP500/PuetzUWS intnl stocks/detrend StockMkt Indices 1871-2022 PuetzUWS2011.plt >> OK So why does it "hang up" on the LAST step of pTmplt_add_pSub? >> idiot - I looked right at the filename - didn't notice that it had changed optr change : +.....+ pTmplt_add_pSub pCurvesTmp pPlotCmd pPlt insertLine ; +.....+ To : +.....+ pTmplt_add_pSub pCurves pPlotCmd pPlt insertLine ; +.....+ >> also - note that I am missing pth_type_exists checks! do manually (don't need to redo all) : pTmplt_add_pSub pCurves pPlotCmd pPlt '# PuetzUWS curvePlots (auto-added by QNial)' >> doesn't work - jut has pCurves now... run all of them : pTmplt_pSubs_create_pPlt >> still freezes, WHY? Weird, if I insert an emptey line in , just before the curves, I produces the correct pPlt, but still freezes : qnial> pTmplt_pSubs_create_pPlt ^C >> WHY? I am missing something here Anyways, reload everything and try : create_pPng pPlt pPng >> oops, have to fix pPlt first I manually fixed pPlt, create_pPng pPlt pPng runs OK, but NO PLOT!! >> arghh! this never ends pPlt missing in 2nd set terminal : fontscale 1.0 size 1400, 400 I added, now rerun (this wan't a problem before?!) : IDIOT! substitute failed (another change) in : set output "<pPlotPng>" I maually put in : detrend StockMkt Indices 1871-2022 PuetzUWS2011.png create_pPng pPlt pPng >> I worked!! finaly, after death by a thousand cuts (many self-imposed). BUT - where are the tSpikeLL & fracLLL? >> they appear in pPlt, PuetzUWS horizontal priceDetrend (auto-added by QNial) PuetzUWS vertical timeSpikes (auto-added by QNial) both seem OK? so why didn't they plot? actually,they DID! but the lines are far too faint to see curve colors are different from symbol chart... I replaced all greys by black for lines I changed the paramTbl colors >> MUCH more visible still no horizontals I need to go down 3 tFrac levels, not just 2? or does it without showing? . >> all tSpikes are showing, I have to go down 3 levels, not two (actually, 3 levels don't show - might be a threshold mistake) Colors are still mixed up - I don't understand NYET! they are fine, it is the DAX that goes back to 1970 +-----+ olde code # problematic ending won't work? # pCurvesTmp := link d_temp 'pTmplt_insert_pCurves.txt' # insertLine := '# PuetzUWS curvePlots (auto-added by QNial)' # pTmplt_add_pSub pCurvesTmp pPlotCmd pPlt insertLine % biases upward somehow? not -log(relStdDevTrn)? ; % yDetrend := log (Y / yEstTrn / relStdDevTrn) ; #08********08 #] 04Aug2022 detrend graph - problem with scaling, non-detrended index prices lots of stuff - create new optrs #08********08 #] 04Aug2022 Puetz intl stock mkt detrend - tie all code together, clean up was all my work today a waste of time? pPlt_gnuplot_pPng doesn't need quote-enclosed colTitles? >> Arrgghhh! Did I actually write? : doAll_calc_yrSpikeLL_fracLLL IS - regenerate [tSpikeLL,fracLLL] globals was this to pick up [yrSpikeLL, fracLLL] from a log file? or just re-run pTmplt_pSubs_create_pPlt? qnial> create_pPng set terminal pngcairo enhanced font courier,10 fontscale 1.0 size 1400, 400 ^ "/home/bill/SG6/web/economics, markets/SP500/PuetzUWS intnl stocks/detrend StockMkt Indices 1871-2022 PuetzUWS2011.plt", line 19: font: expecting string >> oops, forgot about this must be : courier,10 change pTmplt_pSubs_create_pPlt change : +.....+ cmd := link 'cat "' pTmplt '" | sed "s|<terminal>|' terminal '|;s|<datafile_separator>|' datafile_separator '|;s|<title>|' title '|;s|<ylabel>|' ylabel '|;s|<strUWS>|' strUWS '|;s|<yrStrt>|' (string yrStrt) '|;s|<yrEnd>|' (string yrEnd) '|;s|<xtics>|' (string xtics) '|;s|<yrange>|' yrange '|;s|<ytics>|' ytics '|;s|<strDateRange>|' strDateRange '|;s|<pPng>|' pPng '|" >"' pWrdChg '" ' ; +.....+ To : +.....+ cmd := link 'cat "' pTmplt '" | sed ' chr_apo 's|<terminal>|' terminal '|;s|<datafile_separator>|' datafile_separator '|;s|<title>|' title '|;s|<ylabel>|' ylabel '|;s|<strUWS>|' strUWS '|;s|<yrStrt>|' (string yrStrt) '|;s|<yrEnd>|' (string yrEnd) '|;s|<xtics>|' (string xtics) '|;s|<yrange>|' yrange '|;s|<ytics>|' ytics '|;s|<strDateRange>|' strDateRange '|;s|<pPng>|' pPng '|' chr_apo ' >"' pWrdChg '" ' ; +.....+ +-----+ olde code # aryInn_apoStrings_aryOut IS - wrap strings in aryInnTop in apos # IF flag_break THEN BREAK ; ENDIF ; aryInn_apoStrings_aryOut IS OP aryInn { IF (isstring aryInn) THEN link chr_apo aryInn chr_apo ELSE aryInn ENDIF } >> I already had this in strings.ndf : str_wrapWithApos IS OP str - convert str to string with apostrophes or if not a string return >> besides, it was WRONG (EACH needed) #08********08 #] 02Aug2022 doAll_calc_tSpikeLL_fracLLL 'PuetzUWS calc [time, price] fractals.ndf' qnial> doAll_calc_tSpikeLL_fracLLL ^C >> not working... qnial> strUWS yrZero yrP +------------+---------+---------+ |PuetzUWS2011|?no_value|?no_value| +------------+---------+---------+ >> forgot to initialize [yrZero, yrP] in : '220730 Puetz2011 detrended international indexes, gnuplot specs.ndf' nial> strUWS yrZero yrP +------------+-------+---------+ |PuetzUWS2011|1929.75|1860 2040| +------------+-------+---------+ qnial> doAll_calc_tSpikeLL_fracLLL +------------------------------------------------------------------------------------------------------------- |+-----------------------------------------------------------------------+-----------------------+-------+---- ||1872.73 1891.73 1910.74 1929.75 1948.76 1967.77 1986.77 2005.78 2024.79|1872.73 1929.75 1986.77|1929.75|1929 |+-----------------------------------------------------------------------+-----------------------+-------+---- +------------------------------------------------------------------------------------------------------------- ------------+--------------+ ---+-------+|?rsdCalcFrac01| .75|1929.75|| | ---+-------+| | ------------+--------------+ >> ?rsdCalcFrac01 #? ?rsdCalcFrac01 rsdAboveLBelowL_calc_fracLLL (rsdMin >= rsdMax) >> awesome -- error report really helped, mixup of ordering qnial> rsdP ?no_value qnial> rsdMin rsdMax ?undefined identifier: RSDMIN <***> RSDMAX >> these are used as args, but require rsdMax_to_lambdaIdx IS OP rsdMax - rsdMin_to_lambdaIdx IS OP rsdMin - >> put into rsdP_calc_idxAboveLBelowL change : +.....+ rsdMin rsdMax := rsdP ; +.....+ To : +.....+ rsdMin := rsdP ; rsdMax := rsdP ; +.....+ >> ouch! very incomplete, these vary between symDataSets, and overall [rsdMin rsdMax] are required!! fit_linearRegressSingle outputs [yDetrendMin, yDetrendMax] into each pStatLog, which is what I need! # Obs - data format; [Trn,Detrend] - transformed (eg log) : # +-------+-------+-------+-------+-----------+-----------+ # |xObsMin|xObsMax|yObsMin|yObsMax|yDetrendMin|yDetrendMax| # +-------+-------+-------+-------+-----------+-----------+ # | 1970.| 2022.5| 378.2|15958.4| 1.41783| 1.49852| # +-------+-------+-------+-------+-----------+-----------+ BUT : easier to put out of last entry for each symbol in : pStatLog := link d_work 'regression stat log.txt' ; Define pStatLog and make global in : '220730 Puetz2011 detrended international indexes, gnuplot specs.ndf' rather than : 'stock index semi-log [regression, plot]s.ndf' fit_linearRegressSingle_TradingView "$d_Qmkts"'stock index header.ndf' symName_dateRangeL - all the "active" symbols for the current pSpec 'stock index semi-log [regression, plot]s.ndf' - fit_linearRegressSingle_TradingView povr := link d_work 'semi-log ' symName_dateRange ' regression fit.txt' ; use these in optrs I now create in 'PuetzUWS calc [time, price] fractals.ndf' pRegressionALL_find_rsdMinMax pStatLog_symDates_find_rsdMinMax #08********08 #] 30-31Jul2022 adapt 'stock index [semi-log, detrend] data processing.ndf' for changes to 'PuetzUWS calc [time, price] fractals.ndf' 31Jul2022 # initial check : qnial> insertLine := '# PuetzUWS curvePlots (auto-added by QNial)' # PuetzUWS curvePlots (auto-added by QNial) qnial> pCurvesTmp := link d_temp 'pTmplt_insert_pCurvesTmp.txt' /mnt/ramdisk/pTmplt_insert_pCurvesTmp.txt qnial> pTmplt_add_pSub pCurvesTmp pPlotCmd pPlotPlt insertLine >> check : 'detrend StockMkt Indices 1871-2022 PuetzUWS2011.plt' >> YES! IT worked, I can't believe it... +-----+ I removed ', \' from the last curve : change : +.....+ '/home/bill/SG6/web/economics, markets/SP500/PuetzUWS intnl stocks/semi-log UKX100 Apr1988-Jul2022 TradingView regression fit.txt' using 1:6 w lp ls 105, \ +.....+ To : +.....+ '/home/bill/SG6/web/economics, markets/SP500/PuetzUWS intnl stocks/semi-log UKX100 Apr1988-Jul2022 TradingView regression fit.txt' using 1:6 w lp ls 105 +.....+ See if it works : $ gnuplot "$d_Qndfs"'gnuplot template, for subPths, auto-fill.plt' >> NUTS!! I need to change the file reference to : $ gnuplot "$d_web"'economics, markets/SP500/PuetzUWS intnl stocks/detrend StockMkt Indices 1871-2022 PuetzUWS2011.plt' +-----+ $ gnuplot "$d_web"'economics, markets/SP500/PuetzUWS intnl stocks/detrend StockMkt Indices 1871-2022 PuetzUWS2011.plt' If 'gnuplot' is not a typo you can use command-not-found to lookup the package that contains it, like this: cnf gnuplot >> not even installed on Suse? Installation log : Downloading libcerf1 (download size 30.7 KiB) Downloading gnuplot (download size 1.19 MiB) Downloading gnuplot-doc (download size 3.75 MiB) Installing libcerf1-1.5-1.19.x86_64.rpm (installed size 54 KiB) Installing gnuplot-5.2.2-3.9.2.x86_64.rpm (installed size 3.93 MiB) Installing gnuplot-doc-5.2.2-3.9.2.noarch.rpm (installed size 7.53 MiB) $ gnuplot "$d_web"'economics, markets/SP500/PuetzUWS intnl stocks/detrend StockMkt Indices 1871-2022 PuetzUWS2011.plt' set terminal pngcairo enhanced font courier,10 fontscale 1.0 size 1400, 400 ^ "/home/bill/SG6/web/economics, markets/SP500/PuetzUWS intnl stocks/detrend StockMkt Indices 1871-2022 PuetzUWS2011.plt", line 19: font: expecting string set terminal <terminal> >> becomes : set terminal pngcairo enhanced font courier, 10 fontscale 1.0 size 1400, 400 >> in '220730 Puetz2011 detrended international indexes, gnuplot specs.ndf' is : terminal := 'pngcairo enhanced font "courier,10" fontscale 1.0 size 1400, 400' ; >> should be : set terminal pngcairo enhanced font "courrier,10" >> also : datafile_separator := '"{\t}"' ; >> gets put in without quotes : set datafile separator {t} >> pain in the ass... as usual I have a problem with successive substitutions. For now, can manually fix : 1. last curve specification : remove ', \' 2. datafile_separator : "{\t}" 2. set terminal - put quotes around "courrier,10" +-----+ $ gnuplot "$d_web"'economics, markets/SP500/PuetzUWS intnl stocks/detrend StockMkt Indices 1871-2022 PuetzUWS2011.plt' {/: for [data, software, regression coefficients, plots, etc] see http://www.BillHowell.ca/economics, markets/SP500/PuetzUWS intnl stocks/}" " ^ "/home/bill/SG6/web/economics, markets/SP500/PuetzUWS intnl stocks/detrend StockMkt Indices 1871-2022 PuetzUWS2011.plt", line 27: unexpected or unrecognized token >> I assume that '{/: ' must be fllowed by a legitimate code >> just remove the curly braces for notes : set label 1 "yDetrend = (yObsTrans / yEstTrans - 1) / relStdDev" at graph 0.01, 0.95 textcolor "black" set label 2 "yTrend = 10 power (<constant> + <slope>*yrFrac)" at graph 0.01, 0.9 textcolor "black" set label 3 "I could not find old SHCOMP long-term data series for China" at graph 0.01, 0.85 textcolor "black" set label 4 "price [equilibrium, revolution] cross greatly influences results (eg SP500 OR[1926, 1938]-2022)" at graph 0.01, 0.8 textcolor "black" $ gnuplot "$d_web"'economics, markets/SP500/PuetzUWS intnl stocks/detrend StockMkt Indices 1871-2022 PuetzUWS2011.plt' {/: for [data, software, regression coefficients, plots, etc] see http://www.BillHowell.ca/economics, markets/SP500/PuetzUWS intnl stocks/}" " ^ "/home/bill/SG6/web/economics, markets/SP500/PuetzUWS intnl stocks/detrend StockMkt Indices 1871-2022 PuetzUWS2011.plt", line 27: unexpected or unrecognized token >> oops, same thing for title, I addede back a \n !!! set title "{/:Bold PuetzUWS2011 [time, price] multi-fractals, detrended international stock market indices : [DAX40, INDIA50, NI225, SP500, UKX100]}\n for [data, software, regression coefficients, plots, etc] see http://www.BillHowell.ca/economics, markets/SP500/PuetzUWS intnl stocks/" $ gnuplot "$d_web"'economics, markets/SP500/PuetzUWS intnl stocks/detrend StockMkt Indices 1871-2022 PuetzUWS2011.plt' for [data, software, regression coefficients, plots, etc] see http://www.BillHowell.ca/economics, markets/SP500/PuetzUWS intnl stocks/" " ^ "/home/bill/SG6/web/economics, markets/SP500/PuetzUWS intnl stocks/detrend StockMkt Indices 1871-2022 PuetzUWS2011.plt", line 27: invalid command >> might be \n that was entered into code, not retained for title? $ gnuplot "$d_web"'economics, markets/SP500/PuetzUWS intnl stocks/detrend StockMkt Indices 1871-2022 PuetzUWS2011.plt' set arrow from 1872.73, -3.0 to 1872.73, 6.0 nohead lw 2 linecolor "dark grey" ^ "/home/bill/SG6/web/economics, markets/SP500/PuetzUWS intnl stocks/detrend StockMkt Indices 1871-2022 PuetzUWS2011.plt", line 85: unrecognized color name and not a string "#AARRGGBB" or "0xAARRGGBB" >> yuch, have to look it up - there are 12-14 official gnuplot mnemonic colors? #] 24May2021 search "gnuplot and how do I set dashed lines?" set template to : # Howell's standard color scheme - can add more with spacialFeatureLines # $ show colornames - for a list of Linux standard colors OK, finally got output - many, many hannd twigs. I think I need to use : rsd_sl = log((yObs/ytrend_sl/relStdDev_sl)), not rsd = ((yObs/ytrend) - 1) where _sl = semi-log cannot take log after transform (goes through zero from [above, below] # (EACH string constant slope StdErrEst relStdErr relStdDev) EACHLEFT link ' ' # link '# ' (link ((EACH string constant slope StdErrEst relStdErr relStdDev) EACHLEFT link ' ')) #08********08 #] 30Jul2022 upgrade, clean, clear] 'PuetzUWS calc [time, price] fractals.ndf' OK - works very well # full format for checks : 'set label 1 "{yDetrend = (yObsTrans / yEstTrans - 1) / relStdDev" at graph 0.01, 0.95 textcolor "black"' 'set label 2 "{yTrend = 10 power (<constant> + <slope>*yrFrac)}" at graph 0.01, 0.90 textcolor "black"' 'set label 3 "I could not find old SHCOMP long-term data series for China" at graph 0.01, 0.85 textcolor "black"' 'set label 4 "price revolution of SP500 (eg 1926-2022 or 1938-2022) greatly influences results" at graph 0.01, 0.80 textcolor "black"' #08********08 #] 28Jul2022 rsdP_find_aboveBelowLimits after setting up new Seagate 6Tb USB HDD moved code between several Puetz programs - headers etc to make things more [clear, re-usable] fracP_calc_fracLL IS OP fracP - adapt PineScript code which worked well much thinking went into it 29Jul2022 problems with frac # qnial> rsdP_find_aboveBelowLimits 0.02 1 18 16 0 0 >> OK qnial> rsdP_find_aboveBelowLimits -1 -0.02 0 0 2 4 >> OK qnial> rsdP_find_aboveBelowLimits -0.02 1 qnial> rsdP_find_aboveBelowLimits -0.02 1 17 15 3 5 >> Hmm, does this make sense? 3 -0.333333 15 0.037037 4 -0.111111 16 0.111111 5 -0.037037 17 0.333333 >> It does what I coded it to do, but that's wrong!! I need the same "levels" for the "shorter side", but only going up to the higest level that incorporates rsdBelow (in the example above). example : rsdBelowIdx = 5, whereas [3, 4] not used rsdP_find_aboveBelowLimits IS OP rsdP ... % 29Jul2022 current rsdPuetzL : 21 elements, (0 - 20) midIdx == 10 ; shapeRsdPuetzL := gage shape rsdPuetzL ; % midIdx is the index for which rsdPuetz = 0.000 (assumes symmetrical list) ; % This provides the [above, below] threshold ; midIdx := floor (shapeRsdPuetzL / 2) ; rsdMin rsdMax := rsdP ; rsdSpan := rsdMax - rsdMin ; % ; % find largest rsdPuetzL that fits into rsdSpan ; idxBase := last ( (rsdPuetzL EACHLEFT <= rsdSpan) sublist (tell (gage shape rsdPuetzL)) ) ; % use 2 UWS periods, will crash near ends, worry about that later ; idxL := (idxBase - 2) + tell 3 ; % ; ... ELSE % rsdSpan crosses midIdx (where rsd = 0), adjust +1 because off midIdx ; IF (abs(rsdMinIdx - midIdx) > abs(rsdMaxIdx - midIdx)) THEN idxL := rsdAboveIdx - (tell (rsdAboveIdx - rsdAboveStopIdx)) ; bolIdxL := l l l ; i := 0 ; FOR idx WITH idxL DO IF (rsdBelowStopIdx > rsdPuetzL@idxL) THEN bolIdx@i := o ; ENDIF ; i := i + 1 ; ENDFOR ; idxL := bolIdxL sublist idxL ; rsdAboveIdx := midIdx + rsdBasIdx ; rsdAboveStopIdx := midIdx + rsdLesIdx ; ELSE idxL := rsdBelowIdx + (tell (rsdBelowStopIdx - rsdBelowIdx)) ; rsdBelowIdx := midIdx - rsdBasIdx ; rsdBelowStopIdx := midIdx - rsdLesIdx ; ENDIF ; ENDIF ; rsdAboveIdx rsdAboveStopIdx rsdBelowIdx rsdBelowStopIdx # olde code IF (rsdBasIdx < 2) THEN rsdLesIdx := midIdx ; ELSE rsdLesIdx := rsdBasIdx - 2 ; ENDIF ; +-----+ olde code # 22Jul2022 pUnixDate_to_pYrfrac not working, so yFrac files have strs with no apos - screws everything up!!!! #08********08 #] 22Jul2022 change regression [calcc,output]s # singleLinearRegression [estimate, residual, detrend]s xObs yObs yEst yResids pct_resid yDetrend 28800 629.100 ?Integer overflow ?Integer overflow ?Integer overflow ?Integer overflow 2793600 574.600 ?Integer overflow ?Integer overflow ?Integer overflow ?Integer overflow 5212800 558.600 ?Integer overflow ?Integer overflow ?Integer overflow ?Integer overflow 7804800 573.900 ?Integer overflow ?Integer overflow ?Integer overflow ?Integer overflow >> maybe UNix time? >> timestamp_YYMMDD_HMS is screwing up - prevented pinn_backupDatedTo_zArchive from working month_number Month #08********08 #] 21Jul2022 revamp yDetrend to log (yObsTrans / yEstTrans), add PuetzUWS [time, price] multi-fractals # loaddefs link d_Qndfs 'fit_linearRegress.ndf' residual_linearSingle change : +.....+ yDetrend := yObsTrans / yEstTrans ; +.....+ To : +.....+ yDetrend := log (yObsTrans / yEstTrans) ; +.....+ need optr to revamp all symName_dateRange at once +-----+ olde code # loaddefs link d_Qndfs 'economics, markets/stock index [semi-log, detrend] data processing.ndf' IF flag_debug THEN write 'loading fit_linearRegressSingle_TradingView' ; ENDIF ; # fit_linearRegressSingle_TradingView IS - single independent variable linear regression, # semi-log basis, using opening price # standard format of data table - col 0 = Unix time (milliseconds since 01Jan1970); col 1 = open price # pinn_read_csvTable pinn - doesn't work yet, so convert all commas to tabs in data file # colTitles - surround text with apos; tBody - copy string 'NaN' - replace within apos # I use a semi-log plot and fit # IF flag_break THEN BREAK ; ENDIF ; fit_linearRegressSingle_TradingView IS { LOCAL finn fovr openPrice tbl_raw yrFrac ; NONLOCAL d_work symName_dateRange constant slope StdErrorEst relStdErr relStdDev ; % ; finn := link 'tabSep ' symName_dateRange ' data.txt' ; fovr := link 'semi-log ' symName_dateRange ' regression fit.txt' ; % ; % pinn_read_csvTable pinn - doesn't work yet, so convert all commas to tabs in data file, and use ; tbl_raw := tbl_remove_colTitles pinn_read_tabTable (link d_work finn) ; yrFrac := EACH secFrom01Jan1970_to_yFrac tbl_raw|[,0] ; openPrice := tbl_raw|[,1] ; constant slope StdErrorEst relStdErr relStdDev := fit_linearRegressSingle yrFrac openPrice "pass "log (link d_work fovr) ; } # loaddefs link d_Qndfs 'economics, markets/stock index [semi-log, detrend] data processing.ndf' IF flag_debug THEN write 'loading setup_bag' ; ENDIF ; # setup_bag IS - for one-at-a-time work, start with TradingView data download, specify $symName_dateRange setup_bag IS { LOCAL coeffL ; % symName_dateRange constant slope StdErrorEst relStdErr relStdDev ; coeffL := 'DAX40 Jan1970-Jul2022' -61.6477 0.0325751 0.114373 1.30129 0.254864 ; coeffL := 'INDIA50 Mar2013-Jul2022' -96.353 0.0497353 0.031161 1.07439 0.070384 ; coeffL := 'NI225 Jan1984-Jul2022' -1.66843 0.0029173 0.140672 1.38252 0.366279 ; coeffL := 'SP500 1872-1940' -61.6477 0.0325751 0.114373 1.30129 0.254864 ; coeffL := 'SP500 1926-2022' -61.6477 0.0325751 0.114373 1.30129 0.254864 ; coeffL := 'UKX100 Apr1988-Jul2022' -25.5175 0.0145593 0.0894005 1.22857 0.201597 ; % ; } ('# ' EACHRIGHT link (rows picture (5 2 reshape constant slope Rsqr coeffCorrel StdErrEst relStdErr relStdDev ) ) ) ; fout EACHRIGHT writefile '# ' '# ' ; # enddoc