#] #] ********************* #] "$d_Qmkts"'market symbol [regressSL, detrend]- develop.txt' - # www.BillHowell.ca 23Aug2022 initial, taken from other notes files # view in text editor, using constant-width font (eg courier), tabWidth = 3 # 23Aug2022 copied files from other dirs (see below) #48************************************************48 #24************************24 # Table of Contents, generate with : # $ grep "^#]" "$d_Qmkts"'market symbol [regressSL, detrend]- develop.txt' | sed "s/^#\]/ /" # ********************* "$d_Qmkts"'market symbol [regressSL, detrend]- develop.txt' - +-----+ Market-related [file, dir]s : ToDos : ??Apr2023 Long-term symbols [GOLD, OIL] must be two-segment, like SP500 19Aug2022 fix apos in yrFrac files - and optrs 20Aug2022 digitize long-term historical LSE index +-----+ +-----+ Rest of content from earlier notes files : 20Aug2022 continue create : symLo_symHi_calc_crossAreaPerYr 14Aug2022 create keyColPhr_keyItm_colPhrL_rosterGet_valLL - roster access optr 13Aug2022 fix [CsvToTab, unixDate, regressPlot] 12Aug2022 fix upgrades to "$d_Qmkts" coding 08Aug2022 yrL_regressPlotSP500 - process segSP500Ary of cart [1871-yrL, yrL-2022] 08Aug2022 fix [time, price]Fracs Jun-Aug2022 revamp "$d_mkts"'SP500/PuetzUWS intnl stocks/' QNial code 19Jul2022 minimum-manual auto-file adaptations : TVdata -> linear regression -> gnuplot 07Jan2022 Formula for semi-logs 20Sep2020 create composite images from "standard size" image outputs, #24************************24 #] +-----+ #] Market-related [file, dir]s : Market data related : "$d_mktData"'[, detrend, digitize graphs, rawCsv, semi-log, SLregress, TV Fibonacci mirror, unixDate, yrFraction]/' "$d_mkts"'Fischer, David 1996 The Great Wave/0_Fischer notes.txt' also, many other [data, analysis] in "$d_[mkts, Qroot]" "$d_SysMaint"'digitizelt graph from image/digitizelt - produce graphical data from image.txt' PuetzUWS [time, price] multifractals "$d_Qmkts"'PuetzUWS calc [time, price] fractals.ndf modelling as continuous systems : "$d_mkts"'PE Schiller forward vs 10yr Tbills/' "$d_PuetzUWS"'0_PuetzUWS PSOmodel notes.txt' #24************************24 #] ToDos : #] ??Apr2023 Long-term symbols [GOLD, OIL] must be two-segment, like SP500 later : copy latest regression parameters to paramTbl #] 19Aug2022 fix apos in yrFrac files - and optrs #] 20Aug2022 digitize long-term historical LSE index https://globalfinancialdata.com/stocks-for-the-very-long-run-the-uk-100-and-327-years-of-british-equity-history I can pick from this, and a paper that I saw previously #] 12Mar2023 #] +-----+ #24************************24 #08********08 #] ??Apr2023 #08********08 #] ??Apr2023 #08********08 #] ??Apr2023 #08********08 #] ??Apr2023 #08********08 #] ??Apr2023 #08********08 #] ??Apr2023 #08********08 #] 30Mar2023 slogging towards full range of mktDat must first fix (see develop files for these) y ary_adrCompatible "$d_Qtest"'types/types- develop.txt' pth_typ_exists "$d_Qndfs"'file_ops.ndf' pCsv_to_pYrFrac "$d_Qmkts"'market symbol [regressSL, detrend].ndf' #08********08 #] ??24Aug2022 +-----+ olde code IF flag_debug THEN write 'loading pUnixDate_to_pYrfrac IS OP symbol ' ; ENDIF ; # pCsv_to_pYrfrac IS OP symbol - for csv files with dates in yrFrac format # enclose text in chr_apo = \d39 : # IF flag_break THEN BREAK ; ENDIF ; pUnixDate_to_pYrfrac IS OP symbol { LOCAL pUnixDate pYrfrac colTitles tblData tbl_raw tbl_yrFrac unixTimeL ; NONLOCAL d_unixDate d_yrFrac ; % ; % pUnixDate must be same as for pCsv_to_pYrFrac ; pUnixDate := link d_unixDate 'unixDate ' symbol '.dat' ; % ; % pinn_read_csvTable pinn - doesn't work yet, so convert all commas to tabs in data file, and use ; % note - pinn_readExecute_tabTbl "removes" apos from strings, ; % so this puts them back for next readfile! ; tbl_raw := EACH str_wrapWithApos (pinn_readExecute_tabTbl pUnixDate) ; colTitles tblData := tbl_to_colTitles_tblData tbl_raw ; tblData|[,0] := EACH secFrom01Jan1970_to_yFrac tblData|[,0] ; tbl_yrFrac := tbl_add_colTitles tblData colTitles ; tbl_writeTabSeparated_povr tbl_yrFrac pYrfrac ; } # pUnixDate_to_pYrfrac 'CUS2 May2006-Aug2022 TradingView' # 13Aug2022 pth_typ_exists not working : qnial> pYrfrac := link d_data 'yrFrac CUS2 May2006-Aug2022 TradingView.dat' /home/bill/SG6/web/economics, markets/SP500/PuetzUWS intnl stocks/yrFrac CUS2 May2006-Aug2022 TradingView.dat qnial> pth_typ_exists pYrfrac "pinn l qnial> pYrfrac := link d_data 'yrFrac CUS3 May2006-Aug2022 TradingView.dat' /home/bill/SG6/web/economics, markets/SP500/PuetzUWS intnl stocks/yrFrac CUS3 May2006-Aug2022 TradingView.dat qnial> pth_typ_exists pYrfrac "pinn l # fracPuetzL is constant UWS series, where frac@i = 3*frac@(i-1) # must be "symmetrical" - same # items [above, below] midIdx # from "$d_webRawe"'economics, markets/SP500/PuetUWS/' # 'semi-log SP500 1870-2030 y-log detrend, y-data no model, digitizelt fixed.csv.txt' # renamed 'semi-log SP500 1872-2020 y-log detrend, y-data no model, digitizelt fixed.csv.txt' #08********08 #] 24Aug2022 symLo_symHi_calc_crossAreaPerYr { LOCAL colYr colDetrend idx resFltL aryLo idxLoAllL idxBothLoL xLoL xMin1 xMax1 fLo pDatLo symShortLo aryHi idxHiAllL idxBothHiL xHiL xMin2 xMax2 fHi pDatHi symShortHi xLoBothL yLoBothL xDeltaL xDeltaSumChk crossAreaPerYr xHiBothL yHiBothL yDeltaL ; NONLOCAL strUWS ; #08********08 #] 24Aug2022 fix symbol_xP_SLRegress continued many fixes (had copied code from >> now loaddefs OK, haven't tested +-----+ olde code % [colRegress colDetrend] are "fixed" by fit_SLRgress ; colRegress colDetrend := 6 8 ; xHiLowest := first xHiL ; idxBoth1 := null ; xLoHighest := last xLoL ; idxBoth2 := null ; idxLoBothL := ??? ; FOR idx WITH idxLoBothL DO IF (xLo@idx <= xLoHighest) THEN idxBoth2 := idx ; ENDIF ; exit 'found high point of common data' ; ENDIF ; ENDFOR ; idxBothL := idxBoth1L + (tell (idxBoth2 - idxBoth1)) ; % ; % xL data is the same for the "both" (common) region ; xHiBothL yHiBothL := cols ary|[idxHiBothL, xCol yCol] ; xLoBothL yLoBothL := cols ary|[idxLoBothL, xCol yCol] ; IF (~= xLoBothL xHiBothL) THEN resFltL := link resFltL (fault '?dsscca05') ; ENDIF ; % ; % pinn_read_csvTable pinn - doesn't work yet, so : ; % manually replace commas in colTitles (and other strings) with `- or other ; % convert all remaining commas to tabs in data file ; tbl_raw := tbl_remove_colTitles pinn_readExecute_tabTbl pinn ; % assumes that symbol occurs only once in symName_dateRangeL ; % (slc gives ary2D in roster_keyColPhr_keyItm_colPhrL_get_itmLL) ; #] +-----+ #] Rest of content from earlier notes files : archived "$d_webRawe"'economics, markets/SP500/0_SP500 notes.txt' also "$d_Qmkts"'0_PuetzUWS intnl stocks notes.txt' #08********08 #] 20Aug2022 continue create : symLo_symHi_calc_crossAreaPerYr major revamps, still not done 23Aug2022 +-----+ olde code # loaddefs link d_Qmkts 'Puetz detrended specs.ndf' IF flag_debug THEN write 'loading symbol_segYrL_calc_XareaPerYr' ; ENDIF ; # symbol_segYrL_calc_XareaPerYr IS OP symbol segYrL - # 09Aug2022 only SP500 [has, vary] segments, subDir is wrt global d_data # unSegmented : [DAX40, INDIA50, NI225, UKX100] # 08Aug2022 initial symbol_segYrL_calc_XareaPerYr IS OP symbol segYrL { LOCAL yrLo1 yrHi1 yrLo2 yrHi2 pltPreName crossAreaPerYr strLo1 strHi1 strLo2 strHi2 strDateRange ; NONLOCAL d_data d_work strUWS ; % ; % setup parameters for each yrL ; pltPreName := 'detrend MktSym' ; % ; yrLo1 yrHi1 yrLo2 yrHi2 := segYrL ; strLo1 strHi1 strLo2 strHi2 := EACH string segYrL ; strDateRange := link strLo1 '-' strHi1 ',' strLo2 '-' strHi2 ; % ; % crossAreaPerYr = "fit" eg for optimization ; % calculate "area" between [1871-??, ??-2022] cross-overs ; segYrPP := ((yrLo1 yrHi1) (yrLo2 yrHi2)) ; crossAreaPerYr := dir_symbol_segYrPP_calc_crossAreaPerYr d_work symbol segYrPP ; % ; % create a giant image matrix of results (maybe via gimp or other prgm) ; % crossAreaPerYr := dir_symbol_segYrPP_calc_crossAreaPerYr d_work symbol segYrPP ; % ; crossAreaPerYr } # 20Jul2022 Perhaps do this much later - but default settings work well enough for now # ;s||$yearMin|;s||$yearMax|;s||$openMin|;s||$openMax|' # Manually, edit the .plt file for better : # set xrange [ . : . ] # set yrange [ . : . ] # then run gnuplot as per fits linof .plt file, eg # $ gnuplot "$d_webRawe"'economics, markets/SP500/semi-log NI225 Jan1984-Jul2022.plt' # segStrYrPP := # segStrP_calc_crossAreaPerYr d_work 'SP500' segStrYrPP # olde code - use in calling optr... or whatever seg1StrStrt seg1StrEnd seg2StrStrt seg2StrEnd segStr1 segYr1 := segStrYrP1 segStr2 segYr2 := segStrYrP2 seg1StrStrt seg1StrEnd seg2StrStrt seg2StrEnd := EACH string segSP500Itm ; strDateRange := link seg1StrStrt '-' seg1StrEnd ',' seg2StrStrt '-' seg2StrEnd ; oldformat := setformat '%7.2f' ; setformat oldformat ; # this cmd is used in : # pPlt_gnuplot_pPng of 'stock index semi-log [regression, plot]s.ndf' # pTmplt_pSubs_create_pPlt 'gnuplot template, subFiles [create, insert].ndf' #08********08 #] 14Aug2022 create keyColPhr_keyItm_colPhrL_rosterGet_valLL - roster access optr also augmented paramTbl -->[nextv] first (keyColPhr_keyItm_colPhrL_getRoster_valLL "symName_dateRangeL symbol "xyTrnPhrPL) ("pass "log) -->[nextv] xTrnPhr yTrnPhr := first (keyColPhr_keyItm_colPhrL_getRoster_valLL "symName_dateRangeL symbol "xyTrnPhrPL) ?assignment -->[nextv] isString first (keyColPhr_keyItm_colPhrL_getRoster_valLL "symName_dateRangeL symbol "xyTrnPhrPL) l Trickey because keyColPhr_keyItm_colPhrL_rosterGet_valLL returns an ary2D!! OK, finally got working key subSteps : symbol_xyTrnPhrP_SingleLinearRegress_TradingView IS OP symbol symbol_regressPlot_pPng IS OP symbol : % assumes that symbol occurs only once in symName_dateRangeL ; % (slc gives ary2D in keyColPhr_keyItm_colPhrL_rosterGet_valLL) ; xTrnPhr yTrnPhr := keyColPhr_keyItm_colPhrL_rosterGet_valLL "symName_dateRangeL symbol ("xTrnPhrL "yTrnPhrL) ; +-----+ symbol_regressPlot_pPng 'CUS2 May2006-Aug2022 TradingView' >> very good, except yRange not working? >> oops, didn't save ndf file qnial> symbol_regressPlot_pPng 'CUS2 May2006-Aug2022 TradingView' set yrange [ ( 100. : 300. ) ] ^ "/home/bill/SG6/web/economics, markets/SP500/PuetzUWS intnl stocks/220812/SLregress CUS2 May2006-Aug2022 TradingView PuetzUWS2011.plt", line 39: ')' expected >> oops, no parenthesis! (square brackets already in template) # +-----+ # cmdDetrend_create IS OP symName_dateRange - sed # 15Aug2022 also serves as an example for other projects # 06Aug2022 initial cmdDetrend_create IS OP symbol { LOCAL pTmplt terminal datafile_separator noteL title ylabel pltPreName xRange xtics yRange ytics ; NONLOCAL d_work pPng pWrdChg strUWS yrEnd yrP yrStrt yrZero yLow yHigh colYr_colDetrend ; % ; pTmplt := link d_Qndfs 'gnuplot template, for subPths, auto-fill.plt' ; pltPreName := 'detrend StockMkt Indices' ; pPng := link d_work pltPreName ' ' symbol ' ' strUWS '.png' ; % ; terminal := 'pngcairo enhanced font \"arial,10\" fontscale 1.0 size 1400, 400' ; % tab as column separator -> "{\t}", as comma -> "{,}", "{whitespace}" ; datafile_separator := '"{\\t}"' ; % ; title := link '{/:Bold ' strUWS ' [time, price] multi-fractals, detrended international stock market indices : [DAX40, INDIA50, NI225, SP500, UKX100]}\\n {/: for [data, software, regression coefficients, plots, etc] see http://www.BillHowell.ca/economics, markets/SP500/PuetzUWS intnl stocks/}' ; ylabel := 'Detrended stock market index, closing or average price' ; noteL := 'yDetrend = (yTrn/yEstTrn - 1) / relStdDevTrn (Trn = log transformation of [y])\\nyTrend = 10 power ( + * yr) (normal, not log, trend of y)\\nprice [equilibrium, revolution] cross greatly influences results (eg SP500 OR[1926, 1938]-2022)' ; % ; % adjust for different strDateRange! ; % yrStrt yrEnd yrP yrZero xtics ytics yRange ; % colYr_colDetrend ; yrStrt yrEnd := 1860 2040 ; yrP := yrStrt yrEnd ; yrZero := 1929.75 ; xtics := 20 ; yRange := ' -3. : 4. ' ; ytics := '-3, -2, -1, 0.0, 1, 2, 3, 4' ; yLow yHigh := -3. 4. ; colYr_colDetrend := '1:6' ; % ; % sed variables are listed in order of occurence in pTmplt ; link 'cat "' pTmplt '" | sed ' chr_apo 's||' pPng ';s||' datafile_separator ';s||' title '|;s|<ylabel>|' ylabel '|;s|<noteL>|' noteL '|;s|<xtics>|' (string xtics) '|;s|<yRange>|' yRange '|;s|<ytics>|' ytics '|' chr_apo ' >"' pWrdChg '" ' } # cmdDetrend_create 'CUS2 May2006-Aug2022 TradingView' # olde code : cmdDetrend := link sed ' chr_apo '|;s|<title>|' title ; '|;s|<colYr_colDetrend>|' colYr_colDetrend '|;s|<cmtLabel>|' cmtLabel '|;s|<strUWS>|' strUWS '|;s|<pPng>|' pPng |;s|<yrStrt>|' (string yrStrt)'|;s|<yrEnd>|' (string yrEnd) ' #08********08 #] 13Aug2022 fix [CsvToTab, unixDate, regressPlot] process new csvRaw [CUS2, GOLD, HSSSHSC300, TNX, WLFN] qnial> pCsv_to_pYrFrac 'CUS2 May2006-Aug2022 TradingView' >> OK qnial> pUnixDate_to_pYrfrac 'CUS2 May2006-Aug2022 TradingView' >> OK qnial> fit_SLRgress_TradingView 'CUS2 May2006-Aug2022 TradingView' >> good output? but ?noexpr for [xUnTrnPhr, yUnTrnPhr] >> should colTits be enclosed by quotes or apos depends on gnuplot, but apos better for qnial! fit_SLRgress in 'fit_linearRegress.ndf' : >> revamped to include xyTrnPhrP symbol_xyTrnPhrP_SingleLinearRegress_TradingView 'CUS2 May2006-Aug2022 TradingView' >> still ?noexpr for [xUnTrnPhr, yUnTrnPhr] qnial> EACH trnOp_reverse "pass "log ?noexpr ?noexpr >> nuts... qnial> trnPhrUnOpLL ?undefined identifier: TRNPHRUNOPLL <***> qnial> trnPhrUnOpLL := "pass "log "exp "sin "cos "unLog "ln "arcSin "arcCos "pass "unLog "ln "arcSin "arcCos pass log exp sin cos unLog ln arcSin arcCos pass unLog ln arcSin arcCos log exp sin cos qnial> n_rows := 2 ; n_cols := floor ((gage shape trnPhrUnOpLL) / 2) 9 qnial> trnPhrUnOpLL := rows (n_rows n_cols reshape trnPhrUnOpLL) |pass log exp sin cos unLog ln arcSin arcCos|pass unLog ln arcSin arcCos log exp sin cos| qnial> EACH trnPhr_reverse "pass "log pass unLog >> OK, it's fixed try again with new optrName : symbol_xyTrnPhrP_SingleLinearRegress_TradingView 'CUS2 May2006-Aug2022 TradingView' >> good can I use apos instead of quotes for colTitles in gnuplot? hand-convert 'CUS2 May2006-Aug2022 TradingView' & try it qnial> symbol_regressPlot_pPng 'CUS2 May2006-Aug2022 TradingView' "/mnt/ramdisk/symbol_regressPlot_pPng tmp.txt", line 0: Cannot open script file '/mnt/ramdisk/symbol_regressPlot_pPng tmp.txt' >> the .plt file is NOT being ccreated >> where does this come from? : 'symbol_regressPlot_pPng tmp.txt' >> I should save the .plt files to d_work anyways? 'symbol_regressPlot_pPng tmp.txt' look in : symbol_xyTrnPhrP_SingleLinearRegress_TradingView >> this isn't it 'stock index semi-log [regression, plot]s.ndf' >> double check, not in here at all 'fit_linearRegress.ndf' >> not in here at all fit_SLRgress fit_SLRgress IS OP symName xObs yObs xTrnPhr yTrnPhr povr >> double check, not in here at all 'Puetz detrended specs.ndf' >> not in here at all 'gnuplot template, subFiles [create, insert].ndf' >> not in here at all $ find "$d_Qndfs" -maxdepth 4 -type f -name "*" | grep --invert-match "z_Old\|z_Archive" | tr \\n \\0 | xargs -0 -IFILE grep --with-filename --line-number "symbol_regressPlot_pPng tmp.txt" "FILE" Binary file /home/bill/SG6/web/Qnial/MY_NDFS/economics, markets/.0_PuetzUWS intnl stocks notes.txt.kate-swp matches >> nowhere! so how does that even pop up? detrend StkMktIdxs CUS2 May2006-Aug2022 TradingView PuetzUWS2011.plt >> maybe I've mixed up semi-log & detrend? I forgot to host cmd for pPlt! qnial> symbol_regressPlot_pPng 'CUS2 May2006-Aug2022 TradingView' sed: -e expression #1, char 154: unknown option to `s' >> maybe sed search text has illegal chrs try cmd without : ;s|<imageSize>|' imageSize ';s|<datafile_separator>|' datafile_separator '|;s|<title>|' title '| ;s|<noteL>|' noteL ';s|<xLog>|' xLog ';s|<yLog>|' yLog ' ;s|<ylabel>|' ylabel ' pPlt name wrong - should be 'semi-log' not 'detrend' put back sed subExpressions one-by-one to find the problem I was missing many `| in sed - try whole thing? qnial> symbol_regressPlot_pPng 'CUS2 May2006-Aug2022 TradingView' set title "semi-log CUS2 May2006-Aug2022 TradingView TradingView stock market index\n for [data, software, results] see http://www.BillHowell.ca/economics, markets/SP500/"" " ^ "/home/bill/SG6/web/economics, markets/SP500/PuetzUWS intnl stocks/220812/semi-log CUS2 May2006-Aug2022 TradingView PuetzUWS2011.plt", line 20: unexpected or unrecognized token >> crapped out at set ylabel "<yLabel>" add `\ before quotes? but this WASN'T a problem for <datafile_separator>!!? line 20: unexpected or unrecognized token : set title "semi-log CUS2 May2006-Aug2022 TradingView TradingView stock market index\n for [data, software, results] see http://www.BillHowell.ca/economics, markets/SP500/"" >> double quote from title := link 'semi-log ' symbol ' TradingView stock market index\\n for [data, software, results] see http://www.BillHowell.ca/economics, markets/SP500/"' ; Re-run qnial> symbol_regressPlot_pPng 'CUS2 May2006-Aug2022 TradingView' label 1 "<noteL>" at (graph 0.01, 0.93, 0) left not rotated back noenhanced nopoint "/home/bill/SG6/web/economics, markets/SP500/PuetzUWS intnl stocks/220812/semi-log CUS2 May2006-Aug2022 TradingView PuetzUWS2011.plt", line 77: warning: Cannot find or open file "/home/bill/SG6/web/economics, markets/SP500/PuetzUWS intnl stocks/semi-log DAX40 Jan1970-Jul2022 TradingView regression fit.txt" "/home/bill/SG6/web/economics, markets/SP500/PuetzUWS intnl stocks/220812/semi-log CUS2 May2006-Aug2022 TradingView PuetzUWS2011.plt", line 77: warning: Cannot find or open file "/home/bill/SG6/web/economics, markets/SP500/PuetzUWS intnl stocks/semi-log DAX40 Jan1970-Jul2022 TradingView regression fit.txt" "/home/bill/SG6/web/economics, markets/SP500/PuetzUWS intnl stocks/220812/semi-log CUS2 May2006-Aug2022 TradingView PuetzUWS2011.plt", line 77: No data in plot Obviously, I need seds for each curve (only two here) >> oops, had chr_apo in curve[curveNormal, Trend] strings qnial> symbol_regressPlot_pPng 'CUS2 May2006-Aug2022 TradingView' cat "/home/bill/SG6/web/Qnial/MY_NDFS/gnuplot template, simple auto-fill variables.plt" | sed 's|<pPng>|/home/bill/SG6/web/economics, markets/SP500/PuetzUWS intnl stocks/220812/semi-log CUS2 May2006-Aug2022 TradingView PuetzUWS2011.png|;s|<imageSize>|1400, 400|;s|<datafile_separator>|"{\\t}"|;s|<title>|semi-log CUS2 May2006-Aug2022 TradingView TradingView stock market index\\n for [data, software, results] see http://www.BillHowell.ca/economics, markets/SP500/|;s|<ylabel>|closing price monthly average\\n (in currency of country except India in UDS)|;s|<xLog>|nologscale|;s|<yLog>| logscale|;s|<curveNormal>|/home/bill/SG6/web/economics, markets/SP500/PuetzUWS intnl stocks/220812/semi-log CUS2 May2006-Aug2022 TradingView regression fit.txt using 1:2 w lp ls 100, \|;s|<curveTrend>|/home/bill/SG6/web/economics, markets/SP500/PuetzUWS intnl stocks/220812/semi-log CUS2 May2006-Aug2022 TradingView regression fit.txt using 1:3 w lp ls 101|' >"/home/bill/SG6/web/economics, markets/SP500/PuetzUWS intnl stocks/220812/semi-log CUS2 May2006-Aug2022 TradingView PuetzUWS2011.plt" sed: -e expression #1, char 684: unknown option to `s' maybe add another `\ to end of curveNormal? put back apos in curve[curveNormal, Trend] strings? >> would cause same old problem put in weird chr_combo, sub after? ~:~ - substitution didn't happen, sym in wrong place OK, after bug chases all day, symbol_regressPlot_pPng works Try qnial> revampONE_CsvToYrFrac_unixDate_regressPlot 'GOLD Jan1834-Aug2022 TradingView' label 1 "GOLD Jan1834-Aug2022 TradingView, xTrn= pass, yTrn= log, yTrend = 10^(-61.647700 + (0.032575*yFrac))\nrelStdErrTrn = 1.301280; relStdDevTrn = 0.254864" at (graph 0.01, 0.93, 0) left not rotated back noenhanced nopoint >> perfect - except yAxis scaling (because of trend) >> gold is a non-free market before 1971-1975 >> OK, needs multi-segment 1860-[1940, 1975] # revampONE_CsvToYrFrac_unixDate_regressPlot 'HSSSHSC300 Sep2020-Aug2022 TradingView' qnial> revampONE_CsvToYrFrac_unixDate_regressPlot 'HSSSHSC300 Sep2020-Aug2022 TradingView' sed: can't read /home/bill/SG6/web/economics, markets/SP500/PuetzUWS intnl stocks/rawCsv HSSSHSC300 Sep2020-Aug2022 TradingView.txt: No such file or directory >> oops, misnamed rawCsv >> renamed, now OK # revampONE_CsvToYrFrac_unixDate_regressPlot 'SHCOMP Jan1970-Jul2022 TradingView' >> bad dates, fixed & reloaddefed header revampONE_CsvToYrFrac_unixDate_regressPlot 'SHCOMP 2002-Aug2022 TradingView' >> something wrong with columns? - look at later # revampONE_CsvToYrFrac_unixDate_regressPlot 'TNX 1963-Aug2022 TradingView' >> same as SHCOMP, something wrong with columns? - look at later >> maybe trailing chr_tab is needed? # revampONE_CsvToYrFrac_unixDate_regressPlot 'USOIL Dec1861-Aug2022 TradingView' >> OK, needs multi-segment 1860-[1940, 1975] # revampONE_CsvToYrFrac_unixDate_regressPlot 'WLFN Jan2007-Aug2022 TradingView' >> OK, needs better [yRange, yTics] ToDos : Copy latest regression parameters to paramTbl ParamTbl needs columns for y[Label, Range, Tics] Long-term symbols [GOLD, OIL] must be two-segment, like SP500 optr to pull symbol info from paramTbl would be handy +-----+ olde code # BEFORE running pCsv_to_pYrFrac # clean up titleLine - remove commas within quotes, put apos arount colTitles # [INDIA50, NI22, UKX100] are all OK, but new files may not be # saves in ptmp2 := link d_temp 'pCsv_to_pYrFrac 2.txt' ; # How do I put apos around text in first line??!!! # s|\t\([a-zA-Z0-9]+\)\t|\t\d39\1\d39\t|g; didn't work # search escape \t replace '\t', manually add [first, last] apo # 04Aug2022 don't waste more time - # do manually for now, after running pCsv_to_pYrFrac, # replace commas within quotes by a hyphen # CANNOT after-do [fit_SLRgress_TradingView, symbol_regressPlot_pPng] # # olde code tbl_raw|[0,] := (chr_apo EACHRIGHT link tbl_raw|[0,]) EACHLEFT link chr_apo ; # from "$d_webRawe"'economics, markets/SP500/PuetUWS/' # 'semi-log SP500 1870-2030 y-log detrend, y-data no model, digitizelt fixed.csv.txt' # renamed 'semi-log SP500 1872-2020 y-log detrend, y-data no model, digitizelt fixed.csv.txt' # pinn_backupDatedTo_zArchive (link d_webRawe 'economics, markets/SP500/semi-log SP500 1872-1940 regression fit.txt') >was argOrder problem from recent QNial changes, now is OK # FIRST must manually add titleApos to rawCsv files !!?!!!! # no workee : IF (sub_inTop_ary 'TradingView' symName_dateRange) THEN pUnixDate_to_pYrfrac ; ENDIF ; # sub_inTop_ary 'TradingView' 'DAX40 Jan1970-Jul2022 TradingView' qnial> sub_inTop_ary 'TradingView' 'DAX40 Jan1970-Jul2022 TradingView' o # ary_hasFaults (ary_sub_find_idx1stL 'TradingView' 'DAX40 Jan1970-Jul2022 TradingView') qnial> ary_sub_find_idx1stL 'TradingView' 'DAX40 Jan1970-Jul2022 TradingView' l ?itmNotInAry # ary_sub_get_slcLL 'DAX40 Jan1970-Jul2022 TradingView' 'TradingView' +-------------+------+ |?op_parameter|o----+| | ||[22]|| | |+----+| +-------------+------+ >> oops, no workee #08********08 #] 12Aug2022 fix upgrades to "$d_Qmkts" coding right now, I need revampONEregressPlot to work but there are many issues to address loaddefs link d_Qmkts 'Puetz detrended international indexes, gnuplot specs.ndf' +-----+ loading cmdSemiLog_create ?undefined identifier: := 'opening price monthly average' \ <***> \ N ( ?undefined identifier: CMDSEMILOG := CMDSEMILOG_CREATE <***> ; ?expecting expression: CMDDETREND := <***> CMDDETREND_CREATE ; loading subDir_symbol_yrL_regressSLplot ?expecting end of block: ; ?expecting an action: <***> } loading subDir_symbol_segYrL_calc_XareaPerYr ?expecting end of block: % ; <<< loading ended : Puetz detrended international indexes, gnuplot specs.ndf errors found: 6 +-----+ >> minor fixes, revamp again [semiLog, deTrend] gnuplot templates try again "$d_Qmkts"'stock index semi-log [regression, plot]s.ndf' FIRST - put apos around colTitles in each file!!! qnial> revampONE_CsvToTab 'CUS2 May2006-Aug2022 TradingView' cp: cannot stat '/mnt/ramdisk/pCsv_to_pTab 2.txt': No such file or directory >> what? pCsv_to_pTab 2.txt.txt >> must 'rm' existing file >> but output looks good qnial> pWrdChg /mnt/ramdisk/pTmplt_change_words.plt >> OK, a few more minor bugs, now works qnial> revampONE_CsvToTab 'GOLD Jan1834-Aug2022 TradingView' sed: can't read /home/bill/SG6/web/economics, markets/SP500/PuetzUWS intnl stocks/220812/rawCsv GOLD Jan1834-Aug2022 TradingView.txt: No such file or directory >> oops qnial> revampONE_CsvToTab 'GOLD Jan1834-Aug2022 TradingView' >> OK, went well OK revampONE_CsvToTab 'TNX 1963-Aug2022 TradingView' OK revampONE_CsvToTab 'USOIL Dec1861-Aug2022 TradingView' OK revampONE_CsvToTab 'WLFN Jan2007-Aug2022 TradingView' Now for the semi-log files : qnial> revampONE_regressPlot 'CUS2 May2006-Aug2022 TradingView' "/mnt/ramdisk/pTmplt_change_words.plt", line 0: Cannot open script file '/mnt/ramdisk/pTmplt_change_words.plt' >> produces output with '?Integer overflow', inall results >> it's not converting Unixtime to yrFrac? qnial> UnixDateToYrfracL +--------------------------------+--------- |CUS2 May2006-Aug2022 TradingView|DAX40 Jan1970-Jul2022 TradingView|GOLD Jan1834-Aug2022 TradingView|HSSSHSC30 +--------------------------------+--------------------------------- ~|~ +---------------------------------+---------- 0 Sep2020-Aug2022 TradingView|INDIA50 Mar2013-Jul2022 TradingView|NI225 Jan1984-Jul2022 TradingView|SHCOMP Jan -----------------------------+----------------------------------- ~|~ +---------------------------------+---------------------- 1970-Jul2022 TradingView|TNX 1963-Aug2022 TradingView|USOIL Dec1861-Aug2022 TradingView|UKX100 Apr1988-Jul2022 ------------------------+---------------------------- ~|~ ------------+--------------------------------+ TradingView|WLFN Jan2007-Aug2022 TradingView| ------------+--------------------------------+ >> 'pTmplt_change_words.plt' exists but from 08Aug2022 pUnixDate_to_pYrfrac IS OP symbol pUnixDate := link d_work 'tabSep ' symName_dateRange '.txt' ; pYrfrac := link d_work 'yrFrac ' symName_dateRange '.txt' ; tbl_raw := EACH str_wrapWithApos (pinn_readExecute_tabTbl pUnixDate) ; >> not good change : pinn_readExecute_tabTbl to: fit_SLRgress_TradingView 'CUS2 May2006-Aug2022 TradingView' >> runs, but semi-log file is garbage, nothing worked >> oops, forgot to run pUnixDate_to_pYrfrac I had to revamp that optr pUnixDate_to_pYrfrac 'CUS2 May2006-Aug2022 TradingView' revampONE_regressPlot 'GOLD Jan1834-Aug2022 TradingView' revampONE_regressPlot 'HSSSHSC300 Sep2020-Aug2022 TradingView' revampONE_regressPlot 'TNX 1963-Aug2022 TradingView' revampONE_regressPlot 'USOIL Dec1861-Aug2022 TradingView' revampONE_regressPlot 'WLFN Jan2007-Aug2022 TradingView' +-----+ olde code # olde code # pRegression := link d_work 'semi-log ' symDate ' regression fit.txt' ; # loaddefs link d_Qmkts 'stock index semi-log [regression, plot]s.ndf' IF flag_debug THEN write 'loading revampONE_regressPlot' ; ENDIF ; # revampONE_regressPlot IS OP symbol - for one-at-a-time work revampONE_regressPlot IS OP symbol { LOCAL coeffL symbol ; NONLOCAL UnixDateToYrfracL ; IF (symbol in UnixDateToYrfracL) THEN pUnixDate_to_pYrfrac symbol ; ENDIF ; fit_SLRgress_TradingView symbol ; symbol_regressPlot_pPng symbol ; } # revampONE_regressPlot 'CUS2 May2006-Aug2022 TradingView' # revampALL_regressPlot IS - from povrTab to yDetrend gnuplot, revamp results # (eg changes to [formula, format, etc]s) # cannot start with pCsv_to_pTab because QNial csv readfile not working # need code for titleApos # 21Jul2022 initial, 22J2022 fudge # NYET? All datasets in one shot, only AFTER manaul adjust of revampALL_CsvToTab : # can add apos to colTitles in 'rawCsv' files and merge this with revampALL_CsvToTab revampALL_regressPlot IS % NYET : apos manually BEFORE running this! ; % tbl_raw|[,0] := link [link chr_apo 'yrFrac' chr_apo] (EACH secFrom01Jan1970_to_yFrac unixTimeL) ; % ; % how to put apos around text in first line? ; % tabs WHITHIN title - very problematic, can't predict, must manually remove!@!! ; #08********08 #] 08Aug2022 yrL_regressPlotSP500 - process segSP500Ary of cart [1871-yrL, yrL-2022] yrL := 1920 + (5 * (tell 8)) 'stock index semi-log [regression, plot]s.ndf' fit_SLRgress_TradingView I need to change data source to d_data Change : pinn := link d_work 'yrFrac ' symbol '.txt' ; to : pinn := link d_data 'yrFrac ' symbol '.txt' ; 09Aug2022 #08********08 #] 08Aug2022 fix [time, price]Fracs Major changes to [timeFracLL_create_pTimeFracs, priceFracLLL_create_pPriceFracs] ; see subSection below Rerun : qnial> doAll_calc_timeFracLL_priceFracLLL qnial> doAll_create_pSubs timeFracLL priceFracLLL qnial> pTmplt_pSubs_create_pPlt qnial> create_pPng pPlt pPng Lots of more work, now it basically works +-----+ I need to automate : warning!! pTmplt_pSubs_create_pPlt produces an incomplete pPlt 1. copy-paste ALL of pCurves to pPlt (replace all content) 2. edit pPlt : insert 'plot \' just before curves, empty line after targetInsetPoint remove manually the ', \' at end of the last curve also - put date in fname for pPng >> done!! +-----+ Major changes to [timeFracLL_create_pTimeFracs, priceFracLLL_create_pPriceFracs] ; timeFracLL_create_pTimeFracs IS OP timeFracLL { LOCAL fout idxLL timeFrac timeFracL ; NONLOCAL colorSpikeL lwSpikeL pTimeFracs ; % ; fout := open pTimeFracs "w ; FOR idxLL WITH (tell (gage shape timeFracLL )) DO timeFracL := timeFracLL@idxLL ; FOR timeFrac WITH timeFracL DO writefile fout (link chr_tab 'set arrow from ' (string timeFrac) ', -3.0 to ' (string timeFrac) ', 6.0 nohead lw ' (string lwSpikeL@idxLL) ' linecolor "' colorSpikeL@idxLL '"') ; ENDFOR ; ENDFOR ; close fout ; } priceFracLLL_create_pPriceFracs IS OP priceFracLLL { LOCAL fout frac ffracL fracLL idxLL idxLLL ; NONLOCAL d_work colorPriceL lwPriceL pPriceFracs yrStrt yrEnd ; % ; % priceFracLLL has 2 subLists : [above, below] midIdx ; % must adapt [lwPriceL, colorPriceL]@idxLL to different shapes [above, below] ; % including nulls ; shapes := EACH (gage shape) priceFracLLL ; shapeMax := max shapes ; tellAbove := (tell shapes@0) + (shapeMax - shapes@0) ; tellBelow := (tell shapes@1) + (shapeMax - shapes@1) ; tellShapes := tellAbove tellBelow ; % ; fout := open pPriceFracs "w ; FOR idxLLL WITH (tell (gage shape priceFracLLL)) DO fracLL := priceFracLLL@idxLLL ; tellShape := tellShapes@idxLLL ; IF (~= null fracLL) THEN FOR idxLL WITH (tell (gage shape fracLL )) DO fracL := fracLL@idxLL ; idxLwColor := tellShape@idxLL ; FOR frac WITH fracL DO writefile fout (link chr_tab 'set arrow from ' (string yrStrt) ', ' (string frac) ' to ' (string yrEnd) ', ' (string frac) ' nohead lw ' (string lwPriceL@idxLwColor) ' linecolor "' colorPriceL@idxLwColor '"') ; ENDFOR ; ENDFOR ; ENDIF ; ENDFOR ; close fout ; } +-----+ olde code % extra little step just to insert command 'plot \' prior to curves ; % p[, PlotCmd, Plt, Png] are specified in : ; % '220730 Puetz2011 detrended international indexes, gnuplot specs.ndf' ; insertLine := '# PuetzUWS curvePlots (auto-added by QNial)' ; pTmplt_add_pSub pCurvesTmp pPlotCmd insertLine ; % warning!! ', \' added to last curve must be manually removed!! ; % warning!! put quotes around font in pTmplt ; % lastCurveLine := last host_result link 'grep chr_apo ' ', \\' chr_apo ' "' pPlt '" ' ; % fixeCurveLine := ary_sub_cut_subL lastCurveLine ', \\' ; 'gnuplot template, subFiles [create, insert].ndf' pTmplt_pSubs_create_pPlt change : pCurvesTmp := link d_temp 'pTmplt_insert_pCurves.txt' ; insertLine := '# PuetzUWS curvePlots (auto-added by QNial)' ; pTmplt_add_pSub pLinStyTmp pCurves pPlt insertLine ; To : insertLine := '# PuetzUWS curvePlots (auto-added by QNial)' ; pTmplt_add_pSub pLinStyTmp pCurves pPlt insertLine ; Instructions : change : #> +--+ #> doAll_calcFracs_to_pPlt #> +--+ #> for now - previous step works, but freezes (don't know why) #> create_pPng pPlt #> to : #> +--+ #> doAll_calcFracs_to_pPlt #> #08********08 #] Jun-Aug2022 revamp "$d_mkts"'SP500/PuetzUWS intnl stocks/' QNial code #08********08 #] 19Jul2022 minimum-manual auto-file adaptations : TVdata -> linear regression -> gnuplot see "$d_Qndfs""fit_linearRegress - notes.txt" 08********08 #] 07Jan2022 Formula for semi-logs "$d_Qndfs"'SP500 semi-log linear fit 2 points, other oddball calcs.ndf' SP500 - datapoints below are on the Fib, but not on SP500 curve, very approximate 2.3 yr data (weekly avgs) date 06Jan2020 06Dec2021 SP500 3175.4 4811.9 see results in ndf file ******** #] 20Sep2020 create composite images from "standard size" image outputs, criteria : (think of this later) only ONE image has axis labels? obviously use a standard yearFrac duration - much simpler graph generation (just specify start yearFrac, adjustable for searches) want to eliminate graph borders, titles, etc (except for one graph) # enddoc