
    hG                        d dl mZ d dlmZ d dlZd dlZd dlZd dl	Z	d dl
Zd dlZd dlZd dlZd dlmZmZ d dlmZmZmZ d dlmZmZmZmZ  G d d          ZdS )	    )requests)iscloseN)sharedutils)
_BASE_URL__PRICE_COLNAMES_
_SENTINEL_)YFInvalidPeriodErrorYFPricesMissingErrorYFTzMissingErrorYFRateLimitErrorc                   *   e Zd ZdefdZej        ddddddddddedddfdej        fd            Z	ddej        fd
Z
efdefdZd	efdej        fdZd	efdej        fdZd	efdej        fdZd	efdej        fdZddej        fdZej        dd            Zd ZddZej        d             Zej        d             Zej        d             Zej        d             Zej        d             Zej        d             Zej        d d            ZdS )!PriceHistoryNc                 B   || _         |                                | _        || _        |t          ur6t          j        dt          d           | j                             |           |pt          j
        d          | _        i | _        d | _        d| _        d | _        d S )N=Set proxy via new config function: yf.set_config(proxy=proxy)   
stacklevelchrome)impersonateF)_datauppertickertzr	   warningswarnDeprecationWarning
_set_proxyr   Sessionsession_history_cache_history_metadata_history_metadata_formatted_reconstruct_start_interval)selfdatar   r   r    proxys         W/var/www/html/marketdash/venv/lib/python3.11/site-packages/yfinance/scrapers/history.py__init__zPriceHistory.__init__   s    
llnn
""MY[mz{||||J!!%(((H("2x"H"H"H !%+0( ,0(((    1dFT
   returnc           
      $/  9 t          j                    }|t          ur6t          j        dt
          d           | j                            |           |}|}|	r|dv r|dk    rt          d          |x|u|r| j	        }|t          | j                  }t          |          }t          j                    t          j        | j        <   |                    dd	          d	         t          j        | j        <   |r||                    |           t          j                    S |d
k    rKt'          j        t*          j                                                            |          j        d	d	          }nrt*          j                                                            |                                          }|t          j        |          z  }|t'          j        d          z  }|}d}d}|}|}|s|s|r|                                dk    r| j	        }|t          | j                  }t          |          }t          j                    t          j        | j        <   |                    dd	          d	         t          j        | j        <   |r||                    |           t          j                    S |r6t          j        ||          }t=          |                                          }|r6t          j        ||          }t=          |                                          }||s|sd}nu|s:|t          j        d          z
  }t=          |                                          }n9|sRt*          j                                                            |          }t=          |                                          }n|                                dk    rO| t=          tA          j!                              }|*|dk    r|dz
  }n|dv r|dz
  }n|dv r|dz
  }n|dz
  }|dz  }n}|r|rtE          d          |s|rft          j        |          }|&||z   }t=          |                                          }|&||z
  }t=          |                                          }d}|s|r||d}n|                                }d|i}|                                |d<   ||d<   |d         dk    rd|d<   d|d<   tG          |          }| j	        }dD ]X}||v rRt          t+          j        ||         d !          $                    d"                              |                    ||<   Y|%                    | j         d#t          |                      tL           d$| j         }d}| j        j'        }|qt+          j        |d !          $                    d"          }t*          j                                        } t'          j        d%&          }!||!z   | k    r| j        j(        }	  ||||'          }d(|j)        v s|tU          d)          |+                                }n# tX          $ r  t          $ r |r Y nw xY w	 |d*         d+         d,         d-         | _-        n# t          $ r
 i | _-        Y nw xY w|d         d.         d/v }"d0}#|s||                                dk    rW|#d1|d          d2z  }#||#| z  }#n|"sS|#t+          j        |d !          $                    d"                              |                                           z  }#n@|#t+          j        |d !          $                    d"                              |           z  }#|#d3z  }#|	|#| d4z  }#n|"sU|#t+          j        |d !          $                    d"                              |                                           d4z  }#nL|#t+          j        |d !          $                    d"                              |           d4z  }#n	|#d5| d4z  }#d6}$|t]          |tF                    st_          | j        |#          }d7}$nAt]          |tF                    r,d8|v r(|#d9|d8          d4z  }#t_          | j        |#          }d7}$n d*|v rC|d*         d:         r5|#d;|d*         d:         d<         z   d=z   z  }#t_          | j        |#          }d7}$nd*|vsB|d*         d+         4|d*         d+         r&|d*         d+         d,         d>         d?         d,         st_          | j        |#          }d7}$n[|rY|| j-        d@         vrJt          j0        |          s6tc          | j        |dA2                    | j-        d@                             }d7}$|$rt          |          }t          j                    t          j        | j        <   |                    dd	          d	         t          j        | j        <   |r||                    |           | j3        | j3        |k    rd| _3        t          j                    S | j-        dB         }%|%dCv }&| j-        dD         }'| j-        dE         }(|r|| j-        '                    d@g           vrt=          tA          j!                              }t+          j        |d !          $                    d"          }t&          j        4                    |          }|t          j        |          z  }|t'          j        d          z  }t          j5        |d*         d+         d,                   })|r^|)j6        sW|)j7        d.         |                    d"          $                    d          k    r |)8                    |)j7        d.                   })|)j6        r| j         dF}*nQts          |)          d	k    r| j         dG|)j7        d,          dH}*n%| j         dG|)j7        d,          d3|)j7        d.          }*|%                    |*           |                                dk    rB|%                    | j         dI           |):                    dJ          }+t+          j;        |+<                                j7        |+dK         =                                |+dL         >                                |+dM         ?                                |+dN         <                                |+dO         <                                |+dP         @                                dQR          })	 |+dS         >                                |)dS<   |+dT         >                                |)dT<   n# t          $ r Y nw xY wt          jA        |)|d         |'          })t          jB        |)|d                   })|d         d.         d/v }"|s|"rdU| j-        v rw| j-        dU         },t]          |,t*          j;                  s4t          jC        | j-        d7V          | _-        d7| _D        | j-        dU         },t          jE        |)|d         |,          })|)j6        r| j         dW}*nQts          |)          d	k    r| j         dX|)j7        d,          dH}*n%| j         dX|)j7        d,          d3|)j7        d.          }*|%                    |*           t          jF        |d*         d+         d,                   \  }-}.}/|&sd}/|.t          jA        |.||'          }.|-t          jA        |-||'          }-dE|-jG        v r| j-        dE         }0|0d0}0|-dE         |0k    }1|1H                                r|	r|0d0k    r0|-dS         I                    t                    d2z   |-dE         z   |-dS<   nd| J                    |-|0|	          }-|-dE         |0k    H                                r/|-dS         I                    t                    d2z   |-dE         z   |-dS<   |-8                    dEd	Y          }-|/t          jA        |/||'          }/|Z|)j6        sS|)j7        d,         K                    dZ          }2|-|-jL        |2d         }-|/|/jL        |2d         }/|.|.jL        |2d         }.|;|t+          jM        d	          z
  }3|-
|-d|3         }-|/
|/d|3         }/|.
|.d|3         }.|d         d.         d/v }"|"st+          jN        |)j7        j                  $                    |'d7d[\          |)_7        |-jO        d,         d,k    r9t+          jN        |-j7        j                  $                    |'d7d[\          |-_7        |.jO        d,         d,k    r9t+          jN        |.j7        j                  $                    |'d7d[\          |._7        |)P                                9|-jO        d,         d,k    rt          jQ        9|-|          9dS9jG        v r%d,9jL        9dS         R                                dSf<   nd]9dS<   |.jO        d,         d,k    rt          jQ        9|.|          9dT9jG        v r%d,9jL        9dT         R                                dTf<   nd]9dT<   |&rZ|/jO        d,         d,k    rt          jQ        9|/|          9d^9jG        v r%d,9jL        9d^         R                                d^f<   nd]9d^<   9j6        r| j         d_}*nQts          9          d	k    r| j         d`9j7        d,          dH}*n%| j         d`9j7        d,          d39j7        d.          }*|%                    |*           t          jS        9|d         |'||	|(a          \  9}4|4|4dN         |4jT        db| j-        dc<   99j7        U                    dde                    9|	r4|%                    | j         df           9P                                9| V                    9|(          \  9}(|(| j-        dE<   | W                    9||(          99j6        sg| X                    9jY        d.d         ||'|          }5dg9jG        vrd69dg<   t+          jZ        98                    9j7        d.                   |5g          9| [                    9||'|          9| \                    9||'          9| X                    9||'|          99P                                9	 |rt          j]        9          9n|rt          j^        9          9n# t          $ r}6|rdh|6z  }ndi|6z  }t          j                    t          j        | j        <   |t          j        | j        <   |rt          | j        d|          |                    | j        d|           Y d}6~6nd}6~6ww xY w|r3t          j`        9|d*         d+         d,         d-         dj                   99dP         a                    d,          I                    t          jb                  9dP<   |"rdk9j7        _T        ndl9j7        _T        |s98                    g dmdno          9|
syt          dPgz   g dmz   }79fdp|7D             }79|7         R                                9|7         d,k    z  d                    d	Y          }898                    |8j7        |8                   9||k    r| e                    9|||          99j6        r| j         dq}*nQts          9          d	k    r| j         dr9j7        d,          dH}*n%| j         dr9j7        d,          d39j7        d.          }*|%                    |*           | j3        | j3        |k    rd| _3        9S )sa4  
        :Parameters:
            period : str
              | Valid periods: 1d,5d,1mo,3mo,6mo,1y,2y,5y,10y,ytd,max
              | Default: 1mo
              | Can combine with start/end e.g. end = start + period
            interval : str
              | Valid intervals: 1m,2m,5m,15m,30m,60m,90m,1h,1d,5d,1wk,1mo,3mo
              | Intraday data cannot extend last 60 days
            start : str
              | Download start date string (YYYY-MM-DD) or _datetime, inclusive.
              | Default: 99 years ago
              | E.g. for start="2020-01-01", first data point = "2020-01-01"
            end : str
              | Download end date string (YYYY-MM-DD) or _datetime, exclusive.
              | Default: now
              | E.g. for end="2023-01-01", last data point = "2022-12-31"
            prepost : bool
              | Include Pre and Post market data in results?
              | Default: False
            auto_adjust : bool
              | Adjust all OHLC automatically?
              | Default: True
            back_adjust : bool
              | Back-adjusted data to mimic true historical prices
            repair : bool
              | Fixes price errors in Yahoo data: 100x, missing, bad dividend adjust.
              | Default: False
              | Full details at: :doc:`../advanced/price_repair`.
            keepna : bool
              | Keep NaN rows returned by Yahoo?
              | Default: False
            rounding : bool
              | Optional: Round values to 2 decimal places?
              | Default: False = use precision suggested by Yahoo!
            timeout : None or float
              | Optional: timeout fetches after N seconds
              | Default: 10 seconds
            raise_errors : bool
                If True, then raise errors as Exceptions instead of logging.
        r   r   r   )5d1wk1mo3mor/   z<Yahoo's interval '5d' is nonsense, not supported with repairN:    ytd   daysr+   maxr1   1mi 
 )2m5m15m30m90mi O )1h60mi gl   j-t zASetting period, start and end is nonsense. Set maximum 2 of them.)period1period2rangeintervalincludePrePostr>   r=   zdiv,splits,capitalGainseventss)unitUTCz: Yahoo GET parameters: z/v8/finance/chart/   )minutes)urlparamstimeoutzWill be right backz~*** YAHOO! FINANCE IS CURRENTLY DOWN! ***
Our engineers are working quickly to resolve the issue. Thank you for your patience.chartresultr   metamh  (  -> )z	 (period=FTstatus_codez(Yahoo status_code = errorz (Yahoo error = "descriptionz")
indicatorsquotevalidRanges, instrumentType)
MUTUALFUNDETFexchangeTimezoneNamecurrencyz$: yfinance received OHLC data: EMPTYz: yfinance received OHLC data: z onlyz: resampling 30m OHLC from 15m30minOpenHighLowClose	Adj CloseVolume)ri   rj   rk   rl   rm   rn   )indexr&   	DividendsStock SplitstradingPeriods)tradingPeriodsOnlyz: OHLC after cleaning: EMPTYz: OHLC after cleaning: axisDshift_forward)	ambiguousnonexistent        Capital Gainsz$: OHLC after combining events: EMPTYz: OHLC after combining events: )repairrg   )PriceTime	lastTradefirst)keepz: checking OHLC for repairs ...	Repaired?zauto_adjust failed with %szback_adjust failed with %s	priceHintDatetimeDate)rp   rq   r{   ignore)columnserrorsc                 &    g | ]}|j         v |S  r   .0cdfs     r(   
<listcomp>z(PriceHistory.history.<locals>.<listcomp>  s     III1bjQr*   z : yfinance returning OHLC: EMPTYz: yfinance returning OHLC: )fr   get_yf_loggerr	   r   r   r   r   r   	Exceptionr   r   r   strempty_dfr   _DFSsplit_ERRORSr]   	_datetimedatepd	Timestamputcnow
tz_convertyear_interval_to_timedelta	timedeltalower_parse_user_dtint	timestamp_timetime
ValueErrordicttz_localizedebugr   get	cache_gettextRuntimeErrorjsonr   r"   
isinstancer   is_valid_period_formatr
   joinr$   fromtimestampparse_quotesemptyro   droplenresample	DataFramelastr   r9   minsum	set_df_tzfix_Yahoo_dst_issueformat_history_metadatar#   'fix_Yahoo_returning_prepost_unrequestedparse_actionsr   anyastype_dividends_convert_fxfloorloc	Timedeltato_datetimeshape
sort_indexsafe_merge_dfsisna!fix_Yahoo_returning_live_separatename
duplicated_standardise_currency_fix_bad_div_adjust_fix_zeroesilocconcat_fix_unit_mixups_fix_bad_stock_splitsauto_adjustback_adjustnproundfillnaint64r   all	_resample):r%   periodrE   startendprepostactionsr   r   r|   keepnar'   roundingrO   raise_errorsloggerinterval_userperiod_userr   
_exceptionerr_msg
start_userend_userstart_dtend_dt	period_tdrN   params_prettykrM   r&   get_fndt_now
data_delayintraday_price_data_debugfail
quote_typeexpect_capital_gainstz_exchangerg   quotesmsgquotes2tps	dividendssplitscapital_gainsprice_currencyf_currency_mismatchstart_dend_dt_sub1
last_tradedf_lastedata_colnamesmask_nan_or_zeror   s:                                                            @r(   historyzPriceHistory.history!   s}   ^ $&&
""MY[mz{||||J!!%(((  	h"=== 4 ^___}1CW:!1$+!>!>J!*ooG/4~/?/?FK,29--a2H2H2KFN4;/# .((W--- >+++U??%N2<+>+>+@+@+K+KB+O+O+TVWYZ[[EEL//11<<R@@EEGGEU9&AAAEY0a8888E$H
 	(C 	(F 	(v||~~'>'>Bz-dk::
j//+0>+;+;DK(.5mmD!.D.DQ.Gt{+ *$$LL)))~''' 	.+E266H**,,--E 	*)#r22Ff&&(())C> .S . .!E$@$G$GGH..0011 .,,..99"==&**,,--||~~&&;ejll++C=4'' #f!%FFF #g!]22 #h #j 0QJE 
3 
 !deee # !8@@	;%	1Ff..0011C=%	1H 2 2 4 455E 	'C 	'!&377FF\\^^Fv&F%^^--z#*  *&&!&F: 5xVW' 	l 	lAM!!#&r|F1IC'H'H'H'T'TUZ'['['f'fgi'j'j#k#ka QQS=O=OQQRRR <<t{<<?\#C000<<UCCF\((**F",R888J
"f,,-	6  D
 $ty00DL" $M N N N 99;;DD 	 	 	 	 	 	  	
	(%)']8%<Q%?%GD"" 	( 	( 	(%'D"""	( *%b)Z7 	7FNfllnn&=&=!;fZ&8!;!;!;;%!
_4!! j!U(E(E(E(Q(QRW(X(X(c(cdf(g(g(l(l(n(n%pp!!!U(E(E(E(Q(QRW(X(X(c(cdf(g(g%ii!'#!^^^3!! i!Ss(C(C(C(O(OPU(V(V(a(abd(e(e(j(j(l(l%o%o%oo!!!Ss(C(C(C(O(OPU(V(V(a(abd(e(e%h%h%hh!!!6V!6!6!66<z$55<-dk;LMMJDDd## 	(=(=!Om9L!O!O!OO-dk;LMMJDD__gw!7_!4tG}W7Mm7\!\_c!cc-dk;LMMJDDD  DM($;$C4PW=YaKb$Cjnovjw  yA  kB  CD  kE  FR  kS  T[  k\  ]^  k_$C-dk;LMMJDD 	d&<]&KKKTYTpqwTxTxK-dk6499TMcdqMrCsCsttJD 
	$*ooG',~'7'7FK$*1--a*@*@*CFN4;' &  W%%%/;@`dl@l@l370>### +,<=
)-BB,-CD)*5  	1fD$:$>$>}b$Q$QQQejll##C\#C000<<UCCFN0055EU1&999EY(a0000E #DM($;A$>?? 	7v| 	7|B6#4#4U#;#;#G#G#M#MMMV\"%566< 	i[FFFCC[[A[WWaWWWCC[hhahhV\VbceVfhhCS >>u$$LLDKGGGHHHoog..G\(<--//++--u~))++ )..00$[16688!(+//11D D   F&-k&:&>&>&@&@{#)0)@)D)D)F)F~&&    
);[II*66*3EFF*%b)Z7 	d8 	d(8D<R(R(R()9:Cc2<00 ?).)FtG]rv)w)w)w&370,-=>B66R\K]_bccF< 	a[>>>CC[[A[OOaOOOCC[``a``fl[]N^``CS ,1+>tG}X?VWX?Y+Z+Z(	6=# 	! M_VX{CCF 	8[IIIY...!%!7
!C!)%'N&/
&;~&M#&**,, 	v! v^r%9%91:;1G1N1Ns1S1SVY1Y\efp\q1q	+.. %)$>$>y.Z`$a$a	%j1^CHHJJ v5>{5K5R5RSV5W5WZ]5]`ijt`u5uIk2%NN:AN>>	$!OM8[QQM< 2 ,q///44( )ghh 7I ,$1$5ghh$?M%#Z1F? 2<??2K$%l{l3	( -l{l ;!- *%b)Z7 	G >&,*;<<HH`d  sBH  C  CFLq!A%%"$.1E"F"F"R"RS^jn  }L"R  #M  #M	|A""!~fl.?@@LL[dh  wFL   G   G   ?1!!%b)X>>B"*$$:;BF2k?''));677!B{O<?Q%b&(;;BRZ''@ABF2n%**,,n<==!$B~ 	*"1%)))"mXFF"*,,FGr/*//11?BCC&)?#8 	a[FFFCCWW\\[SS!SSSCC[``!``RTRZ[]R^``CS@VJEWYdfmv|  HP  Q  Q  QJ!;Eg;NWaWf2g2gD";/$$'$2223 	!LLDKHHHIIIB
  55b(CCLB19D":.))"hAAB 8 A**27233<;PWXXbj00&+B{OY 5 5w?@@&&r8['JJB++B+FFB!!"hWEEBB	@ +&r** +&r** 
	@ 
	@ 
	@ ;6:6:',~'7'7FK$*1FN4;' @DKKK ABBBgg>????????
	@  	O"d7mH5a8@MNNB(|**1--44RX>>8 	#&BHMM"BHM  	b!O!O!OX`aaB 	C,z9<j<j<jjMIIIIIIIM "= 1 6 6 8 8B}<MQR<R SXX^_X``)/0@ABBB}$$Hm[IIB8 	][BBBCCWW\\[OORXa[OOOCC[\\RXa[\\bhWYl\\CS+7D<\`h<h<h/3D,	sW   2<Y/ /Z	Z	Z- -[ [:v 
v&%v&S;-AT) T)
AV=T3B AV8V8AV=r9   c                 ~    ||f}|| j         v r| j         |         S |                     ||d          }|| j         |<   |S )NT)r   rE   r   )r!   r  )r%   r   rE   	cache_keyr   s        r(   _get_history_cachezPriceHistory._get_history_cache  sQ    v&	+++&y11\\(D\II)+I&	r*   c                 8   |t           ur6t          j        dt          d           | j                            |           | j        	d| j        vr|                     dd           | j        du r%t          j
        | j                  | _        d	| _        | j        S )
Nr      r   rr   r/   r@   )r   rE   FT)r	   r   r   r   r   r   r"   r  r#   r   r   )r%   r'   s     r(   get_history_metadataz!PriceHistory.get_history_metadata  s    
""MY[mz{||||J!!%(((!)-=TE[-[-[##4$#???+u44%*%B4CY%Z%ZD"/3D,%%r*   c                    |t           ur6t          j        dt          d           | j                            |           |                     |          }d|j        v r|d         }||dk             S t          j	                    S )Nr   r  r   r   rp   r   
r	   r   r   r   r   r   r  r   r   Series)r%   r   r'   r   r   s        r(   get_dividendszPriceHistory.get_dividends#  s    
""MY[mz{||||J!!%((($$F$33"*$$;IY!^,,y{{r*   c                    |t           ur6t          j        dt          d           | j                            |           |                     |          }d|j        v r|d         }||dk             S t          j	                    S )Nr   r  r   r  r{   r   r  )r%   r   r'   r   r  s        r(   get_capital_gainszPriceHistory.get_capital_gains.  s    
""MY[mz{||||J!!%((($$F$33bj((/M !!344y{{r*   c                    |t           ur6t          j        dt          d           | j                            |           |                     |          }d|j        v r|d         }||dk             S t          j	                    S )Nr   r  r   r  rq   r   r  )r%   r   r'   r   r  s        r(   
get_splitszPriceHistory.get_splits9  s    
""MY[mz{||||J!!%((($$F$33RZ'''F&A+&&y{{r*   c                    |t           ur6t          j        dt          d           | j                            |           |                     |          }g }d|j        v r|                    d           d|j        v r|                    d           d|j        v r|                    d           |r;||         }||dk             	                    d	
          
                    d          S t          j                    S )Nr   r  r   r  rp   rq   r{   r   r   )how)r	   r   r   r   r   r   r  r   appenddropnar   r   r  )r%   r   r'   r   action_columnsr   s         r(   get_actionszPriceHistory.get_actionsD  s   
""MY[mz{||||J!!%((($$F$33"*$$!!+...RZ''!!.111bj((!!/222 	E(G7a<(//E/::AA!DDDy{{r*   c                 N   ||k    r|S d }|dk    rd}n|dk    rd}nv|dk    rd}nm|dk    rTd}|d	k    rd
}nCt           j                                                            d                                          }d| }nt          d| d          dddddddd}d|j        v rd|d<   d|j        v r|d         |d<   d|j        v rd|d<   d|j        |d         dk    df<   |                    |dd|          	                    |          }	d|	j        |	d         dk    df<   |	S ) Nr0   zW-MONr/   5Dr1   MSr2   QSr5   JANz%bzQS-z(Not implemented resampling to interval ''r   r   r9   r   r   prod)ri   rk   rj   rl   rn   rp   rq   r   r   rm   rl   r{         ?rq   rz   left)labelclosedoffset)
r   datetimenowstrftimer   r   r   r   r   agg)
r%   r   df_intervaltarget_intervalr   r-  resample_periodalign_monthresample_mapdf2s
             r(   r   zPriceHistory._resampleX  s   /))Ie##%OO$$"OO%%"OO%%"O#'04466??EEKKMM1K11OOYYYYZZZE56%  "*$$(-L%"*$$(4W(=L%bj((,1L):=r.!3&67kk/vkVVZZ[ghh<?N#S(.89
r*   rS   c                 &+  O t          j                    }d| j        d}t          t          j                  st          dt                              dk    rS dd          dv rd}d}nd}fd	t          D             }|d
gz   }	g dOfdOD             }
Ofdt          t          O          dz
            D             }d d t          j        d          d}dD ]}t          j        d          ||<   t          j        d          |d<   |v r|         }|
         }n)|                    d d           dj        vrdd<   S | j        | _        | j        k    rQ|| j                 k    r@d                    | j        || j                           }|                    ||           S                                 |	                                         |k    }|                    d          }||         }|d }nk|t          j        d          z  }t          j                                        |z
  }|                    j        j                                      d          }|                    d| d d| |           |wj        |k    }||z  }|                                sSdt9          j        |                    d                     d}|                    ||           dj        vrdd<   S j        |         }t          |          d k    r'|                    d!|           dj        vrdd<   S                                 }d|j        vrd|d<   |                                                             d           }||                                         |k                         d          z  }|         }|d          gg}|d"k    r!tB          j"        "                    d#$          }n|d%k    r!tB          j"        "                    d#$          }n|d&k    r!tB          j"        "                    d#$          }nX|d'k    r!tB          j"        "                    d$          }n1|dk    rt          j        d(          }nt          j        d          }|                    d)| |           t          dt          |                    D ]u}||         }|#                                |d*         d          #                                |z   k     r|d*         $                    |           _|$                    |g           v|                    d+|           |D ])}|                    d,|d           d-|d*                     *t          t          |                    D ]_}||         }|d          }|j        %                    |gd./          d          }|d k    rU||j        |dz
           |k    r?|r8|j        |dz
           #                                |#                                k    r|dz  }|d*         } |j        %                    | gd./          d          }!|!t          |          dz
  k     r?|r8|j        |!dz            #                                | #                                k    r|!dz  }!|j        ||!dz            }"||xx         |"&                                z  cc<   ||         '                                 ad }#|D ]Y}j        (                    |                   }$|                    d0tS          |$          z              |d          }%|%#                                }&d}'|d'k    r;t          j#        *                                |&z
  t          j        d1          k    rd}'n>|d2v r:t          j#        *                                |&z
  t          j        d3          k    rd}'|'r#d4|r|%n|& d5}|                    ||           
t          j        d          }(d%v r#|&|z
  })|d*         #                                |z   }*n9d&k    r |&})|d*         #                                |z   }*n|d          })|d*         |z   }*|)|(z  })|*|(z  }*|r+|)#                                })|*#                                |(z   }*|"tW          |#                                |)          })|                    d6| d7| d8|) d9|* |           t          j                    }tY          |d:          r&|j-        }+|.                    t^          j0                   | 1                    |)|*|dd|dd;          },tY          |d:          r|.                    |+           |,|,j2        r#d4|r|%n|& d5}|                    ||           |,j3        |d          |d*         |
|         z   t          j        d<          z
                                           },|,j2        r&d= d>|r|%n|& d?}|                    ||           -d |,d@<   d%k    rmg dA}-|-|$j        d          4                                dBz   dz
  dBz           }.|,j        5                    d           6                    dC|.z             j7        |,dD<   dD}/nd&k    r$t	          j8        |,j        j#                  |,dE<   dE}/n_d|,j3        |,j        (                    |$j                  d@f<   |,d@         9                                |,dF<   |,:                    d@d          },dF}/|,|,|dGgz                                                                 d                    },|,;                    |/          }0|0<                    dHdIdJdKdLdMdNO          =                    dPdQiR          }1|/dSv r*|1j        5                    |,j        j                  |1_        nl|,dF         >                                |,dT<   t9          j$        |,j        d          g|,j        |,dF         >                                d k                       }2|2|1_        |                    dUtS          |1          z              t9          j?        |$j        |1j                  }3t          |3          d k    r"dV dW|& dX}|                    ||           d&k    r+|1|1j        (                    |3                   }4|$|$j        (                    |3                   }5|5dQ         |k    }6|6                                r|5dQ         |5dY         z  }7t8          j@        |7|6<   |7                                                                 s|7A                                B                                }7t9          jC        |6          d          D ].}8|4j        |8         }t          |7          }9|1j3        |dGf         d k    rk|8|9dz
  k     r|7jD        |8dz            |7jD        |8<   TdZ|4dG         jD        |8         |4dY         jD        |8dz
           z  z
  }:|7jD        |8dz
           |:z  |7jD        |8<   |8d k    r|7jD        |8dz
           |7jD        |8<   |7jD        |8dz            |7jD        |8<   |4dG         jD        |8dz            d k    r@|7jD        |8xx         dZ|4dG         jD        |8dz            |4dY         jD        |8         z  z
  z  cc<   0|5dY         |k    };|7E                    |$j        t8          j@        [          A                                B                                }7|$dY         |7z  |1dQ<   |;                                r|;E                    |1j        d[          }<|7E                    |1j        t8          j@        [          A                                B                                }=|<                                }>|1dY         |>         |=|>         z  |1j3        |>dQf<   d\dYg}?|1|1j        (                    |3                   |?                                         }4|$|$j        (                    |3                   |?                                         }5|5|k    }@|@t9          jF        |4           z  }@|@                                s|                    d]|& dX|           	t          t          |?                    D ]1}A|@d d |Af          }B|B                                rd|5|B|Af<   d|4|B|Af<   2|5|@         |4|@         z  }C|0G                                }D|1j        |D_        |D|Dj        (                    |3                                                   H                    t                    }D|Dd d d f         }Dt9          jJ        |Dt          |?                    }D|D|@         }Dt9          jK        |CdZd^_           }Et9          j        |E          t          |?          k    rdZ}Fnt9          jL        |C|D`          }Ft          |Fdaz  dz
            dbk     rD|$                                }$t          D ]#}G|j3        ||G         |k    |Gfxx         dcz  cc<   $|Fdcz  }F|                    dd|Fde|           t          dZ|Fz  d          }Ht          |Fd          }F|Fdk    r|Hdk    rnP|Fdk    r!|1|xx         |Fz  cc<   |1d
xx         |Fz  cc<   n)|Hdk    r#|1|xx         dZ|Hz  z  cc<   |1d
xx         |Hz  cc<   |$j        |$|d
gz            |k                                                        d                   }Ig }J|ID ] }8|8|1j        vr|J$                    |8           !t          |J          d k    r'|                    dftS          |J          z   |           |ID ]}8|8|1j        vr|1j3        |8         }Kd%k    r7|1jD        |1j        O                    |8          dz
           }L|,j3        |8d          },j3        |8         }M|Mj        |M|k                                             }Ndg|Nv r|Kdg         |j3        |8dgf<   dh|Nv r|Kdh         |j3        |8dhf<   d\|Nv rpd%k    rX|8|,j        d          k    rG|LdY         |j3        |8d\f<   t          |j3        |8d\f         |j3        |8dhf                   |j3        |8dhf<   n|Kd\         |j3        |8d\f<   dY|Nv r%|KdY         |j3        |8dYf<   |KdQ         |j3        |8dQf<   ndQ|Nv r|KdQ         |j3        |8dQf<   d
|Nv r7|Kd
         N                                H                    di          |j3        |8d
f<   d|j3        |8df<   |#dz  }#[|S )jNzprice-reconstructyf_catyf_interval	yf_symbolz#'df' must be a Pandas DataFrame notr:   r4   )dwkmoTFc                     g | ]}|v |	S r   r   r   s     r(   r   z=PriceHistory._reconstruct_intervals_batch.<locals>.<listcomp>  s    ===AQ"WWaWWWr*   rn   )r0   r+   r@   r>   r=   r<   r;   r:   c                 :    i | ]}|t          j                  S r   )r   r   )r   irE   s     r(   
<dictcomp>z=PriceHistory._reconstruct_intervals_batch.<locals>.<dictcomp>  s&    MMMa5/99MMMr*   c                 4    i | ]}|         |d z            S )r4   r   )r   rB  	intervalss     r(   rC  z=PriceHistory._reconstruct_intervals_batch.<locals>.<dictcomp>  s(    SSSA1yQ/SSSr*   i  r7   )r0   r+   r@   )r>   r=   r<   r;   <   rK   z,Have not implemented price reconstruct for 'z' interval. Contact developersr   z%Hit max depth of 2 ('{}'->'{}'->'{}')extrart   rv   zmin_dt=z
 interval=z sub_interval=z	Too old (z rows tagged)r   z/Nothing needs repairing (dts_to_repair[] empty)r1      yearsr0   r+   r@   r   zgrp_max_size = rS   zRepair groups:z- rZ   nearest)methodz
df_block:
i  )r>   r=   ;   z"Cannot reconstruct block starting z9, too old, Yahoo will reject request for finer-grain dataz	Fetching z	 prepost=rY   ->level)r   r   rE   r   r   r   r|   r   )millisecondszCannot reconstruct z block range z3, Yahoo not returning finer-grain data within rangectr)MONTUEWEDTHUFRISATSUN   zW-
Week Start	Day Start
intervalIDrp   )ri   r   )rl   r   )rm   r   )rk   r   )rj   r9   )rp   r   )rn   r   )ri   rl   AdjCloserk   rj   rp   rn   r^  rm   r   )r[  r\  diffzdf_new:
zCan't calibrate z block starting z so aborting repairrl   r)  
fill_valueri   zCan't calibrate block starting gh㈵>)rtol)weightsg-C6?{Gz?d   z Price calibration ratio (raw) = 6fz:Yahoo didn't return finer-grain data for these intervals: rj   rk   r   )Qr   r   r   r   r   r   r   typer   rD   r   r   r   warningr   r$   formatinfor   to_numpyr   r   r   r   ro   r   ceilr   r   r   copyr   r   	_dateutilrelativedeltar   r  get_indexerto_listsortisinr   todayr9   hasattrrP  setLevelloggingCRITICALr  r   r   weekdayr   	to_period
start_timer   cumsumr   groupbyr1  renamer_  intersect1dnanffillbfillwherer   reindexisnansizer   floattiler   averageabsr   get_locr   )Pr%   r   rE   r   tagr   
log_extrasr   
price_cols	data_colsitdsnextsmin_lookbacksrB  sub_intervaltd_ranger   f_repairf_repair_rowsrU   min_dtf_recentdts_to_repairdf_v2f_gooddf_good
dts_groupsgrp_max_sizedtgg0i0glilgood_dtsn_fixeddf_blockr   r  rejecttd_1dfetch_start	fetch_end	log_leveldf_fineweekdaysweek_end_daygrp_coldf_fine_grpdf_new	new_indexcommon_indexdf_new_calibdf_block_calibf_tagdiv_adjustsidxndiv_adjf_close_badf_close_bad_newdiv_adjusts_newdiv_adjusts_new_np
calib_colscalib_filterjfratiosrc  not1ratior   	ratio_rcpbad_dtsno_fine_data_dts
df_new_rowdf_last_week
df_bad_row
bad_fieldsrE  sP    ``                                                                            @r(   _reconstruct_intervals_batchz)PriceHistory._reconstruct_intervals_batch{  s    $&& 3H[_[fgg
"bl++ 	MA488LLLtIABB<,,,GHHH====!1===
(+	 HGG	MMMM9MMMSSSSs9~~PQ?Q9R9RSSS $D	8KQT8U8U8UVV+ 	< 	<A(2;;;M!'1r:::du ?LH~HHNNr(rrrsss"*,,"';I +3/7D,t777HdNnHo<o<o9@@Aachim  jJ  dK  MU  V  VCKK:K...I]]__i=))++s2 !,, ,'9FF$!,,,,A\((**Q.F&&rx{3388==FWvWWWWWW_ijjjx6)H)H4M $$&& M"&1)=)=">">MMMCz222bj00&+B{O	/}""LLJR\L]]]"*,,"';I		em++!&E+j>&&((,,!,4452j>2244;@@a@HHHV* %Q'()
5  $2@@q@IILLU""$2@@q@IILLT!!$2@@q@IILLT!!$2@@q@IILLT!!$.A666LL$.B777L5|55ZHHHq#m,,-- 	( 	(Aq!Bwwyy:b>!,1133lBBB2%%b))))!!2$''''%Z888 	1 	1ALL/ad//"//0000 s:'' 	! 	!A1A1B**B4	*BB1EBAvvNgmBF&;v&E&E& 'F+2=a+@+E+E+G+G27799+T+T!GB2B**B4	*BB1EBCLL1$$$  W]26%:%?%?%A%ARWWYY%N%N!GB}RQY/HqMMMX--///MMMqM     s	 s	A"(--**+HLLX6777tHmmooGFt##)=)=)?)?')IYM`fiMjMjMj(j(j//Y^5I5I5K5Kg5UYbYlrtYuYuYu4u4u  Xx;\88U\  X  X  XCz222'Q///E5  %0bEJJLL83		T!!%bEJJLL83		dbEH,	 5 KI 5)..00%NN,,u4	!!&++--==LL`\``G``k``U^``hrLsss(**Fvw'' 2"L	 0111ll)lhmw{  FM  VZ  cgl  h  hGvw'' +	***'- Xx;\88U\  X  X  XCz222k!A$"\0B(BYEXfgEhEhEh(h"hinnppG}  ZH  Z  ZxCd88]d  Z  Z  ZCz222GEN5  LLL'):)B)B)D)Dq)H1)LPQ(QR(/(A(A$(G(G(Q(QRVYeRe(f(f(q%&T!!')~gm6H'I'I$%IJGM..x~>>EF(/(=(=(?(?%!,,u1,55&wz[M'ABGGIIMMSTMUUUVG!//'22K __&'."$.( % * * +1&*k9R&*S*S  555%|778HII"),"7"<"<">">Iw}Q'7&8'-P\H]HbHbHdHdghHh:ijj	(LL)CKK7888 >(.&,GGL<  A%%___7___Cz222 4%fl&7&7&E&EF!)(.*=*=l*K*K!L&{3s:99;; 'T"0"=w@W"WK
 *,K&&++--1133  T&1&7&7&9&9&?&?&A&A#%8E??1#5 \ \C!-!3C!8B #K 0 0A%z"k/:a??#&Q;;<G<LSSTW<UK$4S$9$9 /2L4M4RSV4WZfgnZoZt(+a[1 51 /1G<G<LSSTW<UX_<_K$4S$9$9#&77<G<LSSTW<UK$4S$9$9 =H<LSSTW<UK$4S$9'3K'@'EcAg'NRS'S'S(3(8(=(=(=|T_G`GefilmfmGn<H<Q<VWZ<[H\ B\ )\(=(=(=&4W&=&D&1&9&9(.UWU[&9&\&\&b&b&d&d&j&j&l&l.6w.?+.M{+&??,, T.9.A.A&,[`.A.a.aO.9.A.A&,[][a.A.b.b.h.h.j.j.p.p.r.rO1@1I1I1K1K.JPQX/ZlJmp  AS  qT  KTFJ'9;'FG !'*J!&,"3"3L"A"AB:NWWYYL%hn&9&9,&G&GHT]]__N*c1L'BH\,B,B+BCL##%% ZgZZZblmmm3z??++ + +!!!!Q$''5577 ++,N1a4()*LA&#L1L4NNF!&&((G"LGMgm00>>?HHJJQQRWXXGaaag&Gggs:77Gl+GJvs9999Dvd||s:..
67;;;5<?##d** $==??) 7 7AIeAhmQ.///36////LLFEFFFjLYYYcEk1--I%OOEzzi1nn199 :&&&%/&&&8$$$-$$$$]] :&&&#	/9&&&8$$$	1$$$ nhzXJ/F&G3&N%X%X%Z%Z%^%^de%^%f%fgG! 1 1fl** %++C000#$$q((Y\_`p\q\qq  zD  E  E  E ! !fl** #Z_
u$$#);v|/C/CC/H/H1/L#ML%k#$$/GVC[
'-jC.?@IIKK
Z''-7-?EIc6k*J&&,6u,=EIc5j)Z''5((SGM!4D-D-D1=g1F	#v+.03EIc6k4JEIVY[`V`La0b0b	#u*--1;F1C	#v+.j((.8.AEIc7l+2<[2IEIc;.// J..2<[2IEIc;./z))/9(/C/I/I/K/K/R/RSX/Y/YEIc8m,.2	#{*+1C!J r*   c                    |dvr||fS |}|dk    rd}d}n|dk    rd}d}n
|dk    rd}d}|d	         d
k    }|                                 s||fS |j        t          j        |          d
         d                  }d}|j        t
          j                                        t          j	        d          z
  k    rC	 | j
        d         |d         z  }t          ||z  dz
            dk     rd}n# t          $ r Y nw xY w|rt          D ]}	||	xx         |z  cc<   || j
        d<   |d         dk    }
|
                                 r|ddg                                         }|d                                                             d|d         j        d
                   |d<   ||
         }|d         |d         z                                  }t%          |          d
k    r(t          j        |          dk    r|dxx         |z  cc<   ||fS )N)GBpZAcILAr  GBPrd  r  ZARr  ILSrn   r   rS   TrK   r7   regularMarketPricerl   r4   皙?Frg   rp   rz   r`  )r   r   r   r  r   r   r   r   r   r   r"   r  r   r   rm  r  shiftrk  r   r  )r%   r   rg   	currency2rU   f_volumelast_rowprices_in_subunitsr  r   f_divdivsdiv_pctss                r(   r   z"PriceHistory._standardise_currency  s[   000x<	uIAAIAAIA h<>||~~ 	 x<728H--a045!=BL//11I4GR4P4P4PPQQ./CDxPWGXXa{##c))).&     	%  1
-6z*;$99;; 	% w{+,1133D M//1177d7mFXYZF[7\\DM;D[)DM9CCEEH8}}q  RZ%9%9A%=%=;1$9}s   :1C, ,
C98C9c                    fd|d                                          D             }g d}|D ]}d }|dk    r d}d}	n,dk    r d}d}	n||v r|v r
|  d}d}	n| d}d}	 d}t          | j        || j                  }
|
                    d|	          d
         j        d         }|	rd|z  }|j        |d         |k    dfxx         |z  cc<   |dt          | j        || j                  }|                    d|	          d
         j        d         }|j        |d         |k    dfxx         |z  cc<   |d<   |S )Nc                      g | ]
}|k    |S r   r   )r   r   fxs     r(   r   z6PriceHistory._dividends_convert_fx.<locals>.<listcomp>1  s    SSSA1PR77a777r*   rg   )USDJPYEURCNYr  CADr  =XFTr1   )r   r|   rl   rS   r4   rp   )uniquer   r   r    r  r   r   )r%   r   r  r|   bad_div_currenciesmajor_currenciesr   fx2_tkrfx_tkrreversefx_datfx_ratefx2_datfx2_rates     `           r(   r   z"PriceHistory._dividends_convert_fx0  s   SSSS:)>)E)E)G)GSSSEEE# 	Q 	QAGEzzu&&&21A+A+Ar
 )))!$*fdlCCFnnE&nAA'JOPRSG $G)M)J/2K?@@@GK@@@"&tz7DLII"??%?GGPUVXYi
3Q6CDDDPDDD "	*r*   c                 v    |j         r|S |                     |||          }|                     ||||          }|S N)r   _fix_unit_switch_fix_unit_random_mixups)r%   r   rE   r   r   r7  df3s          r(   r   zPriceHistory._fix_unit_mixupsW  sF    8 	I##B+>>**3+wOO
r*   c                   # |j         r|S t          j                    }d|| j        d}|j        d         dk    rd|j        vrd|d<   |S |j        d         dk    r'|                    d|           d|j        vrd|d<   |S |                                ##j        j	         #j        
                    |          #_        n/#j        j	        |k    r#j                            |          #_        dd	lm} g d
}#fd|D             }#|         dk                        d                                          }	|	                                r#|	         }
#|	          #||	          }nd }
|}#j        d         dk    r'|                    d|           d|j        vrd|d<   |S #|                                         }|                    |dd          }||z  }|dz                                  dz  }|dk    }d|z  }|dz                                  dz  }|dk    |dk    z  }||z  }|                                s'|                    d|           d|j        vrd|d<   |S d}t'          t)          |                    D ]"}|d d |f         }||         }|#j        ||f<   #||k                                    }|                     #|||          ##|                                         |k    }#|                                         |k                                    }|dk    r#|                                         |k    |z  }t'          |j        d                   D ]}||d d f         }|                                s$#j        |         }dD ];}|                    |          }||         r|j        ||f         dz  #j        ||f<   <d}|                    |          }||         r-#j        |ddgf                                         #j        ||f<   d}|                    |          }||         r-#j        |ddgf                                         #j        ||f<   	#|                                         |k    |z  }t'          |j        d                   D ]}||d d f         }|                                s$#j        |         }dD ];}|                    |          }||         r|j        ||f         dz  #j        ||f<   <d}|                    |          }||         r-#j        |ddgf                                         #j        ||f<   d}|                    |          }||         r-#j        |ddgf                                         #j        ||f<   	#|                                         |k    }|                                }n|}||z
  }||z
  } |dk    r/d| d| d}!| dk    r	|!d |  d!z  }!|                    |!|           #|                                         |k    }t'          t)          |                    D ]C}|d d |f         }"|"                                r!||         }|j        |"|f         #j        |"|f<   D|
Td|
j        vrd|
d<   t5          j        #|
g                                          #t5          j        #j                  #_        #S )"Nprice-repair-100xr9  r   r   Fr4   z3Cannot check single-row table for 100x price errorsrG  )ndimage)rj   ri   rk   rl   rm   c                 &    g | ]}|j         v |S r   r   r   r   r7  s     r(   r   z8PriceHistory._fix_unit_random_mixups.<locals>.<listcomp>  s%    >>>1Q#+-=-=Q-=-=-=r*   rt   z6Insufficient good data for detecting 100x price errors)r  r  wrap)r  mode   re  r)  zNo sporadic 100x errors      )ri   rl   rd  rj   ri   rl   rk         Y@zfixed /z currency unit mixups (z	 crudely))r   r   r   r   r   r   r   rm  ro   r   r   r   scipyr  r   rk  rj  median_filterr   rD   r   r   r   r  r9   r   r   r   r   r   )$r%   r   rE   r   r   r   r  _ndimager  f_zeroes
df2_zeroesdf_origdf2_datamedianr  ratio_roundedr  r  ratio_rcp_roundedf_rcpf_eitherr  rB  fir   n_before
df2_taggedn_afterr  r  n_after_cruder  n_fixed_crudely
report_msgfjr7  s$                                      @r(   r  z$PriceHistory._fix_unit_random_mixups_  s    8 	I $&& 3H[_[fgg
8A;!"*,,"';I8A;!LLNV`Laaa"*,,"';Iggii9<	--k::CIIY\[((	,,[99CI 	.-----AAA	>>>>	>>>		Na',,!,44==??<<>> 	XJxi.C(mGGJG9Q<1KKPXbKccc"*,,"';Iy>**,,''vF'KK6!**,,r1S I	&^2244r9#%*;s*BCu9||~~ 	LL2*LEEE"*,,"';I s9~~&& 	! 	!A!!!Q$B!A CGBENNO((**//XwLL^,,..#5
y>**,,388::Q;; Y((**c1Q6A171:&& L Lq!!!tWvvxx il* @ @A!**A!u @*,&a.4*?QOOA&&a5 L&)gcFG3D.D&E&I&I&K&KCGCFOOOA&&a5 L&)gcFG3D.D&E&I&I&K&KCGCFO^,,..#5>E5;q>** L L1aaa4[vvxx il* A AA!**A!u A*,&a.5*@QOOA&&a5 L&)gcFG3D.D&E&I&I&K&KCGCFOOOA&&a5 L&)gcFG3D.D&E&I&I&K&KCGCFOY0022c9J&NN,,MM#M]*!M1Q;;L'LLHLLLJ""</<<<<
KK
*K555 y>**,,3s9~~&& 	4 	4A!!!Q$Bvvxx 4aL!(RU!3A!*"444*/
;')S*-..99;;Csy11CI
r*   c                 b    | j         d         dk    rd}nd}|                     ||||d          S )Nrg   KWF  re  T)correct_dividend)r"   _fix_prices_sudden_change)r%   r   rE   r   r  s        r(   r  zPriceHistory._fix_unit_switch  sC     !*-66AAA--b(K]a-bbbr*   c                 ,  ) |j         r|S t          j                    }d|| j        d}|d         dv }|                                }|                                ))j        j         )j                            |          )_        n/)j        j        |k    r)j        	                    |          )_        )fdt          D             })|         dk    )|                                         z  }	d }
|rt          j        |	                    d          d	
                              |	j        j                  }|                                |                                z  }|j        |dk             }t'          j        |	j        j        |          })|         }
)|          ))j         rd|j        vrd|d<   |S )                                ))|         dk    )|                                         z  }	)d                                         )d                                         k    }| j                            d          rd }n)d                                                                          )d                                                                          z  })d         dk                                    }||z  |z  }|sa)d                                         }d|j        d<   t'          j        |)d         z            }|dk                                    |z  }||z  }d)j        v rQ)d         dk                                    }|                                r|| z  }|                                rd|	|<   |	                                }	|	                    d          }|||z  }|                                s'|                    d|           d|j        vrd|d<   |S |	                                t9          |          t9          )          z  k    r'|                    d|           d|j        vrd|d<   |S |dgz   }d}t;          t9          |                    D ] }||         }|)j        |	d d |f         |f<   !||)j        |df<   )d                                         dk    )d                                                                         z  }|)j        |	                    d          |z  df<   |)j        ||z  df<   )|                                         |k    }|                                })j        |                    d                   } |                     )|||          ))|                                         |k    }|                                }!)j        |                    d                   }"||!z
  }#|#dk    rt| j         d|# d| d| d}$|#dk     rDtA          tC          tE          |           #                    |"                              }%|$d |% z  }$|                    |$|           |
6d|
j        vrd|
d<   t          j$        )|
g                                          ))|                                         |k    }&t;          t9          |                    D ]C}'|&d d |'f         }(|(                                r!||'         }|j        |(|f         )j        |(|f<   D)S )!Nzprice-repair-zeroesr9  rS   rT   c                 &    g | ]}|j         v |S r   r   r  s     r(   r   z,PriceHistory._fix_zeroes.<locals>.<listcomp>!  s%    FFFAQ#+5E5Ea5E5E5Er*   rz   r4   rt   r  )r         ?r   Frj   rk   r  rn   r   rl   皙?rq   TzNo price=0 errors to repairrG  z;No good data for calibration so cannot fix price=0 bad datar   z: fixed r  z value=0 errors in z price datar6   r3   )%r   r   r   r   r   rm  ro   r   r   r   r   r   r   r  r   r}  r   r   countr   rs  r   rk  endswithr_  r   r  r   r   rD   r   r  sortedlistset
differencer   )*r%   r   rE   r   r   r   r  r   r  f_prices_baddf2_reservegrpnan_pctdtsf_zero_or_nan_ignoref_change	f_vol_badf_high_low_good
f_vol_zero
close_diffclose_chg_pct_absf_bad_price_chgf_splitf_change_expected_but_missing
f_bad_rowsr  r  rB  r   f_vol_zero_or_nanr  r  
dts_taggedr  dts_not_repairedr  r   dts_repairedr  r  r  r7  s*                                            @r(   r   zPriceHistory._fix_zeroes  s!    8 	I$&& 5h]a]hii
B<:-]]__ggii9<	--k::CIIY\[((	,,[99CIFFFF!1FFF
J3.#j/2F2F2H2HH 	M)L,,!,445AAAII,J\JabbCggii#))++-G-#.C#%7<+=+BC#H#H 23K++,Cy bj00&+B{O	((**C
Os2c*o6J6J6L6LLLv;''))SZ-@-@-B-BB;%% 	8II #F 0 0 2 2 ; ; = ==3u:??CTCTC]C]C_C_B_`Oh-1,6688J"_4x?I  8 !\..00
%&
"$&F:G+D$E$E!#4t#;"E"E"G"G*"T%7	 S[((>*c1;;==G{{}} G078)0C-04466 GBFL!>? $,,..!%%1%--
 #i/J~~ 	LL6jLIII"*,,"';IZ3s88!;;;LLV^hLiii"*,,"';I(+	 s:'' 	1 	1A1A-0CGLA&)** +.CGIx'( ]3355:s8}?Q?Q?S?S?\?\?^?^_JM  a ((+<<hFG:=,,h67^,,..#5
>>##Yz~~1~556
//XwLL^,,..#5
..""9Z^^^%;%;<W$Q;;[ff'ffHffQYfffC{{%d3z??+E+EFV+W+W&X&XYY*L***LLJL///"+"555+0K()S+.//::<<C 	N##%%,s9~~&& 	/ 	/A111a4Bvvxx /aL!#AA
r*   c           	      X   ||j         r|S |dv r|S t          j                    }d|| j        d}|d         dk                                    }|                                s|                    d|           |S | j        d         d	k    rd
}nd}d d}|                                }|	                                }	d|	j
        vrd|	d<   d}
|	d                                                                         }|	|         	                                }|	|          	                                }	|	d         dk                                    }|                                s|                    d|           |S t          j        |          d         }g }t          t          |          dz
  dd          D ]@}||         }|dk    r|	j        |dz
           }|d         |d         z
  }|	d         j        |         }|dk    r||z  dz
  dk     r|	j        |dz
           }|d         |z   }||d         k    r||d         k    r||	j        |df<   |	d         j        |         |	d         j        |         z  }|d||	d         j        |dz
           z  z
  z  }|	d         j        |dz
           |z  |	j        |df<   d|	j        |df<   d}
|                    |	j        |                                                    Bt          |          dk    r&dd |D              }|                    ||           t          t          |          dz
  dd          D ]K}||         }|	j        |         }|	d         j        |         }|dk    r2||	d         j        |dz
           z  }d}d}t-          |	d         j        |         |	d         j        |dz
           dz  d          r|	d         j        |dz
           dz  |	d         j        |         z
  }||	d         j        |dz
           dz  z  }d||	d         j        |dz
           dz  z  z
  }|	d         j        |dz
           |	d         j        |dz
           z  }t-          ||d          st||	d         j        d |t/          j        d          z
           z  |	j        d |t/          j        d          z
  df<   d|	j        d |t/          j        d          z
  df<   nt-          |	d         j        |         |	d         j        |dz
           dz  d          r|	d         j        |dz
           dz  |	d         j        |         z
  }||	d         j        |dz
           dz  z  }d||	d         j        |dz
           dz  z  z
  }|	d         j        |dz
           |	d         j        |dz
           z  }t-          ||d          st||	d         j        d |t/          j        d          z
           z  |	j        d |t/          j        d          z
  df<   d|	j        d |t/          j        d          z
  df<   n*|	d         j        |dz
           |	d         j        |         z
  }|} |t          |	          dz
  k     rSt          j        | |	d         j        |         |	d         j        |dz            z
  g          }!t          j        |!          }"nt          j        | g          }!| }"t          |	          |z
  dk     r4t7          t          |	          |dz             }#t5          d|#d z
            }$n3t5          d|dz
            }$t7          t          |	          |$d z             }#|#|$z
  dk     rt          j        }%n{|	d         j        |$|#dz
                                           |	d         j        |$dz   |#                                         z
  }&t          j        t          j        |&                    }%g }'|!dk                                    r|	d!         j        |         dk    r}t          j         |	d!         dk              t          |	          z  }(|dz  d"k     r|'                    d#dd$           n<|(d%k    r|d&k    s|dk    r|'                    d'dd$           n|	d(         j        |         })|)dk    rd }*n^||)z  }*|*|k    r| |%z
  }+n|"}+|+dk    rEt=          ||+z
            }t=          |*|+z
            },|,|z  |k    r|'                    d)|,d$           ||k    r|"dk    r|'                    d'dd$           nt=          ||"z
            }t=          ||z  |"z
            }-|*"|-|z  |k    r|'                    d'|-d$           n^t=          |*|z  |"z
            }.|-|.k     r"|-|z  |k    r|'                    d'|-d$           n!|.|z  |k    r|'                    d*|.d$           t          j!        |%          s| |%z
  }/|/dk    rt=          ||/z
            }t=          ||z  |/z
            }-|*"|-|z  |k    r|'                    d#|-d$           n^t=          |*|z  |/z
            }.|-|.k     r"|-|z  |k    r|'                    d#|-d$           n!|.|z  |k    r|'                    d+|.d$           ||||d,}0| |0d-<   |"|0d.<   |	d!         j        |         |0d/<   |%|0d0<   d|0d1<   d|0d2<   d|0d3<   d|0d4<   d|0d5<   t          |'          dk    r9tE          |'d6 7          }'|'d         }1d|0|1d8         #                    d9d:          <   tI          j%        |0g          &                    d;          5tI          j'        g          M|
s|S d< j
        D             }2                                dd?}3t          t                              D ]}d@         j        |         }j        |         }d=         j        |         }|dk    r=||	d         j        |dz
           z  }|	d         j        |dz
           |	d         j        |dz
           z  }4|	d         j        |         |	d         j        |         z  }5|5|4k    }6|4|5z  }d|z
  }7|7d"|z  k     }8|7dA|z  k    }9|6rd}8dB|i}0|6|0dC<   |9|0dD<   |8|0dE<   |0(                                D ]\  }:};|:vr{tS          |;tT          t          j+        f          rd|:<   nTtS          |;tX                    rd|:<   n9tS          |;tZ                    rd|:<   nt]          |:|;t_          |;                    |;j        ||:f<   |2g dFz  }2ddG<   t/          j        dHI          }<d1dEg                             dJ          }=|=                                rTt                    dk    r@t          j        |=          d         }>j        |=         }?|>D ]}j        |         }|j0        }@d }A|dk    rj        |dz
           }Bnj        |dz            }B|d=         |z  |Bd=         z  }C|d=         |Bd=         z  }Dt7          t=          |Cdz
            t=          |Ddz
                      }Et=          |@|Bj0        z
            |<k    r=|BdG         s5|Edk     r/|Bj0        |?v r$|d-         } |Bd-         }F| dK|Fz  k    r|Bj0        }An|@}An|@}A|Ar&dj        |AdGf<   |2D ]}G|Gj
        v rdj        |A|Gf<   dL}H                                t          dt                              D ]}j        |         }|j0        }@d}Id}J|d-         } j        |dz
           }K|d=         |Kd=         z  }Lt=          |@|Kj0        z
            |<k    rJ|KdG         sB|dG         s:t=          |Ldz
            |Hk     r$d-         j        |dz
           }M| dK|Mz  k    rd}Jnd}I|Js|Ir:|Ir|@nj        |dz
           }Ndj        |NdGf<   |2D ]}G|Gj
        v rdj        |N|Gf<   |2                    dG           dGj
        v rrdG         }O|OdM |2D                                          dJ           z  }O|O                                rj        |O         }PdNdO |PD              }|                    ||           j        |O         }Q|QD ]}|t/          j        d          z
  }R|	j        d |Rdfxx         dB         j        |         z  cc<   d|	j        d |Rdf<   |j        d |Rdfxx         dB         j        |         z  cc<   d|j        d |Rdf<   d|	j        |df<   d}
1                    |          dj        |OdGf<   1                    dGdJ          dG|2v r|22                    dG           |2                                                                         st                    dk    rt          dt                              D ]}dP\  }S}T|dk    r*dQ         j        |dz
           dQ         j        |         z  }S|t                    dz
  k     r*dQ         j        |dz            dQ         j        |         z  }T|Sp|T}S|Tp|S}Tt=          |S|z
            dRk     r/t=          |T|z
            dRk     rj        |         }@dj        |@d2f<   ŉ|2                                                                         s|
r|	S |S t          t                              D ]o}d@         j        |         }j        |         }d=         j        |         }|dk    r=||	d         j        |dz
           z  }|t/          j        dSI          z   }Utg          j4        |	j        |U          }Vt7          |Vt          |	          dz
            }Vt5          d|dTz
            }W|	d         j        ||Vdz            5                                }X|Xdk                                    |XdUk                                     z  }Y|Y                                r3|t          j        |Y          d         d         z   dz
  }V|	j        |V         }Ud}Zd}[tH          j6        }\|V|Wk    r5|	j        |W|Vdz            	                                }]|]d         |]d         z  |]dV<   |]dV         |]d         z  |]dW<   |]d         j        dd                                          |]d         j        d d                                         z
  }^t          j        dg|^          }^|^|]dX<   |]dW         j        dd                                          |]d         j        d d                                         z
  }_t          j        dg|_          }_|_|]dY<   |]dXdYg         }^|dZk    r|dk     r||]dV         j        d         z  }`|^dY         |`d[z  k    }=|=                                rt          j        |=          d         }>|>D ]}a|^dY         j        |a         }b|bd\|^dX         j        |a         z  k    rtd|^dY         z  |bz  }c|cd]k    }d|	d(         j        |         })d3         j        |         }e|)dk    s|es|d                                rt          j        |d          d         }ft          |f          dk    rt          j7        t          |f                    }gt          t          |f                    D ]Q}|f|         }a|cj        |a         }h|h|k     r|g|xx         ||hz
  j8        z  cc<   6|g|xx         d"|h|z
  j8        z  z  cc<   Rt          j9        |g          }|f|         }in|fd         }id}[|cj        |i         }\ n-|bd^|`z  k    r"|]dV         dk                                     rd}Z nj        |dEf         }8|Zr|8rd}Zi }0|Z|0d_<   |[|0d`<   |\|0da<   |Zr|	d(         j        |         })|)dk    r||)z  }*|*|k    r,d0         j        |         }%d-         j        |         } | |%z
  }+nd.         j        |         }"|"}+|+dk    r9t=          ||+z
            }t=          |*|+z
            },|,|dbz  k    rdj        |d3f<   |0(                                D ]\  }:};|:vrtS          |;tT          t          j+        f          rd|:<   nstS          |;tX                    rd|:<   nXtS          |;tZ                    rd|:<   n=|:dak    rtI          j:        dcd          |:<   nt]          |:|;t_          |;                    |;j        ||:f<   d1j
        v r-d`j
        v r$dj        d`                                         d1f<   q|2d_d`gz  }2|2D ]3}G|G                                         s1                    |GdJ          4da}G|Gj
        v rC|G                                                                         r1                    |GdJ          fde|2D             }2;                    dQ           |3dQf          dg<   dg         <                                }j|jD ]}kdg         |kk    }l|l                                         t                    }mdQg         	                                }nt          |n          dk    r|ndQ         j        =                                >                                j?        @                                dhz  }o|oj        d         |oj        |oj        d         <   |o|ndi<   |ndQ         |ndi         z  |ndj<   |2D ]E}G|G                                                                         s1                    |GdJ          Ffdk|2D             }p|pD ]}G|G                                         }qt          j         |q          }r|rd|mfv r8|r|mz  }s|Gd1k    rd}td&}ud`j
        v r%|G         d`         k                                    rvd_j
        v r|G         |G         d_         z  k                                    rdl}td_                                         }vt          j         |v          }mt          j         |q|v                   }r|r|mz  }s|s|tk    r)|ld_                                         z  }=dj        |=|Gf<   ,dEj
        v rJdE                                         r0d0         |l         |q         dk                                    rdm}un5d&}tn2d_j
        v r'|G         d_         k                                    rdl}tndl}u|s|tk    rSdj        |l|Gf<   d`j
        v r<dj        |ld`f<   tH          j6        j        |ldaf<   |l                                         
|s|uk    rdj        |l|Gf<   |Gd2k    r9d}td"}udDj
        vrdn}tdl}u|s|tk    rdj        |l|Gf<   I|s|uk    rdj        |l|Gf<   ]|GdCk    r|G         j        d         r|rdk    r~|GdEk    r|Gd_k    r|GdGk    r| j        dov r|Gd`k    r|Gdpv rd1|2v rdE|2v rdq}Gd|G<   t          t                              D ]}}j        |         }j        |         d1         rYdE         rQd=         }wdQ         }x|w|xz  }ydB         }zd|zz
  |yz  }7|w|7z  }Lt=          |L||z  z
            |k     }{|{rdj        ||Gf<   ~|G                                         s1                    |GdJ          n|2                    |G           d4j
        v rjdrD ];}G|Gv r|G         d4         z  |G<   d4         |G<   |2                    |G           <1                    d4dJ          |22                    d4                                           |2                             dJ                   j         r1|j         s(tI          j'        |	|g                                          }	|	S i }|t          dg         <                                          D ]^}kdg         |kk                                 ds          ddt<   t          t                    dz
  dd          D ]}j        |         j0        }|t/          j        d          z
  }RdCv odC         }}dEv odE         }~dDv odD         }d_v od_         }d2v od2         }d1v od1         }d3v od3         }d`v od`         }[dqv odq         }t          j         fdu|2D                       }|r|r|dk    rd}|dz  }|[r*|rd}dj        |d1f<   |dz  }|~rd}~dj        |dEf<   |dz  }|r|rd}dj        |d_f<   |dz  }|dk    r|~s|r|~rdvndw}:||B                    |:g                               |           ddQ         z
  dB         z  }|	j        d |Rdfxx         |z  cc<   d|	j        d |Rdf<   |j        d |Rdfxx         |z  cc<   d|j        d |Rdf<   dj        |dtf<   |rdx}:|}d=         |z  }||	j        |df<   |:dyz  }:dddB         z
  |z  z
  }|dB         z  }|	j        d |Rdfxx         |z  cc<   d|	j        d |Rdf<   |j        d |Rdfxx         |z  cc<   d|j        d |Rdf<   dj        |dtf<   ||B                    |:g                               |           ||rdz}:d|z  }d=         |z  }||	j        |df<   dQ         |z  }d|z
  }dB         }|:dyz  }:||z  }|	j        d |Rdfxx         |z  cc<   d|	j        d |Rdf<   |j        d |Rdfxx         |z  cc<   d|j        d |Rdf<   dj        |dtf<   ||B                    |:g                               |           C|}rd{}:||B                    |:g                               |           ddQ         z
  }|	j        d |Rdfxx         |z  cc<   d|	j        d |Rdf<   |j        d |Rdfxx         |z  cc<   d|j        d |Rdf<   dj        |dtf<   |[r0d|}:||B                    |:g                               |           ddB         z  }|	j        d |Rdfxx         |z  cc<   |j        d |Rdfxx         |z  cc<   da         }\|	d         j        d@                  }d=         }d||z  z
  }|\t/          j        d          z
  }|	j        d |dfxx         |z  cc<   |j        d |dfxx         |z  cc<   |	j        |\dfxx         |z  cc<   d|	j        |df<   d|	j        d |Rdf<   d|j        d |Rdf<   dj        |dtf<   
|rddQ         z
  }dB         }t=          ||z  dz
            dZk    rf|:dyz  }:||z  }|	j        d |Rdfxx         |z  cc<   d|	j        d |Rdf<   |j        d |Rdfxx         |z  cc<   d|j        d |Rdf<   dj        |dtf<   1                    |          1                    |          |rd}}:d|	d(         j        |         z  }d=         |z  }||	j        |df<   dQ         |z  }d|z
  }dB         }|:dyz  }:||z  }|	j        d |Rdfxx         |z  cc<   d|	j        d |Rdf<   |j        d |Rdfxx         |z  cc<   d|j        d |Rdf<   dj        |dtf<   ||B                    |:g                               |           |dk    r|r|}rd~}:||B                    |:g                               |           ddQ         |z  z
  }|	j        |dfxx         |z  cc<   |	j        d |Rdfxx         |z  cc<   d|	j        d |Rdf<   |j        d |Rdfxx         |z  cc<   d|j        d |Rdf<   dj        |dtf<   Y|rm|~rkd=         }|dQ         z  }ydB         }z|	j        |dfxx         |z  cc<   d'}:||B                    |:g                               |           dj        |dtf<   |r|rd}:||B                    |:g                               |           dQ         |z  }d|z
  }|dB         z  }|	j        |dfxx         |z  cc<   |	j        d |Rdfxx         |z  cc<   d|	j        d |Rdf<   |j        d |Rdfxx         |z  cc<   d|j        d |Rdf<   dj        |dtf<   |r|r|	j        |dfxx         |z  cc<   dx}:dv rvd         rnd }ddQ         |z  z
  }|dB         z  }|	j        d |Rdfxx         |z  cc<   d|	j        d |Rdf<   |j        d |Rdfxx         |z  cc<   d|j        d |Rdf<   |:dz  }:||B                    |:g                               |           dj        |dtf<   	S|dk    r|r|~r|rd}:d|z  d|	d(         j        |         z  z  }d=         |z  }||	j        |df<   dQ         |z  }d|z
  }dB         }|:dyz  }:||z  }|	j        d |Rdfxx         |z  cc<   d|	j        d |Rdf<   |j        d |Rdfxx         |z  cc<   d|j        d |Rdf<   dj        |dtf<   ||B                    |:g                               |           
<|r|~r|rd=         }|dQ         z  }ydB         }zd}:||z  }||yz  }||	j        |df<   d|z
  }||zz  }|	j        d |Rdfxx         |z  cc<   d|	j        d |Rdf<   |j        d |Rdfxx         |z  cc<   d|j        d |Rdf<   ||B                    |:g                               |           dj        |dtf<   j         r^`||D ]>}:d|: dd tE          |||:                   D              }|                    ||           ?dV|	j
        v rt]          d          |j         s(tI          j'        |	|g                                          }	|	S )N)r0   r1   r2   1yzdiv-adjust-repair-badr9  rp   rz   zNo dividends to checkrG  rg   r  r  re  gQ?r   Frl   r   r4   rS   rk   rd  rj   rm   r)  Tz'Repaired double-adjustment on div days c                 ,    g | ]}t          |          S r   )r   )r   r=  s     r(   r   z4PriceHistory._fix_bad_div_adjust.<locals>.<listcomp>  s    <Y<Y<YSVV<Y<Y<Yr*   rI  g?)rel_tol)secondsr6      rn   r  zdiv-too-small)stater_  g      ?g      ?zdiv-too-bigrq   zdiv-pre-splitzdiv-too-big-and-pre-splitzdiv-too-small-and-pre-split)r   r  div%r   
drop_2Dmaxvolumevoldiv_too_bigdiv_too_smalldiv_pre_splitdiv_too_big_and_pre_splitdiv_too_small_and_pre_splitc                     | d         S )Nr_  r   )r   s    r(   <lambda>z2PriceHistory._fix_bad_div_adjust.<locals>.<lambda>r  s
    AfI r*   )keyrA  -_r   c                 <    g | ]}|                     d           |S )div_)
startswithr   r   s     r(   r   z4PriceHistory._fix_bad_div_adjust.<locals>.<listcomp>~  s)    KKKall66J6JK!KKKr*   rB  rZ  c                    t          |           }|                     |          }g }|j        d         g}||         j        d         g}t	          d|          D ]}|j        |         }	||         j        |         }
|
t          j        |          z  |k     r+|                    |	           |                    |
           h|                    |           |	g}|
g}|                    |           t          j        dg|z            }d}t          |          D ]#\  }}t          |          }|||||z   <   ||z  }$|S )Nr   r4   rS   )
r   sort_valuesro   r   rD   r   meanr  array	enumerate)r   column	thresholdr  	sorted_dfclusterscurrent_dtscurrents_valsrB  r  rB  cluster_labelsrR  clusterncs                  r(   cluster_dividendsz;PriceHistory._fix_bad_div_adjust.<locals>.cluster_dividends  sZ   BAv..IH$?1-.K&v.3A67M1a[[ * *_Q'',Q/"'-000I==&&r***!((---- OOK000#%$K%(EMMOOK(((Xrd1f--NC'11  
7\\-.s3r6z*r	!!r*   r  r,   zpresent adjadj_missingadj_exceeds_divdiv_exceeds_adj)rd  re  rf  phantom   r7   rt         ?g{Gz?c                     g | ]
}|d k    |S )rg  r   rT  s     r(   r   z4PriceHistory._fix_bad_div_adjust.<locals>.<listcomp>  s    4Y4Y4Y1!y..Q...r*   zRemoving phantom div(s): c                 P    g | ]#}t          |                                          $S r   r   r   r   r  s     r(   r   z4PriceHistory._fix_bad_div_adjust.<locals>.<listcomp>  s&    2T2T2Tb3rwwyy>>2T2T2Tr*   )NNrC  r  #      gAdjzAdj LowdeltaadjDeltar   g333333?gjt?皙?g(\?adj_exceeds_pricesdiv_date_wrongdiv_true_dateg?zdatetime64[ns, UTC])dtypec                 &    g | ]}|j         v |S r   r   )r   r   div_status_dfs     r(   r   z4PriceHistory._fix_bad_div_adjust.<locals>.<listcomp>  s&    BBBqM,A'A'A!'A'A'Ar*   )rZ  ra  g    ~~Ar   zavg yr yieldc                 &    g | ]}|j         v |S r   r   )r   r   ra  s     r(   r   z4PriceHistory._fix_bad_div_adjust.<locals>.<listcomp>  s%    HHHA13G3Ga3G3G3Gr*   r  gUUUUUU?gtE]t?)zKAP.ILSAND)rI  rJ  adj_too_small)rG  rI  	ascendingzFixed?c                 (    g | ]}|v |         S r   r   )r   r   rows     r(   r   z4PriceHistory._fix_bad_div_adjust.<locals>.<listcomp>~  s"    )M)M)MQAHH#a&HHHr*   ztoo-small div-adjustztoo-big div-adjustztoo-small divz & div-adjustztoo-big divzmissing div-adjustzwrong ex-div datezpre-split divz"too-big div and missing div-adjustztoo-big div & div-adjustzFX was repairedz and FX mixupr  ztoo-big div & pre-splitzdiv-too-big and adj-too-smallz	Repaired r3   c                 P    g | ]#}t          |                                          $S r   rl  rm  s     r(   r   z4PriceHistory._fix_bad_div_adjust.<locals>.<listcomp>	  s&    #T#T#TrC		NN#T#T#Tr*   z"Adj" has snuck in df2)rB  rZ  )Cr   r   r   r   rk  r   r   r"   r   rm  r   r   r   r  rD   r   r   ro   r   r  r   rj  r   r   r   rX  r9   r   r  rW  r  r   r   r  r#  replacer   r   	set_indexr   itemsr   boolbool_r   r  r   rg  r   r   removebisectbisect_left
pct_changeNaTzerosr8   argminr  rV  r  	to_seriesr_  r  total_secondsr$  
setdefault)r%   r   rE   rg   r   r  r  currency_dividetoo_big_check_thresholdr7  df_modifiedf_nandf2_nandiv_indicesfixed_datesrB  div_idxprices_beforer_  rB  	dt_before	new_close	adj_afteradjr   r  div_pct#div_too_small_improvement_threshold!div_too_big_improvement_thresholddrop_c2ltrue_adjustpresent_adjr   dropsrD  r   r   typical_volatilitydiffspossibilitiespct_zero_volr   div_postSplit_dropdiff_postSplitdiff_fxdiff_fxPostSplitdrop_wo_vol
div_statuspchecksrc  pre_adjpost_adjdiv_missing_from_adjcloseimplied_div_yielddiv_adj_is_too_smalldiv_adj_exceeds_divr   vphantom_proximity_thresholdr  indicesdts_to_checkdiv_dt
phantom_dt	other_divratio1ratio2
divergence	drop_nextr   ratio_thresholdthis_is_phantomlast_is_phantomlast_divr  	last_dropphantom_div_dt	f_phantomdiv_dtsphantom_div_dtsenddtr_prer_postlookahead_datelookahead_idxlookback_idxfuture_changesf_big_changediv_adj_exceeds_pricesru  rv  xdeltas	adjDeltasadjDivr  adjDelta_dropr  f_near1_or_above	pre_splitnear_indices	penaltiesdtireversal_idxcluster_idscidfcr  r  
time_diffscluster_checksf_failn_failpct_failtrue_thresholdfals_thresholdf_adj_exceeds_prices	div_yieldpctcloseadj_presentalso_correct_adjdiv_repairsrd  rf  re  rt  rH  rG  rI  r|  n_failed_checksadj_correction
correctioncorrect_div
target_adjtarget_div_pctclose_beforetrue_adjenddt2r  div_truepct_trueadj_correctra  ry  r  s                                                                                                                                                     @@@r(   r   z PriceHistory._fix_bad_div_adjust  s6    ::I222I$&& 7_c_jkk
KC'1133yy{{ 	LL0
LCCCI!*-66"OO!O"']]__ggiick))$C G!!##,,..e*//##5&k  [!S(2244yy{{ 	LL0
LCCCIhuooa( s;'')2r22 	B 	BA!!nG!||HWQY/M '-*@@Dk"'0CaxxT#XaZ--  Igai0	)'2S8	e 444mTZF[9[9[2;CGIw./ #K 0 5g >WARSZA[ [I#sSW1B7191M-M'MNC69'l6GPQ	6RUX6XCGI{236:CGI{23"&K&&sy'9'>'>'@'@AAA{a[<Y<Y[<Y<Y<Y[[CKK:K... s;'')2r22 ]	@ ]	@A!!nG7#Bk"'0C!||CL-gai88G 34/01-s5zw/W1B7191Mc1Q]bccc Sw<,WQY7;c%jog>VVW!2719!=c!AB!C3w<+<WQY+G+K$LL!+.3GAI>WARSZ[\S\A]]{K5III TOZ]`ah]i]m  oQoqr{  sF  OP  sQ  sQ  sQ  pQ  oQ  ^R  PRCG>R	 3A > > >>>KLOSCG>R	 3A > > >>>KLU13w<3DWQY3OPT3T`efff 
Sw<,WQY7<s5zw?WWW!2719!=d!BC!C3w<+<WQY+G+K$LL!+.3GAI>WARSZ[\S\A]]{K5III TOZ]`ah]i]m  oQoqr{  sF  OP  sQ  sQ  sQ  pQ  oQ  ^R  PRCG>R	 3A > > >>>KLOSCG>R	 3A > > >>>KLw<,WQY7#e*/':RRDS!## $G(9'(BSZ_U\]^U^E_(_!`aaVE]]

$((!
C A%%#c((GAI..As1uAwqy))#c((E!G,,5y1}}%'V""G)%A+6??AACJOTYZ[T[\_T_D`DiDiDkDkk%'WRVE]]%;%;"Ms
!! C{c(m&8&A1&D&D!vc(mS&899#c((B3;$$!((/#)N)NOOOO #T))gnn'C--!((-)L)LMMMN+/3C<<$(MM %(%KM$s** $'9 9 *qyy"3u9~~),]5-@)A)A*-NNSWWW)00/R`1a1abbb 444!S((%,,mC-P-PQQQQ"3z>22"%s?':J&F"G"G(0 '*K KPTTT - 4 4mT[5\5\ ] ] ]/2M/4QS]3]/^/^,&)999$+.O$OTX#X#X$1$8$8-X_9`9`$a$a$a$47X$X]a#a#a$1$8$8B]fv9w9w$x$x$x x 233 { #');";K"Q"3{?33"%s?':K&G"H"H(0 '*M MRVVV - 4 4oV]5^5^ _ _ _/2M/4QS^3^/_/_,&)999$+.O$OTX#X#X$1$8$8/Za9b9b$c$c$c$47X$X]a#a#a$1$8$8B_hx9y9y$z$z$z"$GCgNNJ!%Jv'1J|$#&x=#5g#>Jx  2Ju(-J}%*/J'*/J'6;J238=J45=!!A%% &}:M:M N N N!!$;?
1W:--c3778,
|,,66v>>C$ # "	=#*> ? ?  IKK]2KKK%0022	" 	" 	" 	"> s=))** ,	- ,	-A#E*/2G$Q'B&+A.C!||CL-gai88G +&+GAI6W9J7ST99UUG;',W5G8I'8RRH(0G(;% "H,K #k 1#4G#D  #4r'z"B ) -',$'5J(AJ}%,?J(),@J()!'')) - -!M))!!dBH%566 	5+0a((#As++ 5+,a((#Au-- 5+.a(( (!DGG444+,!"a%((- 	GGGG#(i &/&9r&B&B&B#=*;<=AAqAII5577 	Es=))A--hqkk!nG(.q1L E E#(+!
q55 - 21Q3 7II - 21Q3 7Ie*_4	%8HHUi&66 VCZ#fSj//BB
vin,--1LLLU^_hUiLnx{nn ~55"6{$-f$5	#i-//)2JJ)/JJ%+
 E?CM%j)&;<# E E 555?DM-j!m<  %0022q#m,,-- 	E 	EA$Q'CXF#O#Ov;D$)!A#.HJ%0E6(-'((,GGGPXYbPcGlopylzG  @C  DI  JM  DM  @N  @N  Q`  @`  @`)&16qs;	#i-''&*OO&*O E/ E+:!X@STUVWTW@X?C!.)";< E EAM111?D).!*;<i    ---%i0I!m4Y4Y4Y4Y4Y&Z&^&^de&^&f&f%fgI}} @'-i8V2T2TG2T2T2TVVCz222"/"5i"@) ; ;By21====EG 3444m8T8XY[8\\4447;CG 34K 3444m8T8XY[8\\4447;GK 34/0CGBO,"&K$1$6$6r$:$:MM:?!)Y"67)..yq.AAMF""i(((V$((**..00 	J =!!A%%q#m"4"455 
J 
JA$.ME61uu -c 2 7! <}S?Q?VWX?Y YC..q011!.s!3!81!=c@R@WXY@Z!Z!OVE#_uF5011"44VO=S9T9TUW9W9W!.!4Q!7EI)&/*ABV$((**..00 	 
	 s=))** @	e @	eA#E*/2G$Q'B&+A.C!||CL-gai88G  	 3 < < <<N".sy.IIMs3xxz::Mq'"*--L !\.w}Q/FGRRTTN*Q.88::nt>S=]=]=_=__L!! : '"(<*@*@*CA*F Fq H!$=!9%*""NFM|++H\-/9:??AA[>AgJ6% x!E(2)5qrr*3355'
8L8U8U8W8WWC5&11#'
iL-abb1::<<q~?RSVTVSV?W?`?`?b?bb	IseY77	 )*GZ01T>>gmm 1U8=#33Fz*fSj9Auuww %*"$(1++a.#* #* #*C,2:,>,CC,HM,uVG_5I#5N/NNN +-VJ-?*?)N393; 0 ),N(;(?(C,9/,J,Nr,R	$)3JJyJGWG[G[G]G]J358<L3M3Ma3PL'*<'8'81'<'<46HS=N=N4O4O	16s<7H7H1I1I )R )RA2>q/C28,s2CC/2Rxx09!C0M09!SVM@Q0Q,.Ii,@,@7CA7CA59N4:L4NM$)E%2T&[%@%@ )*%'9'9';'; %FAE(>$)E $1#4R9J5J#K % /*> /).&J/EJ+,+9J'(*7J'% JN+/3C<< %(%KM$s**-:5-A-Eb-I*,V48< $'9 9%2<%@%DR%H
 *qyy"3u9~~),]5-@)A)A)d3h77EIM-b/.AB!'')) - -!M))!!dBH%566 	5+0a((#As++ 5+,a((#Au-- 5+.a((o--+-9;P+Q+Q+Qa(('!DGG444+,!"a%(( 555:JmNc:c:c_d!-0@"A"J"J"L"Lm"[\')9:: 	> 	>A #'')) > - 2 211 2 = =%%%-*:*?*?*A*A*E*E*G*G%)..qq.99MBBBBVBBB
 &11#66#4#4]3#O#O#Oi  $I.5577 i	 i	Cy)S0B#B'2244GGAu~**,,H8}}q  %c]0::<<AACCFTTVVZop
6@oa6H
z/23%/"+3C=8H;M+M( 6 6qz**,,0022 6%ll11l55GHHHHHHHN# Y Y ,,..aV##!A:%%%(N%)N'7?::
gVfNg@g?l?l?n?n: +w>>GAJSZ[\S]`gh|`}S}D~  DD  DD  DF  DF> *-/67K/L/U/U/W/W,F#788!#/C(D!E!E#)A:#n44 "]3G%H%Q%Q%S%S SA6:M-ad3 (GO;;HY@Z@^@^@`@`; $EN2.v69>>@@ 2-0NN .2NN-@@gajRYZnRoFoEtEtEvEv@ *- *->1137)"a%0+}/DDDFKM-b2B.BCEGVM-b/.AB&3B&7&B&B&D&DG !^3338)"a%0 ''%(N%(N(??)-),>1137)"a%0 !^3338)"a%0 %%qzr* !v{{ ))),,,	>>dk5G&G&G(((FFF GoYv F""'8F'B'BA$M!3}--.. 8 8"(+#(+}% 
8#.?*@ 
8 #E
Ic(C%cME"%m"4K)*;(=%%&77E'*5//2Q+R'S'SVe'e$' 837)"a%0 #'')) ! - 2 211 2 = =a   &-*???5 % %%%'4Q'7-Hc:d'dM!$$'45P'QM!$MM!$$$$)../JQR.SSMMM5666%0022 &mF&;&?&?Q&?&G&GH 	= =ig//::<<J i07799:: S	 S	C#M)$<c$ABG((5(99G %GH3w<<>2r22 K9 K9l1oX9.q9999+s2Is=7I"3s":"UsCT?U"3s":"UsCT?U%9S%@%^SI]E^" /3 6 O3;O+s2Is=7I /3 6 O3;O "2S!8!RSAQ=R /3 6 O3;O"$&)M)M)M)M&)M)M)M"N"N )#5 )/Q:N:N */&#q(O! 
-" -&+9>B$56'1,& -*/=BB(9$9:'1,  -) --2*@EB(<$<='1,"a''& yA/ yA6E_22K_#..q"55<<R@@@*-C.C<N)NFUFK$7888NJ888;?FUFK$78FUFK$7888NJ888;?FUFK$7848BL11& nA ,%4
&)%j:&=3>K0 _,%(S3}3E-E,S%T
)3c-6H)HFUFK$7888NJ888;?FUFK$78FUFK$7888NJ888;?FUFK$7848BL1#..q"55<<R@@@@$ [A)%(%8
&)%j:&=3>K0),SJ)>%(>%9
&)-&8_,)3k)AFUFK$7888NJ888;?FUFK$78FUFK$7888NJ888;?FUFK$7848BL1#..q"55<<R@@@@$ GA0#..q"55<<R@@@),SXFUFK$7888NJ888;?FUFK$78FUFK$7888NJ888;?FUFK$7848BL11' =A/#..q"55<<R@@@ *-S-?)?FUFK$7888NJ888FUFK$7888NJ888 ),O(<'*7|'8U'D!%j#&\)9#9!.y/B1/M/M/M!MGVG[$8999XE999GVG[$8999XE999 { :;;;sB;;;34K0;?FUFK$78;?FUFK$7848BL11+ #A &)3s8^
&)-&8
; 69::TAA0A-7+-ENG(;<<<N<<<?CCG(;<#K(;<<<N<<<?CGK(;<8<GKH55,9,>,>r,B,BM&-ll2&6&6GG& A+%(^)<)@)D%D
&)%j:&=3>K0),SJ)>%(>%9
&)-&8_,)3k)AFUFK$7888NJ888;?FUFK$78FUFK$7888NJ888;?FUFK$7848BL1#..q"55<<R@@@$))" :9{ :9@#..q"55<<R@@@),s3x/G)GK000OC000FUFK$7888NJ888;?FUFK$78FUFK$7888NJ888;?FUFK$7848BL11$ .9 .9!%j #CH&)-&8 K000OC000)#..q"55<<R@@@48BL11$ #9); #9 7#..q"55<<R@@@),S/)A%(>%9
)3c-6H)HK000OC000FUFK$7888NJ888;?FUFK$78FUFK$7888NJ888;?FUFK$7848BL11& 9? 9 K000OC000+,33<M8N3 #'C&)SXo-E&FG-4s=7I-ING(;<<<N<<<?CCG(;<#K(;<<<N<<<?CGK(;<0A#..q"55<<R@@@48BL1$))" &9 &9= &95&)/&9c#nBUBYZ\B]>]%^
&)%j:&=3>K0),SJ)>%(>%9
&)-&8_,)3k)AFUFK$7888NJ888;?FUFK$78FUFK$7888NJ888;?FUFK$7848BL1#..q"55<<R@@@@$ 9 9] 9 "%j #CH&)-&8;#&#6#+e#33;K0&)Hn)4{)BFUFK$7888NJ888;?FUFK$78FUFK$7888NJ888;?FUFK$78#..q"55<<R@@@48BL1}   	/ 	/AVaVV#T#TVKPQN=S=S#T#T#TVVCKK:K....CK4555} 	9)S'N++6688C
r*   c           
         |j         r|S t          j                    }d|| j        d}|dv }|s|S |                                }|d                                         dk    }|                                s|                    d           |S |                    dt          |d         |         	                                           |           d	|j
        vrd
|d	<   t          j        |          d         D ]}|j        |         }	|j        |	df         }
|	|j        d         k    r1|dv r|dz  }n|dz  }t          |j        d         |          }|j        d|dz            }|                    d| d|	                                 d|
d|           |                    d|j        d                                          d|j        d                                          |           |                     ||||
dd          }||j        d         dz
  k    r|}*|j        |dz   d          }|j         r|                                }Yt)          j        |                                |g          }|S )Nzsplit-repairr9  r+   r0   r1   r2   rq   r   z%price-repair-split: No splits in datazSplits: rG  r   Fr0   r1   r2   r4   r   z
split_idx=z
 split_dt=z split=.4fzdf dt range: rZ   rS   T)correct_volumer  )r   r   r   r   r   rk  r   r   r   to_dictr   r   r  ro   r   r   r   r   r   r  r   r   )r%   r   rE   r   r   r  interdaysplit_f	split_idxsplit_dtr   
cutoff_idxdf_pre_splitdf_pre_split_repaireddf_post_cutoffs                  r(   r   z"PriceHistory._fix_bad_stock_splits	  s    8 	I$&& .xVZVabb
:: 	I]]__^$--//14{{}} 	LL@AAAILB~$6w$?$G$G$I$I J JLLT^___bj((#B{O'**1- 	Y 	YIx	*HF8^34E28A;&& 000Q		Q	RXa[)44J71Z\>2LLL^i^^8==??^^SX^^^fpLqqqLLj);A)>)C)C)E)Ejj<K]^`KaKfKfKhKhjjr|L}}}$($B$B<QY[fhm  C  VZ$B  %[  %[!RXa[]***!#A!7!' Y.99;;BB$9$D$D$F$F#WXXBB	r*   c                 9  efg |j         r|S t          j                    }d|| j        d}|gdgz  }	|dv }
|dv }|dv r
d}d|d	<   d }nwd
}d|d	<   |d                                         dk    }|j        |                                         t          j                            d          z
  	                                }|
                    d| d|ddd|z  dd|           g d}dgcxk     rdk     rn n|
                    d|           |S |                                                    d          }|j        j         |j                            |          |_        n/|j        j        |k    r|j                            |          |_        |j        d         }|d         dk    }|||                                                             d          z  }|                                o#t)          j        |          d         d         dk    }| }t)          j        |t)          j        |d          z            d         }t/          |          dk    rt)          j        |          d         }t/          |          dk    rd }nt1          |d                   }d| d | }|&|d!|j        |         	                                 dz  }|
                    ||           |                    t4          j                  rV|                                }|                    g d"dd#$          }d%gf|                    ffd&|D             dd#$          }ng f|
rD|d'k    r>gd(vr:t)          j        |d)fd          }|d*d%g                                         }|dk    }n7t)          j        |d+fd          }||                                         }|dk    }|                                rd||<   |d%         dk    }|                                rit)          j        t/          |                    }|d,                                         |          |d%                                         |          z  || <   n5|d,                                         |d%                                         z  }|j        }|t(          j         k    r|!                    d-          }tE          |j        d                   D ]Z}|d d |fxx         |z  cc<   |                    t4          j                  r%||         |j#        v r|||         xx         |z  cc<   [|t(          j         k    r|!                    d.          }|dd f         |d d/f         z  |dd <   |t)          j        |dd          z  }|                                rd||<   |
r|d'k    rt)          j$        |d          }nt)          j%        |d          }t)          j&        |          } |                                 rd|| <   |                    t4          j                  r6||d0<   |d0         '                    d)          !                    d1          |d0<   tQ          g|	          }!t)          j(        |          |!dz
  d2z  dz   k     rAt)          j        |          d|!dz
  d2z  dz   z  k    r|
                    d3| d4|           |S t)          j)        |d5d6g          \  }"}#|#|"z
  }$|"d7|$z  z
  }%|#d7|$z  z   }&||%k    ||&k    z  }t)          j*        ||                   }'t)          j+        ||                   }(|(|'z  })|
                    d8|'d9d:|(dd!|)d;z  d<d=|           d>|)z  }*|
r|d'k    r|*d?z  }*|d@v r|*d)z  }*tQ          g|	          d|*z   k     r8|
                    dA|           |
                    dB|)ddC|*d|           |S ||	z  }+tQ          g|	          }!|
                    dD|!dEdF|*d|           |!dz   |*z   d2z  },|
                    dG|,dE|           dH|j#        vrd|dH<   |
r	|d'k    rdI}-nd}-|-rbt)          j        |d+fd          }||         ,                    dd                                          }|dd f         |d d/f         z  |dd <   n|}||	z  }+|d|,z  k     }.||,k    }/t/          |/j                  }0|0dk    r|/n|/                    d          }1|1                                rut)          j        |1          d         D ]Y}2|d         j-        |2         }3|3dk    rdn|d         j-        |2dz
           |3z  }4|rV|2dz   t/          |          k     r@|d         j-        |2dz            }5|5dk    r$tQ          |4|d         j-        |2         |5z            }4|4d>k    rtQ          d|2dJz
            }6t          t/          |          |2dJz             }7|d         j-        |6|7         dk                                    r|d         j-        |6|2dz
           *                                }8t)          j&        |8          s|8dk    r|3|8z  dKk     rC|0dk    rd|/|2<   Pd|/|2d d f<   [|.|/z  }|                    t4          j                  r|-sA|+|dL<   |.|dM<   |/|dN<   |dL         '                    d)          !                    d1          |dL<   ntE          t/          |                    D ]{}||         }9|9fv rm|+d d |f         ||9dOz   <   ||9dOz            '                    d)          !                    d1          ||9dOz   <   |.d d |f         ||9dPz   <   |/d d |f         ||9dQz   <   ||                                s|
                    d3| d4|           |S dR}:|d                                         dk    };|dv r|;                                rt)          j        |;          d         }<t)          j        |          d         }=t/          |<          rt/          |=          sd S |=d d d f         |<z
  }>|>d/z  }>|>dk    }?|?                                r#|>|?                                         }@t          j.        |          |@z  }At_          |At          j        j                  r!t          j                            |:S          },nta          j1        |:S          },t_          |,t          j        j                  r]t_          |At          j        j                  r>t)          j        |>|@k              d         d         }B|j        |B         }C|C|Az   |C|,z   k     }Dn|A|,k     }D|Dr|2                    dT|           |S |                    t4          j                  r^tg          tE          d|j        d                             |dU<   |j        d         dz
  |dU         z
  |dV<   |-rN|dW fD                                          d          |dX fD                                          d          z  }En|dM         |dN         z  }E|Et)          j        |Ed/          z  t)          j        |Ed          z  t)          j        |EdY          z  t)          j        |Ed)          z  }Eti          j5        dZd d[dJd\d]          5  |
                    d^tm          ||E                   z              d d d            n# 1 swxY w Y   gfd_}F|0|j        d         dz
  |z
  }G|
                    d`| da|G |           |-rYt)          j        |d          }H|r*t)          j        |d          }It)          j        |d          }Jg dbetE          t/          |                    D ]}||         }9t)          j        |          d         d         }K|r|||Kk    r|d d |f         }L|/d d |f         }M|.d d |f         }N |F|L|d          |M|d          |N|d                    }Ot/          |O          dk    rGtE          t/          |O                    D ]*}2|O|2         }+|+d         |z   |+d         |z   |+d)         f|O|2<   +t)          j7        t)          j        |Md/                    }Pt)          j7        t)          j        |Nd/                    }Q|Q|Pz  }R |F|R|Gd          |Q|Gd          |P|Gd                    }St/          |S          dk    rtE          t/          |S                    D ]*}2|S|2         }+|+d         |Gz   |+d         |Gz   |+d)         f|S|2<   +tE          t/          |S                    D ]*}2|S|2         }+||+d         z
  ||+d         z
  |+d)         f|S|2<   +|O}T|T8                    |S           n+ |F|d d |f         |/d d |f         |.d d |f                   }T|
                    dc|9 dd|T |           |tE          t/          |T          dz
  d/d/          D ]|}2|T|2         }+|j        |+d                  	                                |k     rI|
                    de|9 df|j        |+d                   dg|j        |+d         dz
            dh|           |T|2= }t/          |T          dk    r|Te|<   ts          di eD                       }U|Udk    rn|Udk    refdjtE          t/          |                    D             }Vt)          j        t)          j:        |V          d/k              d         d         }B||B         }W|
                    dk| dl|W dm|           nfg dn}XtE          t/          |                    D ]}||         }9e|         }T|Tg }T|TD ]}+|+d)         dok    rg}Y|	}Zn|	}Yg}Z|
r\dp| dq|9 dr|j        |+d         dz
           	                                 ds|j        |+d                  	                                 dt|Yd
}[n7dp| dq|9 dr|j        |+d         dz
            ds|j        |+d                   dt|Yd
}[|
                    |[|           |X|xx         |+d         |+d         z
  z  cc<   |j-        |+d         |+d         |j#        ;                    |9          fxx         |Yz  cc<   |9d%k    r=|j-        |+d         |+d         |j#        ;                    d,          fxx         |Yz  cc<   |r3|9d*k    rdI|I|+d         |+d         <   n|9d%k    rdI|J|+d         |+d         <   dI|H|+d         |+d         <   ts          |X          dk    rkdu}\tE          t/          |                    D ]0}|X|         dk    r"|\duk    r|\dvz  }\|\||          dw|X|          dxz  }\1dy|\ }[|2                    |[|           |r|I|Jz  }]t)          j<        |I|J          }^|]                                rA|j=        |]df         |Zz  '                                !                    d.          |j=        |]df<   |^                                rD|j=        |^df         d2z  |Zz  '                                !                    d.          |j=        |^df<   dI|j=        |HdHf<   njd}Xt)          j        |          d         d         }K|r|||Kk    r |F||d          |/|d          |.|d                    }Ot/          |O          dk    rGtE          t/          |O                    D ]*}2|O|2         }+|+d         |z   |+d         |z   |+d)         f|O|2<   +t)          j7        t)          j        |/d/                    }_t)          j7        t)          j        |.d/                    }`|`|_z  }a |F|a|Gd          |`|Gd          |_|Gd                    }St/          |S          dk    rtE          t/          |S                    D ]*}2|S|2         }+|+d         |Gz   |+d         |Gz   |+d)         f|S|2<   +tE          t/          |S                    D ]*}2|S|2         }+||+d         z
  ||+d         z
  |+d)         f|S|2<   +|O}T|T8                    |S           n |F||/|.          }T|tE          t/          |T          dz
  d/d/          D ]y}2|T|2         }+|j        |+d                  	                                |k     rF|
                    dz|j        |+d                   dg|j        |+d         dz
            dh|           |T|2= z|TD ]Q}+|+d)         dok    rg}Y|	}Zn|	}Yg}Z|
                    d{|+ d||Y |           d}D ]?}9|j-        |+d         |+d         |j#        ;                    |9          fxx         |Yz  cc<   @|r=|j-        |+d         |+d         |j#        ;                    d~          fxx         |Yz  cc<   |rw|j#        ;                    d          }b|j-        |+d         |+d         |bf         |Zz  '                                !                    d.          |j-        |+d         |+d         |bf<   dI|j-        |+d         |+d         |j#        ;                    dH          f<   |+d         |+d         dz
  k    rH|
r,dp| d|j        |+d                  	                                 }[ndp| d|j        |+d                   }[nh|j        |+d         dz
           }c|j        |+d                  }d|
r1dp| d|c	                                 d|d	                                 d}[ndp| d|c d|d d}[|
                    |[|           |X|+d         |+d         z
  z  }XSt/          |T          d)k    r\d}[|TD ]V}+|[d|j        |+d         dz
           	                                 d|j        |+d                  	                                 z  }[Wndy|X dx}[|2                    |[|           |r|d                                         } |                                 rA|d         |           '                    d          !                    d.          |j=        |  df<   n1|d         '                    d          !                    d.          |d<   |                                S )Nzprice-change-repairr9  r)  r  r  )r  rd  z
100x errorr  r:  z	bad splitzprice-repair-splitrq   rz   r4   rJ  z
start_min=z change=r   z (rcp=r[   rG  )ri   rj   rk   rl   rs  g      ?z(Split ratio too close to 1. Won't repairFr}  r   rn   rt   zappears_suspended=z, idx_latest_active=rX   )rm   rn   rp   rq   r   r   )ru   r   rl   c                     g | ]}|v|	S r   r   )r   r   
debug_colss     r(   r   z:PriceHistory._fix_prices_sudden_change.<locals>.<listcomp>	
  s#    %M%M%MA*9L9La9L9L9Lr*   r+   )r  re  gMbP?rI  ri   r6   rm   r  r   rS   z1D %r   r  zNo z
s detected   K   ri  z+Estimation of true 1D change stats: mean = z.2fz, StdDev = r  z.1fz
% of mean)r   r  )r1   r2   z>Split ratio too close to normal price volatility. Won't repairz	sd_pct = z  largest_change_pct = z
split_max=z.3fz largest_change_pct=z
threshold=r   Tr,   g       @rdownup_r_down_uprK   r7   z<100x changes are too soon after stock split events, abortingrB  i_revc                     g | ]}|d z   S )r  r   rT  s     r(   r   z:PriceHistory._fix_prices_sudden_change.<locals>.<listcomp>
  s    $C$C$C1QwY$C$C$Cr*   c                     g | ]}|d z   S )r  r   rT  s     r(   r   z:PriceHistory._fix_prices_sudden_change.<locals>.<listcomp>
  s     \y\y\yij]^_d]d\y\y\yr*   zdisplay.max_rowszdisplay.max_columnszdisplay.widthr  z!price-repair-split: my workings:
c                    | d         rKt          j        |           } d| d<   t          j        |          }d|d<   t          j        |          }d|d<   |                                 sg S t          j        |           d         }g }t	          t          |          dz
            D ]]}|dz  dk    rRdk    r|||                  rdnd}n|||                  rdnd}|                    ||         ||dz            |f           ^t          |          dz  dk    rVdk    r||d                  rdnd}n||d                  rdnd}|                    |d         t          |           |f           |S )	Nr   Fr4   rI  r)  r   z	1.0/splitrS   )r   rm  r   r  rD   r   r  )r  f_upf_downtrue_indicesrangesrB  r  r   s          r(   map_signals_to_rangeszEPriceHistory._fix_prices_sudden_change.<locals>.map_signals_to_ranges
  s   t "GAJJ!wt}}Q!q	5577 	8A;;q>LF3|,,q011 O Oq5A::s{{)/Q)@Qggk-3LO-DQkk'MM<?LQ4G"MNNN<  1$))3;;%+L,<%=N'';CC)/R0@)AN++wC|B/Q=>>>Mr*   zidx_latest_active=z, idx_rev_latest_active=)NNNNzcolumn 'z
' ranges: zPruning z range rO  z because too old.c                     g | ]}|dnd	S )Nr4   r   r   )r   r  s     r(   r   z:PriceHistory._fix_prices_sudden_change.<locals>.<listcomp>A  s     PPPqamPPPr*   c                 (    g | ]}|         r|nd S rS   r   )r   rB  OHLC_correct_rangess     r(   r   z:PriceHistory._fix_prices_sudden_change.<locals>.<listcomp>F  s(    VVV03;VVVr*   z
Potential z detected only in column z(, so treating as false positive (ignore))r   r   r   r   r   z
Corrected z on col=z range=[:z] m=rW   rb   =r  zCorrected: zPruning range zrange=z m=)ri   rj   rk   rl   rm   rp   z on interval z across intervals rZ   z (inclusive)z
Corrected:rY   )>r   r   r   r   rk  ro   r   rn  ro  r   r   rm  r   r   r   r   r   r   r   r   r   r  rollr   r   isEnabledForrw  DEBUGr   fullonesrw  r   r   rD   r   r  r  r  r   r9   
percentilerW  stdr  r   r   r   r   r   rj  r$  r   option_contextr   flipextendr   rX  r  logical_xorr   )hr%   r   rE   r   changer  r  r   r  	split_rcpr  multidayfix_type	start_minr  OHLCr7  r  f_no_activityappears_suspendedf_activeidx_latest_activelog_msgdf_debug_1d_change_x
price_dataf_zeror  df_dtyper  f_zero_num_denom_1d_change_minxf_na	split_maxq1q3iqrlower_boundupper_boundavgsdsd_pctlargest_change_pctr  r[  correct_columns_individuallyr  r  
f_up_ndimsf_up_shiftsrB  r  vol_change_pctnext_vlookback	lookaheadavg_vol_afterr   threshold_daysf_splits	indices_A	indices_Bgapsf_posgap_mingap_tdr  r  within_thresholdr-  r  idx_rev_latest_activef_correctedf_open_fixedf_close_fixedidx_first_fr  f_upjf_downjranges_beforef_rev_downj	f_rev_upjf_revjranges_afterr  r!  idxscoln_correctedrU   m_rcpr   counts_prettyf_open_and_closed_fixedf_open_xor_closed_fixed
f_rev_downf_rev_upf_revcol_locr   r   r#  r  r   sh                                                                                                        @@@r(   r  z&PriceHistory._fix_prices_sudden_change	  s!   8 	I$&& 5h]a]hii
%K	::44]""#H#6Jx II"H#7Jx >"++--4A!**Y-D-R-RYZ-R-[-[[aaccIX)XXVXXXs6zXXXX`jkkk/// LLC:LVVVIggii""U"339<	--k::CIIY\[((	,,[99CIIaL Hq(%D	(8(8(<(<!(<(D(DD)--//TBH]4K4KA4Nq4QST4T!>HX!0D0D%DEEaH !!Q&& " 2 21 5 !!Q&& $ #$5a$8 9 9a'8aaN_aa(BCI&78==??BBBBGWJ///w}-- 	xxzzH}}%f%f%fmnw}  A  AH!J}}%M%M%M%M%M%M%MTU^f}ggHHJ  		'D((U:M-M-M 7Aq63//LfW-.7799J3&FF7Aq63//LT++--J3&F::<< 	%!$Jv W"::<< 	H'#c((##C{+4466w?#g,BWBWBYBY[aZaBbbCLLk"++--G0E0E0G0GGC#rx#**733Jz'*++ 	- 	-AqqqsOOOs"OOO""7=11 -7h...T!W%%%,%%%rx#**511J%abbc+j".??QRR!BGFAA$>$>$>>!! 	1-0L)* 		>D(( jA>>>OO !i1===Ox((88:: 	($'OD!w}-- 	G.HV'/55a88??FFHV y))	6/""i!ms%:Q%>>>26/CZCZ]`enqrervydy|}d}]~C~C~LL3x333:LFFFI R99B2g39n39n+;0NOgoa())VOA&''c}3}}}WY}}}aghmam}}}}  FP  	Q  	Q  	Q Z 	(D((!#>))"a'"ui  3);#;;;LLYakLlllLL`V```HZ```hrLsssI i'y))	])]]]EW]]]eoppp_'99S@	1)111DDDck))$C 	1D(( ,0((+0(' 	+7Aq63//LT**344==??J)!""#/*SbST2CCL*L9$i/ i'__
(!mmddq1A1A?? 	*Xk**1- * *M&q)&'1ff#h-2DQqS2IA2M ]1s3xx ]/!4Fzz),^S]=OPQ=RU[=[)\)\!A%%  #1ad||H #CHHad 3 3IN+0)1CDcINNPP ! !$'M$6x!|$D$I$I$K$KM8M22 !}q7H7HQ}__bMbMb !!Q"'Q$)QqqqS	TMw}-- 	9/ 9 !#) !% ( 3 3A 6 6 = =e D Ds4yy)) 9 9AQAJ-.qqq!tWT*-5a$h-?-E-Ea-H-H-O-OPU-V-VT*06qqq!tW-.2111a4jU+uuww 	LL3x333:LFFFI ~&//11S8]""x||~~"**1-IAIy>> Y tQQQW%	1DBJD1HEyy{{ u+//++5h??'Ifi&=&KLL I ) 7 E E> E Z ZII ) 3 H H HIi)@)NOO :T^_egpg~  hM  UN  UN :(4=11!4Q7C3B(*VY'G$$'-	'9$# KK ^fpKqqqIw}-- 		b q(.*;!<!<==HSM (q 1! 3hsm CHW+ =#$C$C
$C$C$CDHHaHPPS[\y\ynx\y\y\ySzS~S~  EFS~  TG  TG  G#F+htn<"'(B"7"77"'(A:N:NNQSQXYaceQfQffikipqy{|i}i}}H"#5t=RTVXgimnn b bFXV^M_I`I``aaab b b b b b b b b b b b b b b	 	 	 	 	B ($&HQK!O6G$G!LLp.?ppYnpp  yCL  D  D  D' E	/'!U++K 2  "wq%00 "5 1 1":":":3t99%% -4 -4G hqkk!nQ/$ V*;*GL]alLlLl 111a4B AJE$QQQTlG$9$9"=N=O=O:PRWXiXjXjRkmt  vG  vH  vH  nI  %J  %JM=))A--!&s='9'9!:!: j jA -a 0A01!7H0H!A$QbJbdefgdh/iM!,,"$'"'%*<*<"="=K "(<(< = =I&4F#8#8@U@V@V9WYbcxcycyYz  }H  I^  I_  I_  }`  $a  $aL<((1,,!&s<'8'8!9!9 q qA ,QA/0t6K/KQqTTiMiklmnko.pLOO!&s<'8'8!9!9 E EA ,QA/01vq1vqt.DLOO*FMM,////221QQQT7DAJqqqRStUUF===V==ZPPP("3v;;q="b99 * *"1I9QqT?//11I=="LL)uA)u)uci!o)u)uQTQZ[\]^[_`a[aQb)u)u)u  ~HL  I  I  I &q	v;;??-3'*PP<OPPPQQEzz!VVVVU3t99EUEUVVVhrx~~344Q7:3iz(zzSzzz  CM  N  N  N  N (iis4yy)) 6 6AQA03F~!## 6 6Q47?? %A$-EE )A$)E# } #Kx  #K  #K  #K  #KCIVWXYVZ[\V\L]LbLbLdLd  #K  #Kgjgpqrstqugvg{g{g}g}  #K  #K  DE  #K  #K  #KCC"|x"|"|"|"|CIVWXYVZ[\V\L]"|"|`c`ijklmjn`o"|"|uv"|"|"|CS
;;;#A!A$qt)31adCK,?,?,B,B!BCCCqHCCC<<HQqT!A$Y0C0CK0P0P%PQQQUVVQQQ) @ F{{:>QqT!A$Y 7 7!"g;?ad1Q4i 815AaD1I..-6. {##a''$&M"3t99-- L L&q>Q..,22 - 5)Q-K-K+a.-K-K-KKM777CKK:K666 R*6*F'*,.}*U*U'*..00 LBE'JackJkBlotBtA{A{A}A}  BE  BE  FK  BL  BLCG3X=>*..00 RBE'JackJkBlorBruzBz  BB  BB  BD  BD  BK  BK  LQ  BR  BRCG3X=>04CGK,-- K(1++a.+K  @&7&CHY]hHhHh !6 5a8I8J8J6KTRcRdRdMegmn  oA  oA  hB  !C  !C}%%))"3}#5#566 f f)!,,-aD3D,DadM^F^`abc`d+ea((WRWT2%6%677
72762#6#677 :-44U;P;Q;Q5RT\]r]s]sTt  wA  BW  BX  BX  wY   Z   Z|$$q(("3|#4#455 m m(O+,Q42G+G1PeIeghijgk*lQ"3|#4#455 A A(O+,QqT61QqT61Q4*@Q&l++++..q$??$s6{{1}b"55 & &Aq	Ay1++--	99%mci!o%m%mSTUVSWXYSYIZ%m%m%mu  A  A  A"1I + +Q47??A%EE!A!E/a//A//zBBBF E EAHQqT!A$Y(;(;A(>(>>???1D????# OHQqT!A$Y(;(;K(H(HHIIIQNIII! p!k11(;;G47HQqT!A$Y=O4PSX4X3_3_3a3a3h3hin3o3oCHQqT!A$Y/0HL1adCK$7$7$D$DDEQ41Q4!8## TZ8ZZ#)AaD/BVBVBXBXZZS8SS#)AaD/SS  IadQh/E)AaD/C dq8qquzz||qqY\YaYaYcYcqqqc8ccuccRUcccS
333qtad{*6{{a" V VAUsy1a05577UUSYqt_=Q=Q=S=SUUUCCV 3K222KK:K... 	Ex=%%''Dxxzz E*-h-*>*D*DQ*G*G*N*Nu*U*Uh'' #H 3 3A 6 6 = =e D DH~~s   .,|&&|*-|*)r9   r+   r  r"  )F)FF)__name__
__module____qualname__r	   r)   r   log_indent_decoratorr   r   r  r  r   r  r  r  r  r  r!  r   r  r   r   r   r  r  r   r   r   r  r   r*   r(   r   r      s       15Z 0 0 0 0  !DeT eE% 5""	g g (*|	g g g  gR      *4 & & & & & & $)
 	 	ry 	 	 	 	 (-J 	 	29 	 	 	 	 !&Z 	 	BI 	 	 	 	 "'j  RY    (! !", ! ! ! !F { { {  {z5 5 5n% % % %N     X X  Xt c c  c  @ @  @D y y  yv : :  :x K  K  K   K  K  K r*   r   )	curl_cffir   mathr   r  r.  r   dateutilrn  rw  numpyr   pandasr   r   r   r   yfinancer   r   yfinance.constr   r   r	   yfinance.exceptionsr
   r   r   r   r   r   r*   r(   <module>r     s                                      " " " " " " " " C C C C C C C C C C n n n n n n n n n n n nC/  C/  C/  C/  C/  C/  C/  C/  C/  C/ r*   