o
    ̿Sin                  
   @   sf  d dl Z d dlZd dlZd dlZd dlmZ d dlmZmZmZm	Z	 d dl
mZ d dlmZ d dlmZ d dlmZ eeZdadgd
dZeeddZeeddZeeddZeddped Zdedededee d	df
ddZ eeddZ!edZ"eddpd # Z$ed d!Z%ed"d#Z&ed$d%pd% # d&vZ'ed'd(pd( # d)v Z(eed*d+Z)ed,d-Z*d.d/ ed0d1pd1+d2D Z,d3d/ ed4dpd+d2D Z-ed5d6pd Z.ed7d8pd Z/dd9d:ed	eeef fd;d<Z0d=ed>ed?ed	e1eef fd@dAZ2dhdBe	e d	e3fdCdDZ4dhdBe	e d	e3fdEdFZ5dhdBe	e d	efdGdHZ6dhdBe	e d	efdIdJZ7d	efdKdLZ8dMed	dfdNdOZ9dPeeef d	dfdQdRZ:dSdTdUed	e3fdVdWZ;dSdXdYdUedZed	e3fd[d\Z<d	efd]d^Z=dMed	dfd_d`Z>d:edaed	eeeef  fdbdcZ?dhdde	e d	dfdedfZ@dS )i    N)datetime)AnyDictListOptional)ZoneInfo)get_mongo_db)_get_global_zerodha_client)refresh_stock_history_for_stockFreturnc              
   C   s   t rdS z| d jdgdddd W n ty, } ztdt| W Y d}~nd}~ww z| d jd	gdd
dd W da dS  ty[ } ztdt| W Y d}~da dS d}~ww )zBest-effort index creation.

    Ensures we never create duplicate stock history documents for the same stock_id,
    even if multiple processes accidentally run background loops.
    Nstock_history)stock_id   Tux_stock_history_stock_id)uniquename
backgroundzJ[StockHistory] Could not create unique index on stock_history.stock_id: %ssystem_meta)keyr   ux_system_meta_keyzC[StockHistory] Could not create unique index on system_meta.key: %s)_INDEXES_ENSUREDcreate_index	Exceptionloggerwarningstr)dbe r   E/var/www/html/Trade-python/app/v1/background/stock_history_refresh.py_ensure_indexes   s4   
	r    "STOCK_HISTORY_REFRESH_LOOP_SECONDS1800 STOCK_HISTORY_REFRESH_BATCH_SIZE10STOCK_HISTORY_REFRESH_SLEEP_SECz0.05STOCK_HISTORY_ERROR_LOG_PATHzstock_history_error_logs.txt run_idsymbolr   errorsc                 C   s   t sd S zpt jddd }|pd }|pd  }|s"W d S g }|p'g D ]&}t|}	|	dddd }	|| d|  d| d| d|	 d
 q(|sTW d S t	t d	d
d}
|

| W d    W d S 1 smw   Y  W d S  ty~   Y d S w )Nseconds)timespecZr'   
 z | azutf-8)encoding)ERROR_LOG_PATHr   utcnow	isoformatstripupperr   replaceappendopen
writelinesr   )r(   r)   r   r*   tssidsymlinesr   msgfr   r   r   _append_error_log?   s*   *&rB    STOCK_HISTORY_MAX_STOCKS_PER_RUN500zAsia/KolkataSTOCK_HISTORY_REFRESH_MODEbootstrap_then_eodSTOCK_HISTORY_EOD_START_HHMMz17:00STOCK_HISTORY_EOD_END_HHMMz18:30STOCK_HISTORY_EOD_WEEKDAYS_ONLY1)0falseno$STOCK_HISTORY_ALLOW_WEEKEND_BACKFILLrK   rJ   trueyes&STOCK_HISTORY_BOOTSTRAP_FRESHNESS_DAYS3 STOCK_HISTORY_RUN_LOG_COLLECTIONstock_history_run_logsc                 C       g | ]}|  r|   qS r   r6   r7   .0tr   r   r   
<listcomp>u   s     r[   STOCK_HISTORY_INSTRUMENT_TYPESzEQ,INDEX,c                 C   rV   r   rW   rX   r   r   r   r[      s    
%STOCK_HISTORY_ALLOWED_SYMBOL_SUFFIXES STOCK_HISTORY_EXCLUDE_NAME_REGEXzLGOI LOAN|TREASURY|T-BILL|TBILL|G-SEC|GSEC|GOVT|BHARATBOND|GOLDBOND|GOLD BOND"STOCK_HISTORY_EXCLUDE_SYMBOL_REGEXz2(?:-(SG|GS|TB|GB|N[0-9A-Z]{1,2}|IV|IT|W\d+)|INAV)$after_symbolrb   c                 C   s   ddiddd idd iddddt id}g }tr/|d	td
didtd
didtd
dig tr@|ddtiiddtiig |rF||d< trfddd tD }ddddiiiddd| diig|d< | rnd| i|d< |S )N$neFNSETr'   )z$existsrc   $in)	is_activeexchangeinstrument_tokenr   r   instrument_typer   i)$regexz$optionstradingsymbolr)   rk   z$nor|c                 S   s   g | ]}| d dqS )rm   r'   )r8   rY   sr   r   r   r[      s    z#_universe_query.<locals>.<listcomp>z$not-z-(z)$z$orz$gt)INSTRUMENT_TYPESEXCLUDE_NAME_REGEXextendEXCLUDE_SYMBOL_REGEXALLOWED_SYMBOL_SUFFIXESjoin)rb   qnorsuffix_groupr   r   r   _universe_query   s>   

rz   raw	default_h	default_mc                 C   s   zQ| pd  d}t|dkr||fW S t|d }t|d }d|  kr+dkrCn nd|  kr7dkrMn n||fW S W ||fS W ||fS W ||fS W ||fS  ty]   Y ||fS w )Nr'   :   r   r      ;   )r6   splitlenintr   )r{   r|   r}   partshmr   r   r   _parse_hhmm   s(   
0
r   now_utcc                 C   s~   | pt  } | jtddt}ttdd\}}ttdd\}}|j||ddd}|j||ddd}||  ko<|kS   S )	NUTCtzinfo   (   r      r   )hourminutesecondmicrosecond)	r   r4   r8   r   
astimezoneISTr   EOD_START_HHMMEOD_END_HHMM)r   now_istshsmehemstartendr   r   r   _is_in_eod_window   s   r   c                 C   s.   | pt  } | jtddt}| dk S )Nr   r      )r   r4   r8   r   r   r   weekday)r   r   r   r   r   _is_weekday_ist   s   r   c                 C   s   | pt  } | jtddt}| }| dkr3|j|jd}ddl m	} ||dd }|
 S | d	krFddl m	} ||d
d }|
 S )zBest-effort latest trading day (weekdays only).

    We don't have exchange holiday calendars here; this simply maps Sat/Sun -> Fri.
    r   r   r   )dayr   )	timedeltar   )days   r   )r   r4   r8   r   r   r   dater   r   r   r5   )r   r   d_tdr   r   r   _latest_trading_day_ist   s   r   c                 C   s   t | S )zReturn the trading-day key that this EOD refresh corresponds to.

    This should match the latest trading day (weekends roll back to Friday),
    not the calendar date the job happened to run.
    )r   )r   r   r   r   _last_eod_run_key  s   r   c                 C   s@   z| d  ddipi }t|dpdW S  ty   Y dS w )Nr   r   "stock_history_refresh_last_eod_runvaluer'   )find_oner   getr   r   docr   r   r   _load_last_eod_run  s   r   r   c                 C   L   z| d j ddiddt|pdt didd W d S  ty%   Y d S w )	Nr   r   r   $setr'   r   r   
updated_atTupsert
update_oner   r   r4   r   r   r   r   r   r   _save_last_eod_run     r   r   c                 C   sD   zt |ts	W dS | t | W dS  ty!   td Y dS w )z2Persist a small run summary (never store candles).Nz&[StockHistory] Failed to write run log)
isinstancedictRUN_LOG_COLLECTION
insert_oner   r   	exceptionr   r   r   r   _save_run_log   s   
r      )sample_limitr   c             	      sF  zt | d t dddtd|}|sW dS dd |D }|s&W dS t | d	 d
d|iiddddd}t  |D ]C}t|d
pGd}|sLq>|d}t|t rbt	|dkrb 
| q>|d}t|tr|d}	t|	t rt	|	dkr 
| q>t fdd|D }
|
tdt	| dkW S  ty   Y dS w )zBest-effort heuristic: consider history "ready" if most sampled stocks have DAILY history.

    Weekly/monthly are derived at read-time from daily candles, so they are not a coverage requirement.
    stocksr   r   _idr      Fc                 S   $   g | ]}| d rt| d qS r   r   r   rn   r   r   r   r[   8     $ z(_history_coverage_ok.<locals>.<listcomp>r   r   re   )r   r   r   candlesr'   r   r   c                 3   s    | ]	}| v rd V  qdS )r   Nr   )rY   r=   has_dayr   r   	<genexpr>P  s    z'_history_coverage_ok.<locals>.<genexpr>皙?)listfindrz   limitmaxsetr   r   r   r   addr   sumr   )r   r   r   idshaverr=   arrr   arr2okr   r   r   _history_coverage_ok*  sD   &





r      r   max_age_daysr   c             
   C   sR  zt | d t dddtd|}|sW dS dd |D }|s'W dS t | d	 d
d|iidddddd}i }|D ]}t|d
pHd}|sMq?|d}	t|	trt|	 rtzt	
|	   ||< W q? tys   Y nw |d}
t|
t r|
s|d}t|tr|d}
t|
t r|
sq?|
d pi }|d}t|trzt	
|dd  ||< W q? ty   Y q?w t|t	r|  ||< q?|sW dS t }t	
|}d}|D ]0}||}|sqzt	
|}|| j}|dkr|t|kr|d7 }W q ty   Y qw |tdt| dkW S  ty(   Y dS w )zHeuristic: history is "fresh" if most sampled symbols have a recent day candle.

    We consider "recent" relative to the latest trading day (weekdays-only approximation).
    r   r   r   r   r   Fc                 S   r   r   r   rn   r   r   r   r[   f  r   z)_history_fresh_enough.<locals>.<listcomp>r   r   re   )r   r   day_last_dater   r   r'   r   r   r   r   r-   z+00:00r   )r   r   rz   r   r   r   r   r   r6   r   fromisoformatr   r5   r   r   r8   r   r   r   r   )r   r   r   r   r   docs
last_by_idr   r=   r   day_candlescandles_objlastdtlatest_trade_day	latest_dtr   ro   ager   r   r   _history_fresh_enoughW  s   







"




r   c                 C   sD   z| d  ddipi }|d}t|pdW S  ty!   Y dS w )Nr   r   stock_history_refresh_cursorr   r'   )r   r   r   r   )r   r   curr   r   r   _load_cursor  s   
r   c                 C   r   )	Nr   r   r   r   r'   r   Tr   r   r   r   r   r   _save_cursor  r   r   
batch_sizec              	   C   sF   t |d}| d |dddddddgtd|}t|}|S )Nra   r   r   r   )r   r)   r   rg   rh   )r)   r   )rz   r   sortr   r   r   )r   rb   r   rw   r   rowsr   r   r   _next_batch  s   
r   interval_secondsc           $         s|  t | pt} td| } tddpd  dv }	 zt D ]}t| t	
 jd d }t }t}d }d}d}d}	d }
d }d}z#t }|d	 |}d
didd
d id
d idtid}|d	 |}W n tyt   d }d }Y nw tdv rt|ot|dtdtd}t|}tdkptdkr| ptdko|}t|}|rtrt stsd} nt sd} nt }t||krd} nt|}|std  nd}t  }t }t!d||tt"t#t$||	 |tdt%k r
|st rd}|rt sd}nt&|}t'||t(}|s$|rt)|d t!d d}nt!d d}n|}|D ]}|tdt%kr4 n|*dp;d + }|sEq(t,|*d pMd}z;|r|d! -d |iddd"pbi }|*d#}t.|t,r| r| |kr|}|d7 }|d7 }|}
W q(W n
 ty   Y nw t/|||d$gd%} |}|d7 }|d7 }|}
t.| t0r| *d&}!t.|!t1r|!rd$|!v r|d7 }| *d'}"t.|"t1r|"r|	t2|"7 }	t3||||"d( t4dkrt56t4I d H  q(t)|| t!d)|t2|||| |tdt%k stdv r|r|dkrt7|t  |d u r/|tdt%kr-d*}nd}zdt j8t9d+d,:t;}#t<|i d-t d.|#= > d/|d0|d1|d2td3t$d4t"d5t#d6|d7|d8t?t@d9d:td;td<t(d=t4d*t%|d$|i|	|
|tAt | B d>d? W n ty   tCd@ Y nw  W n ty   tCdA Y nw t56| I d H  q)BN<   "STOCK_HISTORY_BOOTSTRAP_CONTINUOUSrK   rO   T   r   Fr   rc   rd   re   )rf   rg   rh   r   ri   )eodrF   r   r   r   r   rF   not_weekdayoutside_eod_windowalready_ran_todayz:[StockHistory] Global Zerodha client unavailable; skippingz[StockHistory] Run start id=%s mode=%s instrument_types=%s excluded_name=%s excluded_symbol=%s allowed_suffixes=%s universe_raw=%s universe_filtered=%sfell_outside_eod_windowr'   z8[StockHistory] Reached end of universe; resetting cursor	completedz)[StockHistory] No stocks found to refresh	no_stocksr)   r   r   )r   r   r   r   )r   zerodha_clientstock
timeframesupdatedr*   )r(   r)   r   r*   zG[StockHistory] id=%s Refreshed batch size=%d cursor=%s->%s processed=%dmax_stocks_per_runr   r   	timestampist_datemodebootstrap_readystop_reasoninstrument_typesallowed_symbol_suffixesexclude_name_regexexclude_symbol_regexuniverse_totaluniverse_with_stock_id
eod_window)r   r   weekdays_onlyallow_weekend_backfillr   sleep_between_calls_secr   )	processedr  r*   cursor_last_symbolr(   duration_secondsz)[StockHistory] Failed to assemble run logz![StockHistory] Refresh loop error)Dr   LOOP_INTERVAL_SECONDSr   osgetenvr6   lowerr   r    uuiduuid4hexr   r4   MODErz   count_documentsrq   r   r   r   BOOTSTRAP_FRESHNESS_DAYSboolWEEKDAYS_ONLYr   ALLOW_WEEKEND_BACKFILLr   r   r   r	   r   r   r   inforr   rt   ru   MAX_STOCKS_PER_RUNr   r   
BATCH_SIZEr   r   r7   r   r   r   r
   r   r   r   rB   SLEEP_BETWEEN_CALLS_SECasynciosleepr   r8   r   r   r   r   r   r5   r   r   roundtotal_secondsr   )$r   BOOTSTRAP_CONTINUOUSr   r(   session_startedsession_modesession_bootstrap_readysession_processedsession_updated_daysession_errorssession_last_symbolsession_stop_reasonsession_enforce_eod_schedulebase_qr  raw_qr  r  enforce_eod_schedule	today_keyzerodhar  r   touched_eod_windowcursor_beforebatchlast_symbolr  r>   r=   r   r   resr  errsr   r   r   r   stock_history_refresh_loop  s  








 
$




Q
	

 rE  )r   N)N)Ar,  loggingr  r  r   typingr   r   r   r   zoneinfor   app.db.databaser   !app.v1.background.global_intradayr	   app.v1.services.stock_historyr
   	getLogger__name__r   r   r    r   r  r  r*  floatr+  r6   r3   r   rB   r)  r   r  r"  r   r   r&  r'  r$  r   r   rq   ru   rr   rt   rz   tupler   r%  r   r   r   r   r   r   r   r   r   r   r   r   rE  r   r   r   r   <module>   sx    

#" 
 "/
	
-U	"