o ݈ h @s,ddlZddlZddlZddlmZmZddlmZddlm Z m Z m Z m Z ddl ZddlZddlmZddlmZddlmZddlmZdd lmZdd lmZmZmZejjd d d \Z Z!e!\Z"Z#Z$Z%Z%ddZ&ddZ'ddZ(ddZ)ddZ*ddZ+ddZ,ddZ-ddZ.eGd d!d!Z/eGd"d#d#Z0eGd$d%d%Z1eGd&d'd'Z2d(d)Z3d*d+Z4d,e e5fd-d.Z6d/e e e5e7fd0e8d1e7d2e7fd3d4Z9d5e7d6e7d7e7d1e7d0e8f d8d9Z:d:e e5e fd;e8d0e8fdejfdFdGZ?dS)HN) dataclassfield)Path)AnyDictListUnion)IndicNormalizerFactory)EncDecCTCModel)EncDecHybridRNNTCTCModel)logging)tqdm) BLANK_TOKEN SPACE_TOKENV_NEGATIVE_NUMzsnakers4/silero-vad silero_vad) repo_or_dirmodelcCs4t|j| d}td|j}|dd}|S)N_ -)rpartsjoinstemreplace)audio_filepathaudio_filepath_parts_in_utt_idfp_partsutt_idrX/var/www/eduai.edurigo.com/doc_train/edurigo_ai/Puru/tts/BhasaAnuvaad/utils/data_prep.py _get_utt_id#s r!cCs|t|d}tdd|D}Wdn1swYddtd||D}dd|D}|d||||fS) zN Get the start and end ids of the lines we will use for each 'batch'. rcss|]}dVqdS)Nr).0rrrr 0sz(get_batch_starts_ends..NcSg|]}|qSrrr$xrrr 2z)get_batch_starts_ends..rcSsg|]}|dqS)r#rr'rrr r)3)opensumrangepopappend)manifest_filepath batch_sizefnum_lines_in_manifeststartsendsrrr get_batch_starts_ends*s   r7cCsTd}t|d}|D] }t|}||vrd}q Wd|S1s#wY|S)zV Returns True if entry is a key in any of the JSON lines in manifest_filepath Fr"TNr,jsonloads)r1entryentry_in_manifestr3linedatarrr is_entry_in_any_lines:s   r?cCs^t|d }|D]}t|}||vrWddSqWddS1s(wYdS)zW Returns True is entry is a key in all of the JSON lines in manifest_filepath. r"NFTr8)r1r;r3r=r>rrr is_entry_in_all_linesKs   r@c CsTt}z||}Wn|YS|tddtjddd}||}|S)z Perform NFC -> NFD normalization for a sentence and a given language sentence: string lang_code: language code in ISO format |u।۔'-) r get_normalizer translatestr maketransstring punctuationr normalize)sentence lang_codefactory normalizernormalized_sentencerrr normalize_sentenceYs rOc Csg}t|ddd^}t|D]H\}}||krP||krPt|}d|vrK|ddd|d<d|d|d<|ddd |d<t|d||d<||||krVn qWd|SWd|S1sjwY|S) Nr"z utf-8-sig)encodingtexturAru…z...) r, enumerater9r:rrsplitrOr0) r1startendlang_idmanifest_lines_batchr3line_ir=r>rrr get_manifest_lines_batchks,    rYcCsFg}|D]}||jjvr||jj|q|t|jjq|S)N)decoder vocabularyr0indexlen)rQrtokens characterrrr get_char_tokenss  r`cCsiddddddddd d d d d ddddddddddddddddd dd dddddd}||vrG|||krGdSdS)zFreturns True if ref_text is a subscript or superscript version of textu⁰0¹1²2³3u⁴4u⁵5u⁶6u⁷7u⁸8u⁹9u₀u₁u₂u₃u₄u₅u₆)u₇u₈u₉TFr)ref_textrQsub_or_superscript_to_numrrr is_sub_or_superscript_pairsV      rpcCsJd|vr|dd}d|vsd|vr|dd}d|vsdd|D}|S)Nu▁▁u▁__rcSr&rr)r$tokenrrr r)r*z&restore_token_case..)r)word word_tokensword_tokens_casedrrr restore_token_cases  #rvc@sVeZdZUdZeed<dZeed<dZeed<dZ eed<dZ e ed<dZ e ed<dS)TokenNrQ text_caseds_starts_endt_startt_end) __name__ __module__ __qualname__rQrE__annotations__rxryintrzr{floatr|rrrr rws     rwc@s`eZdZUdZeed<dZeed<dZeed<dZ e ed<dZ e ed<e e dZeeed<dS) WordNrQryrzr{r|default_factoryr^)r}r~rrQrErryrrzr{rr|rlistr^rrwrrrr rs     rc@sleZdZUdZeed<dZdZeed<dZ eed<dZ e ed<dZ e ed<e edZeeeefed<dS) SegmentNrQryrzr{r|rwords_and_tokens)r}r~rrQrEr pred_textryrrzr{rr|rrrrrrrwrrrr rs     "rc@sxeZdZUeedZeeed<eedZ ee e e fed<dZ eed<dZeed<dZeed<eedZeed<dS) Utterancertoken_ids_with_blankssegments_and_tokensNrQrrsaved_output_files)r}r~rrrrrrrrrrrwrQrErrdictrrrrr rs    rc Cs|s|g}n||}dd|D}dd|D}t|||d}t|drzt|dr/|j} nt|jj} | g|_t|dkrA|S|j|} d} t dt| D]} | | | | dkr`| d7} qPt| | |krtt d |d |S|j tttddd d} d}|D]}|j|}|j t|| | t|d d d | t|d 7} |d}t|D]\}}|j|}|j|}t||}|j dj t|||t|d d d |t|d 7}tt|||D]P\}\}}}|j|| g|j djdj t||t|jd t|jd d |t|dkr=|j djdj tttt|jdt|jdd q|t|dkr`|j dj tttt|jdt|jdd q|j tttt|jdt|jdd q|St|jdr\t|jj} |jjd}| g|_t|dkr|St||} d} t dt| D]} | | | | dkr| d7} qt| | |krt d |d |S|j tttddd d} d}t|D]t\}}t||}|j t|| | t|d d d | t|d d 7} |d}t|D]\}}t|}t||}|j dj t|||t|d d d |t|d d 7}tt||D]V\}\}}|j|g|j djdj t||t|jdt|jdd |t|dkr|j| g|j djdj tttt|jdt|jdd qH|t|dkr|j| || g|j dj tttt|jdt|jdd |j dj tttt|jd t|jd d |j dj tttt|jdt|jdd q|j| g|j tttt|jdt|jdd |t|dkrX|j|| g|j tttt|jd t|jd d |j tttt|jdt|jdd q|Std)at Function to create an Utterance object and add all necessary information to it except for timings of the segments / words / tokens according to the alignment - that will be done later in a different function, after the alignment is done. The Utterance object has a list segments_and_tokens which contains Segment objects and Token objects (for blank tokens in between segments). Within the Segment objects, there is a list words_and_tokens which contains Word objects and Token objects (for blank tokens in between words). Within the Word objects, there is a list tokens tokens which contains Token objects for blank and non-blank tokens. We will be building up these lists in this function. This data structure will then be useful for generating the various output files that we wish to save. cSsg|]}|qSr)stripr$segrrr r)(r+zget_utt_obj..cSsg|] }t|dkr|qS)r)r]rrrr r)*s)rQrr tokenizerblank_idrr#z Utterance zv has too many tokens compared to the audio file duration. Will not generate output alignment files for this utterance.)rQrxryrz)rQryrzrr[z Cannot get tokens of this model.) rSrhasattrrr]rvocabr text_to_idsr.r inforr0rwrtext_to_tokensrrRrvrrzipextendr^rZr[r\r`rr RuntimeError)rQr language_id separatorTrrsegmentsuttBLANK_ID all_tokensn_token_repetitionsi_toksegment_s_pointerword_s_pointersegmentsegment_tokenswordsword_irsrtword_token_idsrutoken_irrtoken_id token_casedSPACE_ID i_segmenti_wordrrr get_utt_objs                                                          rc Cst}t}d}t|D]\}}||kr"|||<|dkr"|d||<|}q t|d||<|jD]}t|tur|} || j|| _|| jd|| _ | j D]m} t| t ur| } || j|| _|| jd|| _ | j D]&} | j|vr~|| j|| _nd| _| j|vr|| jd|| _ qnd| _ qnqO| } | j|vr|| j|| _nd| _| j|vr|| jd|| _ qOd| _ qOq0|} | j|vr|| j|| _nd| _| j|vr|| jd|| _ q0d| _ q0|S)a| Function to add t_start and t_end (representing time in seconds) to the Utterance object utt_obj. Args: utt_obj: Utterance object to which we will add t_start and t_end for its constituent segments/words/tokens. alignment_utt: a list of ints indicating which token does the alignment pass through at each timestep (will take the form [0, 0, 1, 1, ..., ]). output_timestep_duration: a float indicating the duration of a single output timestep from the ASR Model. Returns: utt_obj: updated Utterance object. rrr#) rrRr]rtyperryr{rzr|rrr^) utt_obj alignment_uttoutput_timestep_duration!num_to_first_alignment_appearance num_to_last_alignment_appearanceprev_stssegment_or_tokenr word_or_tokenrsrrrrr add_t_start_end_to_utt_objs              -   raudio_filepaths_batchc Cs~d}t}|D]}t|}t|t}t|dkr$|dt|ddtdt|dD].}||ddd||d<t||d|d||d<t||d|d||d <q-t|d|d d<t|d d|d|d d<t|d d|d|d d <t||d d }t }g} t |D]\}} |j d |d} t | || d| d| | q| |d||<q|S)Ni>rr#)rTrUrTrUr start_secsend_secsr/z.wav) chunk_pathsspeech_timestamps)r read_audioget_speech_timestampsrr]r0r.roundadjust_timestampstempfileTemporaryDirectoryrRname save_audio) r sample_rate tmp_paths file_pathwavriadjusted_timestamps tmpdirnamecurrent_chunk_paths speech_dictpathrrr get_chunkssH       rrrmax_chunk_duration_smin_chunk_duration_sc Cs|sgSd|dd<|D]}t|d|d|d<t|d|d|d<q g}|dd}|dd}|dd}tdt|D]0} ||} | |krQ|| d}q@t||| ||} || || d}|| d}|| d}q@||} t||| ||} || |S)z Takes the speech timestamps output by the vad model and further splits/merges based on the max_chunk_duration_s/min_chunk_duration_s. Returns a list of adjusted timestamps. rrTrrrUrr#)rr.r]windowed_chunkingr) rrrrrr curr_startcurr_start_secs curr_end_secsrchunk_durationchunked_timestampsrrr r6sH          rrrrc Csg}|}|}|}|dkrA||dkr|n|} || 8}|t||t|| ||| d|ddd}t||d}|dks |S)a` Checks if the chunk duration is greater than the max_chunk_duration_s. If it is greater, divide into chunks of max_chunk_duration_s till chunk duration is fully split and return split timestamps. For example, if chunk duration is 10 seconds and max_chunk_duration_s is 3 seconds, the split timestamps will be 3s, 3s, 3s, 1s. r)rTrrUrrrUr#r)r0rr) rrrrrrrTrremaining_chunk_duration chunk_sizerrr rws*  rspeech_timestamp timestepscCs&|d|d|}t||d}|S)NrUrTr)r)rrrrrrrr get_output_timestep_durations routputoutput_timestep_duration_logitscCst|d|d|}|jd|kr|d|ddfS||jd}t||jdfj|jd}d|dddf<tj||fddS)NrUrTrr#)devicer)axis)rshapetorchzerostorcat)rrrrequired_timestepsblanks_timestepsblanksrrr resize_model_outputsrFrruse_silero_vadc 6 Cstdd|D} t| } g}g}g}| s| s|rt| }tg}d}|D]\}}|j|dd||d}t|tkrHt|dkrH|d}|s]t|dd|dj j d|j j j }||j j j }t|dj |dd||d_ ||dtd t|D]0}t||j |d||||_ t|d j ||j f|d _ |d jd ||j7_q|q)Wd n1swYn6|j| d| |d}t|tkrt|dkr|d}nt|j| d| d }Wd n1swY|D]}||j ||j j d||jqn=| d}| d}| d}t| ddD]*}||||||j||dd\}}||||j d||q,g}g} g}!t|D]|\}"}#|rrd ||"}$n|#ddd}$t|$|||||"| |"t| |"|}%|r||"|%_t|%jdkrt d|%j!dd|#vr|#d|%_n|#d|%_t|%jdkrt d|%j!d||%j"| t|%j"|!|%qat#|}&t#| }'t$|drt|j%j&d }(nt|j'j(d }(t)|})t)| }*t*t+| |&|(f}+t|D]\},}-|-j d}.|-|+|,d |.d d f<q|(tj+| |'ftj,d}/t|D]\},}0|*|,}1t)|0|/|,d |1f<q;|d urd|j j vr`t-dd|j j vrkt-dt./| d}2|2j0|2j1}3Wd n 1swY|3|j j j2}4t3|4t4|)d}5|j j5j6|5|j j j }t d |5d!|d"|+|/|)|*|!|fS)#az Returns: log_probs, y, T, U (y and U are s.t. every other token is a blank) - these are the tensors we will need during Viterbi decoding. utt_obj_batch: a list of Utterance objects for every utterance in the batch. output_timestep_duration: a float indicating the duration of a single output timestep from the ASR Model. cSsg|]}|dqS)rr)r$r=rrr r)r+z'get_batch_variables..rrT)return_hypothesesr2rrrr#rrN)rr2delaymodel_stride_in_secstokens_per_chunkzSample:)desc) keep_logitsrQu�rAz'pred_text' of utterance zN is empty - we will not generate any output alignment files for this utterancez'text' of utterance r)dtype window_stridezDon't have attribute 'window_stride' in 'model.cfg.preprocessor' => cannot calculate model_downsample_factor => stopping processrzDon't have attribute 'sample_rate' in 'model.cfg.preprocessor' => cannot calculate start and end time of segments => stopping processz/Calculated that the model downsample factor is z9 and therefore the ASR model output timestep duration is z! -- will use this for all batches)7r]rrno_graditems transcribertupler y_sequencercfg preprocessorrrr0r.rrQcleanup)transcribe_simulate_cache_aware_streamingr resetread_audio_filerRrrSrrr!rr rrrmaxrrrrZr[tensorronesint64 ValueErrorsf SoundFileframes sampleraterrr featurizer hop_length)6rWrrralign_using_pred_textrrrvad_chunked_batch_sizesimulate_cache_aware_streaminguse_buffered_chunked_streamingbuffered_chunk_paramsrBlog_probs_list_batch T_list_batchpred_text_batch file_paths hypothesesrtmp_dirvtmp_hypothesesr hypothesisrrrlhyplogits y_list_batch U_list_batch utt_obj_batchi_liner=gt_text_for_alignmentrT_maxU_maxVT_batchU_batchlog_probs_batchb log_probs_uttry_batchy_uttU_uttr3 audio_durn_input_framesmodel_downsample_factorrrr get_batch_variablessJ        /               r;)@r9rGr dataclassesrrpathlibrtypingrrrr soundfilerr"indicnlp.normalize.indic_normalizer &nemo.collections.asr.models.ctc_modelsr 2nemo.collections.asr.models.hybrid_rnnt_ctc_modelsr nemo.utilsr tqdm.autor utils.constantsrrrhubloadrutilsrrrrr!r7r?r@rOrYr`rprvrwrrrrrrErrrrrrTensorrboolr;rrrr s       .    q1 A /