o jl>@sddlmZddlZddlmZddlmZddlmZm Z m Z m Z m Z m Z mZddlmZmZmZddlmZdd lmZmZd6ddZd7ddZed6ddZed6ddZee dd8ddZ d9d:d d!Zd;d$d%Zdd.d/Z! d?d@d4d5Z"dS)A) annotationsN)IncrementalDecoder) lru_cache) FREQUENCIESKO_NAMESLANGUAGE_SUPPORTED_COUNTTOO_SMALL_SEQUENCEZH_NAMES_FREQUENCIES_SET_FREQUENCIES_RANK)_ASCII_CHAR_INFO _char_info is_suspiciously_successive_range)CoherenceMatches)is_multi_byte_encodingis_unicode_range_secondary iana_namestrreturn list[str]cst|rtdtd|j}|dd}idtddD];}|t|g}|rZt|}|dkr8t |jnt |j}|d urBqt |sZ|vrNd|<|d 7<d 7qt fd d DS) zF Return associated unicode ranges in a single byte code page. z.Function not supported on multi-byte code pagez encodings.ignore)errorsr@Nrcs g|] }|dkr|qS)g333333?).0character_rangecharacter_count seen_rangesr/root/aizidognhua/tmp/workspace/projects/ec89d86c-575f-41c9-af57-ac45cbdbf775/venv/lib/python3.10/site-packages/charset_normalizer/cd.py <s z*encoding_unicode_range..) rOSError importlib import_modulerrangedecodebytesordr rrsorted)rdecoderpichunkchunk_codepointrrrr"encoding_unicode_ranges:   r1 primary_rangecCs\g}tD]%\}}|D]}t|}|dkrt|nt|}|j|kr*||nq q|S)z> Return inferred languages used with a unicode range. r)ritemsr*r rr'append)r2 languageslanguage characters character codepointinforrr"unicode_range_languagesDs  r;cCsXzt|}Wn tygYSwd}|D] }d|vr |}nq|dur(dgSt|S)z Single-byte encoding language association. Some code page are heavily linked to particular language(s). This function does the correspondence. NLatin Latin Based)r1 ImportErrorr;)runicode_rangesr2specified_rangerrr"encoding_languagesYs  rAcCs`|ds|ds|ds|dkrdgS|ds|tvr"dgS|ds+|tvr.d gSgS) z Multi-byte encoding language association. Some code page are heavily linked to particular language(s). This function does the correspondence. shift_ iso2022_jpeuc_jcp932JapanesegbChinese iso2022_krKorean) startswithr r)rrrr"mb_encoding_languagesqsrL)maxsizer6tuple[bool, bool]cCsZd}d}t|D] }t|}|dkrt|nt|}|s!|jr!d}|r(|js(d}q||fS)zg Determine main aspects from a supported language if it contains accents and if is pure Latin. FTr)rr*r r accentuatedlatin)r6target_have_accentstarget_pure_latinr8r9r:rrr"get_target_featuress   rSFr7ignore_non_latinboolcCsg}t|}d}|D]}t|}|dkrt|nt|}|jr#d}nq tD]/\}} t|\} } |r7| s7q(| s<|rz$alphabet_languages..keyreversecSg|]}|dqSrr)rcompatible_languagerrr"r#sz&alphabet_languages..) frozensetr*r rrOrr3rSlenr r4r+)r7rTr5characters_setsource_have_accentsr8r9r:r6language_charactersrQrRr character_match_countratiorrr"alphabet_languagess0 riordered_charactersfloatcCs|tvr t|dd}t|}t|}tt|}|dk}|d}||}g} g} t|D]\} } || } | durF| | | | q/t| | D]p\}}t||}|sat ||dkraqL|rpt |||krp|d7}qL|dkry|d7}qL||}d}d}t| | D]5\}}||kr||kr|d7}d|d |kr|d7}nq||kr|d7}d|d |kr|d7}nqqL|t|S) aN Determine if a ordered characters list (by occurrence from most appearance to rarest) match a particular language. The result is a ratio between 0. (absolutely no correspondence) and 1. (near perfect fit). Beware that is function is not strict on the match in order to ease the detection. (Meaning close match is 1.) z not availablerNr) r ValueErrorr rc enumerategetr4zipintabs)r6rjcharacter_approved_count lang_rankordered_characters_count target_language_characters_countlarge_alphabetlarge_alphabet_thresholdexpected_projection_ratio common_lr common_orrpopularity_rankr8 language_rankcharacter_rank_in_languagecharacter_rankcharacter_rank_projection after_lenbefore_match_countafter_match_countlr_iorr_irrr"characters_popularity_comparesh       rdecoded_sequencec Cs i}d}d}d}d}|D]o}t|}|dkrt|}nt|}|js#q |j} | dur+q | |kr;|dur:|||q d} |rN|D] } t| | sL| } nqAn |durYt|| sY|} | dur_| } | |vrpg|| <|durn| }nd}|| || }| }q dd|DS)a Given a decoded text sequence, return a list of str. Unicode range / alphabet separation. Ex. a text containing English/Latin with a bit a Hebrew will return two items in the resulting list; One containing the latin letters and the other hebrew. NFrTcSsg|] }d|qS))joinlower)rcharsrrr"r#`sz'alpha_unicode_split..)r*r ralphar'r4rvalues) rlayerssingle_layer_key multi_layerprev_character_rangeprev_layer_targetr8r9r:rlayer_target_rangediscovered_rangerrr"alpha_unicode_splitsV  rresultslist[CoherenceMatches]rcsfi|D]}|D]}|\}}|vr|g|<q||qqfddD}t|ddddS)z This function merge results previously given by the function coherence_ratio. The return type is the same as coherence_ratio. cs.g|]}|tt|t|dfqS)rn)roundsumrc)rr6per_language_ratiosrr"r#qsz*merge_coherence_ratios..cSrVrWrrXrrr"rZ|r[z(merge_coherence_ratios..Tr\)r4r+)rresult sub_resultr6rhmergerrr"merge_coherence_ratioscs   rcst|D]}|\}}|dd}|vrg|<||qtfddDr@g}D] }||t|fq0|S|S)u We shall NOT return "English—" in CoherenceMatches because it is an alternative of "English". This function only keeps the best match and remove the em-dash in it. u—rc3s |] }t|dkVqdS)rN)rc)re index_resultsrr" sz/filter_alt_coherence_matches..)dictreplacer4anymax)rrr6rh no_em_namefiltered_resultsrrr"filter_alt_coherence_matchess r皙? threshold lg_inclusion str | NonecCsg}d}d}|dur|dng}d|vrd}|dt|D]X}i}|D] } || dd|| <q&t|} | tkr.cSrVrWr)rrrr"rZr[z!coherence_ratio..r\g?rnrmcSrVrWrrXrrr"rZr[) splitremoverrsrcr r+r3rirr4rr)rrrrrTsufficient_match_countlg_inclusion_listlayer char_countslayer_characterr popular_character_orderedr6rhrrr"coherence_ratiosJ  r)rrrr)r2rrr)r6rrrN)F)r7rrTrUrr)r6rrjrrrk)rrrr)rrrr)rrrr)rN)rrrrkrrrr)# __future__rr%codecsr functoolsrconstantrrrr r r r mdr rrmodelsrutilsrrr1r;rArLrSrirrrrrrrrr"s0   $   ,    ( Y I