ÿØÿà JFIF    ÿÛ „ ( %!1!%*+...983,7(-.- PK[]YtOD+D+cd.pynu[import importlib from codecs import IncrementalDecoder from collections import Counter, OrderedDict from functools import lru_cache from typing import Dict, List, Optional, Tuple from .assets import FREQUENCIES from .constant import KO_NAMES, LANGUAGE_SUPPORTED_COUNT, TOO_SMALL_SEQUENCE, ZH_NAMES from .md import is_suspiciously_successive_range from .models import CoherenceMatches from .utils import ( is_accentuated, is_latin, is_multi_byte_encoding, is_unicode_range_secondary, unicode_range, ) def encoding_unicode_range(iana_name: str) -> List[str]: """ Return associated unicode ranges in a single byte code page. """ if is_multi_byte_encoding(iana_name): raise IOError("Function not supported on multi-byte code page") decoder = importlib.import_module("encodings.{}".format(iana_name)).IncrementalDecoder # type: ignore p = decoder(errors="ignore") # type: IncrementalDecoder seen_ranges = {} # type: Dict[str, int] character_count = 0 # type: int for i in range(0x40, 0xFF): chunk = p.decode(bytes([i])) # type: str if chunk: character_range = unicode_range(chunk) # type: Optional[str] if character_range is None: continue if is_unicode_range_secondary(character_range) is False: if character_range not in seen_ranges: seen_ranges[character_range] = 0 seen_ranges[character_range] += 1 character_count += 1 return sorted( [ character_range for character_range in seen_ranges if seen_ranges[character_range] / character_count >= 0.15 ] ) def unicode_range_languages(primary_range: str) -> List[str]: """ Return inferred languages used with a unicode range. """ languages = [] # type: List[str] for language, characters in FREQUENCIES.items(): for character in characters: if unicode_range(character) == primary_range: languages.append(language) break return languages @lru_cache() def encoding_languages(iana_name: str) -> List[str]: """ Single-byte encoding language association. Some code page are heavily linked to particular language(s). This function does the correspondence. """ unicode_ranges = encoding_unicode_range(iana_name) # type: List[str] primary_range = None # type: Optional[str] for specified_range in unicode_ranges: if "Latin" not in specified_range: primary_range = specified_range break if primary_range is None: return ["Latin Based"] return unicode_range_languages(primary_range) @lru_cache() def mb_encoding_languages(iana_name: str) -> List[str]: """ Multi-byte encoding language association. Some code page are heavily linked to particular language(s). This function does the correspondence. """ if ( iana_name.startswith("shift_") or iana_name.startswith("iso2022_jp") or iana_name.startswith("euc_j") or iana_name == "cp932" ): return ["Japanese"] if iana_name.startswith("gb") or iana_name in ZH_NAMES: return ["Chinese", "Classical Chinese"] if iana_name.startswith("iso2022_kr") or iana_name in KO_NAMES: return ["Korean"] return [] @lru_cache(maxsize=LANGUAGE_SUPPORTED_COUNT) def get_target_features(language: str) -> Tuple[bool, bool]: """ Determine main aspects from a supported language if it contains accents and if is pure Latin. """ target_have_accents = False # type: bool target_pure_latin = True # type: bool for character in FREQUENCIES[language]: if not target_have_accents and is_accentuated(character): target_have_accents = True if target_pure_latin and is_latin(character) is False: target_pure_latin = False return target_have_accents, target_pure_latin def alphabet_languages( characters: List[str], ignore_non_latin: bool = False ) -> List[str]: """ Return associated languages associated to given characters. """ languages = [] # type: List[Tuple[str, float]] source_have_accents = any(is_accentuated(character) for character in characters) for language, language_characters in FREQUENCIES.items(): target_have_accents, target_pure_latin = get_target_features(language) if ignore_non_latin and target_pure_latin is False: continue if target_have_accents is False and source_have_accents: continue character_count = len(language_characters) # type: int character_match_count = len( [c for c in language_characters if c in characters] ) # type: int ratio = character_match_count / character_count # type: float if ratio >= 0.2: languages.append((language, ratio)) languages = sorted(languages, key=lambda x: x[1], reverse=True) return [compatible_language[0] for compatible_language in languages] def characters_popularity_compare( language: str, ordered_characters: List[str] ) -> float: """ Determine if a ordered characters list (by occurrence from most appearance to rarest) match a particular language. The result is a ratio between 0. (absolutely no correspondence) and 1. (near perfect fit). Beware that is function is not strict on the match in order to ease the detection. (Meaning close match is 1.) """ if language not in FREQUENCIES: raise ValueError("{} not available".format(language)) character_approved_count = 0 # type: int for character in ordered_characters: if character not in FREQUENCIES[language]: continue characters_before_source = FREQUENCIES[language][ 0 : FREQUENCIES[language].index(character) ] # type: List[str] characters_after_source = FREQUENCIES[language][ FREQUENCIES[language].index(character) : ] # type: List[str] characters_before = ordered_characters[ 0 : ordered_characters.index(character) ] # type: List[str] characters_after = ordered_characters[ ordered_characters.index(character) : ] # type: List[str] before_match_count = [ e in characters_before for e in characters_before_source ].count( True ) # type: int after_match_count = [ e in characters_after for e in characters_after_source ].count( True ) # type: int if len(characters_before_source) == 0 and before_match_count <= 4: character_approved_count += 1 continue if len(characters_after_source) == 0 and after_match_count <= 4: character_approved_count += 1 continue if ( before_match_count / len(characters_before_source) >= 0.4 or after_match_count / len(characters_after_source) >= 0.4 ): character_approved_count += 1 continue return character_approved_count / len(ordered_characters) def alpha_unicode_split(decoded_sequence: str) -> List[str]: """ Given a decoded text sequence, return a list of str. Unicode range / alphabet separation. Ex. a text containing English/Latin with a bit a Hebrew will return two items in the resulting list; One containing the latin letters and the other hebrew. """ layers = OrderedDict() # type: Dict[str, str] for character in decoded_sequence: if character.isalpha() is False: continue character_range = unicode_range(character) # type: Optional[str] if character_range is None: continue layer_target_range = None # type: Optional[str] for discovered_range in layers: if ( is_suspiciously_successive_range(discovered_range, character_range) is False ): layer_target_range = discovered_range break if layer_target_range is None: layer_target_range = character_range if layer_target_range not in layers: layers[layer_target_range] = character.lower() continue layers[layer_target_range] += character.lower() return list(layers.values()) def merge_coherence_ratios(results: List[CoherenceMatches]) -> CoherenceMatches: """ This function merge results previously given by the function coherence_ratio. The return type is the same as coherence_ratio. """ per_language_ratios = OrderedDict() # type: Dict[str, List[float]] for result in results: for sub_result in result: language, ratio = sub_result if language not in per_language_ratios: per_language_ratios[language] = [ratio] continue per_language_ratios[language].append(ratio) merge = [ ( language, round( sum(per_language_ratios[language]) / len(per_language_ratios[language]), 4, ), ) for language in per_language_ratios ] return sorted(merge, key=lambda x: x[1], reverse=True) @lru_cache(maxsize=2048) def coherence_ratio( decoded_sequence: str, threshold: float = 0.1, lg_inclusion: Optional[str] = None ) -> CoherenceMatches: """ Detect ANY language that can be identified in given sequence. The sequence will be analysed by layers. A layer = Character extraction by alphabets/ranges. """ results = [] # type: List[Tuple[str, float]] ignore_non_latin = False # type: bool sufficient_match_count = 0 # type: int lg_inclusion_list = lg_inclusion.split(",") if lg_inclusion is not None else [] if "Latin Based" in lg_inclusion_list: ignore_non_latin = True lg_inclusion_list.remove("Latin Based") for layer in alpha_unicode_split(decoded_sequence): sequence_frequencies = Counter(layer) # type: Counter most_common = sequence_frequencies.most_common() character_count = sum(o for c, o in most_common) # type: int if character_count <= TOO_SMALL_SEQUENCE: continue popular_character_ordered = [c for c, o in most_common] # type: List[str] for language in lg_inclusion_list or alphabet_languages( popular_character_ordered, ignore_non_latin ): ratio = characters_popularity_compare( language, popular_character_ordered ) # type: float if ratio < threshold: continue elif ratio >= 0.8: sufficient_match_count += 1 results.append((language, round(ratio, 4))) if sufficient_match_count >= 3: break return sorted(results, key=lambda x: x[1], reverse=True) PK[]}T::__pycache__/md.cpython-37.pycnu[B bG@s<ddlmZddlmZmZddlmZmZddlm Z m Z m Z m Z m Z mZmZmZmZmZmZmZmZmZmZGdddZGdd d eZGd d d eZGd d d eZGdddeZGdddeZGdddeZGdddeZGdddeZ ee!ee!e"dddZ#eddd#e!e$e"e$dd d!Z%d"S)$) lru_cache)ListOptional)COMMON_SAFE_ASCII_CHARACTERSUNICODE_SECONDARY_RANGE_KEYWORD)is_accentuatedis_asciiis_case_variableis_cjk is_emoticon is_hangul is_hiragana is_katakanais_latinis_punctuation is_separator is_symbolis_thai remove_accent unicode_rangec@sPeZdZdZeedddZeddddZddd d Ze e dd d Z dS) MessDetectorPluginzy Base abstract class used for mess detection plugins. All detectors MUST extend and implement given methods. ) characterreturncCstdS)z@ Determine if given character should be fed in. N)NotImplementedError)selfrrF/opt/alt/python37/lib/python3.7/site-packages/charset_normalizer/md.pyeligibleszMessDetectorPlugin.eligibleNcCstdS)z The main routine to be executed upon character. Insert the logic in witch the text would be considered chaotic. N)r)rrrrrfeed$szMessDetectorPlugin.feed)rcCstdS)zB Permit to reset the plugin to the initial state. N)r)rrrrreset+szMessDetectorPlugin.resetcCstdS)z Compute the chaos ratio based on what your feed() has seen. Must NOT be lower than 0.; No restriction gt 0. N)r)rrrrratio1szMessDetectorPlugin.ratio) __name__ __module__ __qualname____doc__strboolrrr propertyfloatr!rrrrrs rc@sZeZdZddddZeedddZedddd Zddd d Ze e dd d Z dS) TooManySymbolOrPunctuationPluginN)rcCs"d|_d|_d|_d|_d|_dS)NrF)_punctuation_count _symbol_count_character_count_last_printable_charZ_frenzy_symbol_in_word)rrrr__init__;s z)TooManySymbolOrPunctuationPlugin.__init__)rrcCs|S)N) isprintable)rrrrrrCsz)TooManySymbolOrPunctuationPlugin.eligiblecCsp|jd7_||jkrf|tkrft|r8|jd7_n.|dkrft|rft|dkrf|jd7_||_dS)NrF) r-r.rrr+isdigitrr r,)rrrrrrFs   z%TooManySymbolOrPunctuationPlugin.feedcCsd|_d|_d|_dS)Nr)r+r-r,)rrrrr Xsz&TooManySymbolOrPunctuationPlugin.resetcCs0|jdkrdS|j|j|j}|dkr,|SdS)Nrgg333333?)r-r+r,)rZratio_of_punctuationrrrr!]s   z&TooManySymbolOrPunctuationPlugin.ratio) r"r#r$r/r&r'rrr r(r)r!rrrrr*:s r*c@sZeZdZddddZeedddZedddd Zddd d Ze e dd d Z dS)TooManyAccentuatedPluginN)rcCsd|_d|_dS)Nr)r-_accentuated_count)rrrrr/jsz!TooManyAccentuatedPlugin.__init__)rrcCs|S)N)isalpha)rrrrrrnsz!TooManyAccentuatedPlugin.eligiblecCs(|jd7_t|r$|jd7_dS)Nr)r-rr4)rrrrrrqszTooManyAccentuatedPlugin.feedcCsd|_d|_dS)Nr)r-r4)rrrrr wszTooManyAccentuatedPlugin.resetcCs*|jdkrdS|j|j}|dkr&|SdS)Nrggffffff?)r-r4)rZratio_of_accentuationrrrr!{s  zTooManyAccentuatedPlugin.ratio) r"r#r$r/r&r'rrr r(r)r!rrrrr3is r3c@sZeZdZddddZeedddZedddd Zddd d Ze e dd d Z dS)UnprintablePluginN)rcCsd|_d|_dS)Nr)_unprintable_countr-)rrrrr/szUnprintablePlugin.__init__)rrcCsdS)NTr)rrrrrrszUnprintablePlugin.eligiblecCs@|dkr.|dkr.|dkr.|jd7_|jd7_dS)NFr)isspacer0r7r-)rrrrrrs   zUnprintablePlugin.feedcCs d|_dS)Nr)r7)rrrrr szUnprintablePlugin.resetcCs|jdkrdS|jd|jS)Nrg)r-r7)rrrrr!s zUnprintablePlugin.ratio) r"r#r$r/r&r'rrr r(r)r!rrrrr6s  r6c@sZeZdZddddZeedddZedddd Zddd d Ze e dd d Z dS)SuspiciousDuplicateAccentPluginN)rcCsd|_d|_d|_dS)Nr)_successive_countr-_last_latin_character)rrrrr/sz(SuspiciousDuplicateAccentPlugin.__init__)rrcCs|ot|S)N)r5r)rrrrrrsz(SuspiciousDuplicateAccentPlugin.eligiblecCst|jd7_|jdk rjt|rjt|jrj|rJ|jrJ|jd7_t|t|jkrj|jd7_||_dS)Nr)r-r=risupperr<r)rrrrrrs  z$SuspiciousDuplicateAccentPlugin.feedcCsd|_d|_d|_dS)Nr)r<r-r=)rrrrr sz%SuspiciousDuplicateAccentPlugin.resetcCs|jdkrdS|jd|jS)Nrgr1)r-r<)rrrrr!s z%SuspiciousDuplicateAccentPlugin.ratio) r"r#r$r/r&r'rrr r(r)r!rrrrr;s r;c@sZeZdZddddZeedddZedddd Zddd d Ze e dd d Z dS)SuspiciousRangeN)rcCsd|_d|_d|_dS)Nr)"_suspicious_successive_range_countr-_last_printable_seen)rrrrr/szSuspiciousRange.__init__)rrcCs|S)N)r0)rrrrrrszSuspiciousRange.eligiblecCsx|jd7_|s&t|s&|tkr0d|_dS|jdkrD||_dSt|j}t|}t||rn|jd7_||_dS)Nr)r-r9rrrAr is_suspiciously_successive_ranger@)rrunicode_range_aunicode_range_brrrrs  zSuspiciousRange.feedcCsd|_d|_d|_dS)Nr)r-r@rA)rrrrr szSuspiciousRange.resetcCs.|jdkrdS|jd|j}|dkr*dS|S)Nrgr1g?)r-r@)rZratio_of_suspicious_range_usagerrrr!s zSuspiciousRange.ratio) r"r#r$r/r&r'rrr r(r)r!rrrrr?s r?c@sZeZdZddddZeedddZedddd Zddd d Ze e dd d Z dS)SuperWeirdWordPluginN)rcCs:d|_d|_d|_d|_d|_d|_d|_d|_d|_dS)NrF) _word_count_bad_word_count_foreign_long_count_is_current_word_bad_foreign_long_watchr-_bad_character_count_buffer_buffer_accent_count)rrrrr/szSuperWeirdWordPlugin.__init__)rrcCsdS)NTr)rrrrrr szSuperWeirdWordPlugin.eligiblecCs|rd|j|g|_t|r0|jd7_|jdkrt|dksNt|rt|dkrt|dkrt |dkrt |dkrt |dkrd|_dS|jsdS| st |st|r|jr|jd7_t|j}|j|7_|dkr:|j|dkrd|_t|jdr:|jdr:|jd7_d|_|dkr`|jr`|jd7_d|_|jr|jd7_|jt|j7_d|_d|_d|_d |_n6|d kr|dkrt|rd|_|j|7_dS) NrFrFTg(\?r><>|=-_~)r5joinrMrrNrKrr r rrrr9rrrGlenr-rJr>rIrHrLr2r)rrZ buffer_lengthrrrrsR           zSuperWeirdWordPlugin.feedcCs4d|_d|_d|_d|_d|_d|_d|_d|_dS)NrFFr)rMrJrKrHrGr-rLrI)rrrrr DszSuperWeirdWordPlugin.resetcCs$|jdkr|jdkrdS|j|jS)N rg)rGrIrLr-)rrrrr!NszSuperWeirdWordPlugin.ratio) r"r#r$r/r&r'rrr r(r)r!rrrrrEs 6 rEc@s^eZdZdZddddZeedddZeddd d Zddd d Z e e dd dZ dS)CjkInvalidStopPluginu GB(Chinese) based encoding often render the stop incorrectly when the content does not fit and can be easily detected. Searching for the overuse of '丅' and '丄'. N)rcCsd|_d|_dS)Nr)_wrong_stop_count_cjk_character_count)rrrrr/\szCjkInvalidStopPlugin.__init__)rrcCsdS)NTr)rrrrrr`szCjkInvalidStopPlugin.eligiblecCs4|dkr|jd7_dSt|r0|jd7_dS)N>丅丄r)r]r r^)rrrrrrcs zCjkInvalidStopPlugin.feedcCsd|_d|_dS)Nr)r]r^)rrrrr jszCjkInvalidStopPlugin.resetcCs|jdkrdS|j|jS)Ng)r^r])rrrrr!ns zCjkInvalidStopPlugin.ratio) r"r#r$r%r/r&r'rrr r(r)r!rrrrr\Vsr\c@sZeZdZddddZeedddZedddd Zddd d Ze e dd d Z dS)ArchaicUpperLowerPluginN)rcCs.d|_d|_d|_d|_d|_d|_d|_dS)NFrT)_buf_character_count_since_last_sep_successive_upper_lower_count#_successive_upper_lower_count_finalr-_last_alpha_seen_current_ascii_only)rrrrr/vsz ArchaicUpperLowerPlugin.__init__)rrcCsdS)NTr)rrrrrrsz ArchaicUpperLowerPlugin.eligiblecCs$|ot|}|dk}|r|jdkr|jdkrV|dkrV|jdkrV|j|j7_d|_d|_d|_d|_|j d7_ d|_dS|jdkrt |dkrd|_|jdk r| r|j s| r|j r|jdkr|jd7_d|_qd|_nd|_|j d7_ |jd7_||_dS)NFr@rTr1) r5r rdr2rhrfrergrcr-r r>islower)rrZ is_concernedZ chunk_seprrrrs8      zArchaicUpperLowerPlugin.feedcCs.d|_d|_d|_d|_d|_d|_d|_dS)NrFT)r-rdrerfrgrcrh)rrrrr szArchaicUpperLowerPlugin.resetcCs|jdkrdS|j|jS)Nrg)r-rf)rrrrr!s zArchaicUpperLowerPlugin.ratio) r"r#r$r/r&r'rrr r(r)r!rrrrrbus  * rb)rCrDrcCs~|dks|dkrdS||kr dSd|kr4d|kr4dSd|ksDd|krHdSd|ksXd|krld|kshd|krldS|d|d}}x"|D]}|tkrq||krdSqW|dk|dk}}|s|rd |ksd |krdS|r|rdSd |ksd |kr"d |ksd |kr dS|d ks|d kr"dSd |ksJd |ksJ|d krz|d krzd |ks^d |krbdSd|ksvd|krzdSdS)za Determine if two Unicode range seen next to each other can be considered as suspicious. NTFZLatinZ EmoticonsZ Combining )HiraganaKatakanaZCJKZHangulz Basic Latin)rmrlZ PunctuationZForms)splitr)rCrDZkeywords_range_aZkeywords_range_belZrange_a_jp_charsZrange_b_jp_charsrrrrBsP    rBi)maxsize皙?F)decoded_sequencemaximum_thresholddebugrc CsddtD}t|d}d}|dkr0d}n|dkr>d}nd }x|t|d t|D]f\}}x |D]} | |rd| |qdW|d kr||d ks||dkrVtd d |D}||krVPqVW|rx|D]} t| j | j qWt |dS)zw Compute a mess ratio given a decoded bytes sequence. The maximum threshold does stop the computation earlier. cSsg|] }|qSrr).0Zmd_classrrr szmess_ratio..rgi iri rcss|] }|jVqdS)N)r!)rudtrrr &szmess_ratio..) r__subclasses__rZziprangerrsumprint __class__r!round) rrrsrtZ detectorslengthZmean_mess_ratioZ!intermediary_mean_mess_ratio_calcrindexZdetectorrzrrr mess_ratios*      rN)rqF)& functoolsrtypingrrZconstantrrutilsrr r r r r rrrrrrrrrrr*r3r6r;r?rEr\rbr&r'rBr)rrrrrs" D"/%6ZM DPK[]cC"C"__pycache__/cd.cpython-37.pycnu[B bD+@sddlZddlmZddlmZmZddlmZddlm Z m Z m Z m Z ddl mZddlmZmZmZmZdd lmZdd lmZdd lmZmZmZmZmZee ed d dZee edddZ eee ed ddZ!eee ed ddZ"eedee e#e#fdddZ$d,e ee#e edddZ%ee ee&ddd Z'ee ed!d"d#Z(e eed$d%d&Z)ed'dd-ee&e eed)d*d+Z*dS).N)IncrementalDecoder)Counter OrderedDict) lru_cache)DictListOptionalTuple) FREQUENCIES)KO_NAMESLANGUAGE_SUPPORTED_COUNTTOO_SMALL_SEQUENCEZH_NAMES) is_suspiciously_successive_range)CoherenceMatches)is_accentuatedis_latinis_multi_byte_encodingis_unicode_range_secondary unicode_range) iana_namereturncst|rtdtd|j}|dd}idxltddD]^}|t|g}|r@t |}|dkrjq@t |d kr@|krd|<|d 7<d 7q@Wt fd d DS) zF Return associated unicode ranges in a single byte code page. z.Function not supported on multi-byte code pagez encodings.{}ignore)errorsr@NFr cs g|]}|dkr|qS)g333333?).0character_range)character_count seen_rangesrF/opt/alt/python37/lib/python3.7/site-packages/charset_normalizer/cd.py 2sz*encoding_unicode_range..) rIOError importlib import_moduleformatrrangedecodebytesrrsorted)rdecoderpichunkrr)r r!r"encoding_unicode_ranges(    r0) primary_rangercCsDg}x:tD].\}}x$|D]}t||kr||PqWqW|S)z> Return inferred languages used with a unicode range. )r itemsrappend)r1 languageslanguage characters characterrrr"unicode_range_languages9s    r8cCs>t|}d}x|D]}d|kr|}PqW|dkr6dgSt|S)z Single-byte encoding language association. Some code page are heavily linked to particular language(s). This function does the correspondence. NZLatinz Latin Based)r0r8)rZunicode_rangesr1Zspecified_rangerrr"encoding_languagesHs r9cCsb|ds&|ds&|ds&|dkr,dgS|ds>|tkrFddgS|d sX|tkr^d gSgS) z Multi-byte encoding language association. Some code page are heavily linked to particular language(s). This function does the correspondence. Zshift_ iso2022_jpZeuc_jcp932ZJapaneseZgbZChinesezClassical Chinese iso2022_krZKorean) startswithrr )rrrr"mb_encoding_languages\s   r>)maxsize)r5rcCsFd}d}x4t|D](}|s&t|r&d}|rt|dkrd}qW||fS)zg Determine main aspects from a supported language if it contains accents and if is pure Latin. FT)r rr)r5target_have_accentstarget_pure_latinr7rrr"get_target_featuresqs rBF)r6ignore_non_latinrc sg}tddD}xxtD]l\}}t|\}}|rB|dkrBq |dkrP|rPq t|}tfdd|D} | |} | dkr ||| fq Wt|ddd d }d d|DS) zE Return associated languages associated to given characters. css|]}t|VqdS)N)r)rr7rrr" sz%alphabet_languages..Fcsg|]}|kr|qSrr)rc)r6rr"r#sz&alphabet_languages..g?cSs|dS)Nr r)xrrr"z$alphabet_languages..T)keyreversecSsg|] }|dqS)rr)rZcompatible_languagerrr"r#s)anyr r2rBlenr3r+) r6rCr4Zsource_have_accentsr5Zlanguage_charactersr@rAr Zcharacter_match_countratior)r6r"alphabet_languagess    rN)r5ordered_charactersrcs6|tkrtd|d}x |D]}|t|kr6q"t|dt||}t|t||d}|d|||||dfdd|Dd}fdd|Dd}t|dkr|dkr|d 7}q"t|dkr|dkr|d 7}q"|t|d ks|t|d kr"|d 7}q"q"W|t|S) aN Determine if a ordered characters list (by occurrence from most appearance to rarest) match a particular language. The result is a ratio between 0. (absolutely no correspondence) and 1. (near perfect fit). Beware that is function is not strict on the match in order to ease the detection. (Meaning close match is 1.) z{} not availablerNcsg|] }|kqSrr)re)characters_beforerr"r#sz1characters_popularity_compare..Tcsg|] }|kqSrr)rrP)characters_afterrr"r#sr g?)r ValueErrorr'indexcountrL)r5rOZcharacter_approved_countr7Zcharacters_before_sourceZcharacters_after_sourceZbefore_match_countZafter_match_countr)rRrQr"characters_popularity_compares: rW)decoded_sequencercCst}x|D]}|dkrq t|}|dkr0q d}x |D]}t||dkr:|}Pq:W|dkrb|}||krx|||<q |||7<q Wt|S)a Given a decoded text sequence, return a list of str. Unicode range / alphabet separation. Ex. a text containing English/Latin with a bit a Hebrew will return two items in the resulting list; One containing the latin letters and the other hebrew. FN)risalpharrlowerlistvalues)rXZlayersr7rZlayer_target_rangeZdiscovered_rangerrr"alpha_unicode_splits(    r])resultsrcsptxD|D]<}x6|D].}|\}}|kr6|g|<q||qWq WfddD}t|ddddS)z This function merge results previously given by the function coherence_ratio. The return type is the same as coherence_ratio. cs.g|]&}|tt|t|dfqS)rS)roundsumrL)rr5)per_language_ratiosrr"r#sz*merge_coherence_ratios..cSs|dS)Nr r)rFrrr"rG"rHz(merge_coherence_ratios..T)rIrJ)rr3r+)r^resultZ sub_resultr5rMmerger)rar"merge_coherence_ratios s    rdi皙?)rX threshold lg_inclusionrcCsg}d}d}|dk r|dng}d|kr8d}|dxt|D]}t|}|} tdd| D} | tkrrqBd d | D} xZ|pt| |D]H} t| | } | |krqn| d kr|d 7}| | t | d f|dkrPqWqBWt |ddddS)z Detect ANY language that can be identified in given sequence. The sequence will be analysed by layers. A layer = Character extraction by alphabets/ranges. FrN,z Latin BasedTcss|]\}}|VqdS)Nr)rrEorrr"rD<sz"coherence_ratio..cSsg|] \}}|qSrr)rrErirrr"r#Asz#coherence_ratio..g?r rScSs|dS)Nr r)rFrrr"rGTrHz!coherence_ratio..)rIrJ) splitremover]r most_commonr`rrNrWr3r_r+)rXrfrgr^rCZsufficient_match_countZlg_inclusion_listZlayerZsequence_frequenciesrmr Zpopular_character_orderedr5rMrrr"coherence_ratio%s4    rn)F)reN)+r%codecsr collectionsrr functoolsrtypingrrrr Zassetsr Zconstantr r rrZmdrZmodelsrutilsrrrrrstrr0r8r9r>boolrBrNfloatrWr]rdrnrrrr"s4      % #:' PK[]Bt355#__pycache__/constant.cpython-37.pycnu[B bK@sddlmZmZmZmZmZddlmZddlm Z ddl m Z m Z ddlmZmZmZmZddlmZedefd d d d d dgfddeegfdeegfgZdZedZdZededdeddeddeddeddeddeddeddeddeddeddedd ed d!ed!d"ed"d#ed#d$ed$d%ed%d&ed&d'ed'd(ed)d*ed*d+ed+d,ed,d-ed-d.ed.d/ed/d0ed0d1ed1d2ed2d3ed3d4ed4d5ed5d6ed6d7ed7d8ed8d9ed9d:ed:d;ed;d<eded>d?ed?d@ed@dAedAdBedBdCedCdDedDdEedEdFedFdGedGdHedHdIedIdJedJdKedKdLedLdMedMdNedNdOedOdPedPdQedQdRedRdSedSdTedUdVedVdWedWdXedXdYedYdZedZd[ed[d\ed\d]ed]d^ed^d_ed_d`ed`daedadbedbdcedcddedddeededfedfdgedgdhedhdiedidjedjdkedkdledldmedmdnedndoedodpedpdqedqdredrdsedsdtedtduedudvedvdwedwdxedxdyedydzedzd{ed{d|ed|d}ed}d~eddeddeddeddeddeddeddeddeddeddeddeddeddeddeddeddeddeddeddeddeddeddeddeddeddeddeddeddeddeddeddeddeddeddeddeddeddeddeddeddeddeddeddeddeddeddeddeddeddeddeddeddeddeddeddeddeddeddeddeddeddeddeddeddeddeddƒeddÃeddăeddŃeddƃeddǃeddȃeddʃedd˃edd̃edd̓edd΃eddσeddуeddӃeddԃeddՃeddփedd؃eddكeddڃedd܃edd݃eddރedd߃eddeddeddeddeddeddeddeddeddeddeddeddeddeddeddeddeddeddeddeddeddeddeddeddeddeddeddeddeddedded d ed d ed d eddeddeddeddeddeddeddeddeddeddedded d!ed!d"ed"d#ed$d%ed%d&ed'd(ed)d*ed*d+ed,d-ed-d.ed.d/ed0d1ed1d2ed3d4ed4d5ed6d7ed8d9ed:d;ed<d=ed>d?ed?d@ed@dAedAdBedBdCedCdDedDdEedEdFedFdGedGdHedHdIedIdJedJdKedLdMedNdOedOdPedPdQedQdRedSdTedUdVedWdXdYZdZd[d\d]d^d_d`dadbdcdddedfdgdhgZe die Zeedjdke e!e "Z#e$e#Z%dldmdndogdpdmdndogdqgdpdldndogdrgdsdtgdudvdwgdxgdudvdwgdygdpdldmdogdzd{d|d}d~ddgdpdldmdngddd{dgdzd{dgddzddgdd}d~ddgdd|d~ddgdd|d}ddgdd|d}d~dgddzdd{d|d}d~dgdgdduddvdwgdgdgddudddvdwgdddddddvdwgddudrddvdwgdddgdduddvdwgddrdvdwgdgdddddduddddwg ddtgdddddduddddvg ddgddgddgddsgdgd(Z&ddddddddddddddddddddddddddddddddd Z'dddddddddddddÐdĐdhZ(dƐdǐdhZ)dɐdʐdːdhZ*e d̓Z+e$eZ,dZ-dS()BOM_UTF8 BOM_UTF16_BE BOM_UTF16_LE BOM_UTF32_BE BOM_UTF32_LE) OrderedDict)aliases) IGNORECASEcompile)DictListSetUnion) FREQUENCIESutf_8utf_7s+/v8s+/v9s+/v+s+/v/s+/v8-)gb18030s13utf_32utf_16 gcAiiiPiiipiii0iiiiPiiii@i`ipii i i i i i i i i i iiiiiiiiiiiiii i@i`iiiiiPiiii iiiiiiPiiiiiiiiii ip i i i!iP!i!i"i#i$i@$i`$i%i%i%i&i'i'i'i(i)i)i*i+i,i`,i,i-i0-i-i-i.i.i/i/i/i0i@0i0i1i01i1i1i1i1i2i3i4iMiNiiiФii@iii ii0i@iiii0i`iiii`iiii0ipiiiiiiiiiiPiii i0iPipiiiiii@iiiiiiii0iPiiiiiPiiii0ipiiii@i`iiii i i@ i i i i` i i i i i@ i` i i i iP i i i`iiiiiiPiiiiPiiiiiiiiii`iiii@iiiiPiiiiipiii`i i$i$iP%i0i04iDiFihi@jipjijikikioioioipiiiii0ipiiiiiiiiPii`iiiiii0iiii`iiii0iiiiiiPiiiiiiiiii@i iiii iiii(zControl characterz Basic LatinzLatin-1 SupplementzLatin Extended-AzLatin Extended-BzIPA ExtensionszSpacing Modifier LetterszCombining Diacritical MarkszGreek and CopticZCyrilliczCyrillic SupplementZArmenianZHebrewZArabicZSyriaczArabic SupplementZThaanaZNKoZ SamaritanZMandaiczSyriac SupplementzArabic Extended-AZ DevanagariZBengaliZGurmukhiZGujaratiZOriyaZTamilZTeluguZKannadaZ MalayalamZSinhalaZThaiZLaoZTibetanZMyanmarZGeorgianz Hangul JamoZEthiopiczEthiopic SupplementZCherokeez%Unified Canadian Aboriginal SyllabicsZOghamZRunicZTagalogZHanunooZBuhidZTagbanwaZKhmerZ Mongolianz.Unified Canadian Aboriginal Syllabics ExtendedZLimbuzTai Lez New Tai Luez Khmer SymbolsZBuginesezTai Thamz$Combining Diacritical Marks ExtendedZBalineseZ SundaneseZBatakZLepchazOl ChikizCyrillic Extended CzSundanese SupplementzVedic ExtensionszPhonetic ExtensionszPhonetic Extensions Supplementz&Combining Diacritical Marks SupplementzLatin Extended AdditionalzGreek ExtendedzGeneral PunctuationzSuperscripts and SubscriptszCurrency Symbolsz'Combining Diacritical Marks for SymbolszLetterlike Symbolsz Number FormsZArrowszMathematical OperatorszMiscellaneous TechnicalzControl PictureszOptical Character RecognitionzEnclosed Alphanumericsz Box DrawingzBlock ElementszGeometric ShapeszMiscellaneous SymbolsZDingbatsz$Miscellaneous Mathematical Symbols-AzSupplemental Arrows-AzBraille PatternszSupplemental Arrows-Bz$Miscellaneous Mathematical Symbols-Bz#Supplemental Mathematical Operatorsz Miscellaneous Symbols and ArrowsZ GlagoliticzLatin Extended-CZCopticzGeorgian SupplementZTifinaghzEthiopic ExtendedzCyrillic Extended-AzSupplemental PunctuationzCJK Radicals SupplementzKangxi Radicalsz"Ideographic Description CharacterszCJK Symbols and PunctuationZHiraganaZKatakanaZBopomofozHangul Compatibility JamoZKanbunzBopomofo Extendedz CJK StrokeszKatakana Phonetic ExtensionszEnclosed CJK Letters and MonthszCJK Compatibilityz"CJK Unified Ideographs Extension AzYijing Hexagram SymbolszCJK Unified Ideographsz Yi Syllablesz Yi RadicalsZLisuZVaizCyrillic Extended-BZBamumzModifier Tone LetterszLatin Extended-Dz Syloti NagrizCommon Indic Number FormszPhags-paZ SaurashtrazDevanagari ExtendedzKayah LiZRejangzHangul Jamo Extended-AZJavanesezMyanmar Extended-BZChamzMyanmar Extended-AzTai VietzMeetei Mayek ExtensionszEthiopic Extended-AzLatin Extended-EzCherokee Supplementz Meetei MayekzHangul SyllableszHangul Jamo Extended-BzHigh SurrogateszHigh Private Use SurrogateszLow SurrogateszPrivate Use AreazCJK Compatibility IdeographszAlphabetic Presentation FormszArabic Presentation Forms-AzVariation SelectorszVertical FormszCombining Half MarkszCJK Compatibility FormszSmall Form VariantszArabic Presentation Forms-BzHalfwidth and Fullwidth FormsZSpecialszLinear B SyllabaryzLinear B IdeogramszAegean NumberszAncient Greek NumberszAncient Symbolsz Phaistos DiscZLycianZCarianzCoptic Epact Numbersz Old ItalicZGothicz Old PermicZUgariticz Old PersianZDeseretZShavianZOsmanyaZOsageZElbasanzCaucasian AlbanianzLinear AzCypriot SyllabaryzImperial AramaicZ PalmyreneZ NabataeanZHatranZ PhoenicianZLydianzMeroitic HieroglyphszMeroitic CursiveZ KharoshthizOld South ArabianzOld North ArabianZ ManichaeanZAvestanzInscriptional ParthianzInscriptional PahlavizPsalter Pahlaviz Old Turkicz Old HungarianzRumi Numeral SymbolsZBrahmiZKaithiz Sora SompengZChakmaZMahajaniZSharadazSinhala Archaic NumbersZKhojkiZMultaniZ KhudawadiZGranthaZNewaZTirhutaZSiddhamZModizMongolian SupplementZTakriZAhomz Warang CitizZanabazar SquareZSoyomboz Pau Cin HauZ BhaiksukiZMarchenz Masaram GondiZ Cuneiformz!Cuneiform Numbers and PunctuationzEarly Dynastic CuneiformzEgyptian HieroglyphszAnatolian HieroglyphszBamum SupplementZMroz Bassa Vahz Pahawh HmongZMiaoz#Ideographic Symbols and PunctuationZTangutzTangut ComponentszKana SupplementzKana Extended-AZNushuZDuployanzShorthand Format ControlszByzantine Musical SymbolszMusical SymbolszAncient Greek Musical NotationzTai Xuan Jing SymbolszCounting Rod Numeralsz!Mathematical Alphanumeric SymbolszSutton SignWritingzGlagolitic Supplementz Mende KikakuiZAdlamz&Arabic Mathematical Alphabetic Symbolsz Mahjong Tilesz Domino Tilesz Playing Cardsz Enclosed Alphanumeric SupplementzEnclosed Ideographic Supplementz%Miscellaneous Symbols and PictographszEmoticons range(Emoji)zOrnamental DingbatszTransport and Map SymbolszAlchemical SymbolszGeometric Shapes ExtendedzSupplemental Arrows-Cz$Supplemental Symbols and Pictographsz"CJK Unified Ideographs Extension Bz"CJK Unified Ideographs Extension Cz"CJK Unified Ideographs Extension Dz"CJK Unified Ideographs Extension Ez"CJK Unified Ideographs Extension Fz'CJK Compatibility Ideographs SupplementTagszVariation Selectors SupplementZ SupplementZExtendedZ ExtensionsZModifierZMarksZ PunctuationZSymbolsZFormsZ OperatorsZ MiscellaneousZDrawingZBlockZShapesZ Supplementalrz^(?:(?:encoding)|(?:charset)|(?:coding))(?:[\:= ]{1,10})(?:[\"\']?)([a-zA-Z0-9\-_]+)(?:[\"\']?)cCs|ddko|dkS)NZ_codecF>rot_13mbcstactis)endswith)xrL/opt/alt/python37/lib/python3.7/site-packages/charset_normalizer/constant.pyVsr!cp1026cp1140cp273cp500cp037cp866 iso8859_2kz1048ptcp154 iso8859_15 iso8859_9latin_1 iso8859_7 iso8859_13cp850cp858cp860cp861cp862cp863cp865cp437cp857cp1125 iso8859_14 iso8859_4tis_620cp1257 iso8859_10 iso8859_16 iso8859_3cp1252cp1254cp1250cp1253cp1258cp1251 mac_roman mac_turkish mac_iceland iso8859_11)(r&r"r9r#rCrFrArDrBr=r$r7r%r0r8r1r2r3r4r5r6r'r>rJr/r:r+r?r(r@r;r.r,r)r-rIrGrHr*r<z ISO-2022-KRz ISO-2022-JPzEUC-KRzTIS-620zUTF-32zEUC-JPzKOI8-Rz ISO-8859-1z ISO-8859-2z ISO-8859-5z ISO-8859-6z ISO-8859-7z ISO-8859-8zUTF-16ZIBM855Z MacCyrillicZGB2312ZGB18030ZCP932ZIBM866zutf-8z UTF-8-SIGZ SHIFT_JISZBig5z windows-1250z windows-1251z Windows-1252z windows-1253z windows-1255z windows-1256z Windows-1254ZCP949) iso2022_kr iso2022_jpeuc_krr<reuc_jpkoi8_r iso8859_1r( iso8859_5 iso8859_6r. iso8859_8rcp855 mac_cyrillicgb2312rcp932r'rZ utf_8_sig shift_jisbig5rCrFrArDcp1255cp1256rBcp949<>=:/&;{}[],|"-johabr\rMrYcp950 big5hkscshzz[0-9\W\n\r\t]+N).codecsrrrrr collectionsrZencodings.aliasesrrer r Z re_compiletypingr r r rZassetsrZENCODING_MARKSZTOO_SMALL_SEQUENCEintZTOO_BIG_SEQUENCEZUTF8_MAXIMAL_ALLOCATIONrangeZUNICODE_RANGES_COMBINEDZUNICODE_SECONDARY_RANGE_KEYWORDZRE_POSSIBLE_ENCODING_INDICATIONsortedfilterlistsetvaluesZIANA_SUPPORTEDlenZIANA_SUPPORTED_COUNTZIANA_SUPPORTED_SIMILARZCHARDET_CORRESPONDENCEZCOMMON_SAFE_ASCII_CHARACTERSZKO_NAMESZZH_NAMESZNOT_PRINTABLE_PATTERNZLANGUAGE_SUPPORTED_COUNTZTRACErrrr s                                                                 "       PK[]ׂ55)__pycache__/constant.cpython-37.opt-1.pycnu[B bK@sddlmZmZmZmZmZddlmZddlm Z ddl m Z m Z ddlmZmZmZmZddlmZedefd d d d d dgfddeegfdeegfgZdZedZdZededdeddeddeddeddeddeddeddeddeddeddedd ed d!ed!d"ed"d#ed#d$ed$d%ed%d&ed&d'ed'd(ed)d*ed*d+ed+d,ed,d-ed-d.ed.d/ed/d0ed0d1ed1d2ed2d3ed3d4ed4d5ed5d6ed6d7ed7d8ed8d9ed9d:ed:d;ed;d<eded>d?ed?d@ed@dAedAdBedBdCedCdDedDdEedEdFedFdGedGdHedHdIedIdJedJdKedKdLedLdMedMdNedNdOedOdPedPdQedQdRedRdSedSdTedUdVedVdWedWdXedXdYedYdZedZd[ed[d\ed\d]ed]d^ed^d_ed_d`ed`daedadbedbdcedcddedddeededfedfdgedgdhedhdiedidjedjdkedkdledldmedmdnedndoedodpedpdqedqdredrdsedsdtedtduedudvedvdwedwdxedxdyedydzedzd{ed{d|ed|d}ed}d~eddeddeddeddeddeddeddeddeddeddeddeddeddeddeddeddeddeddeddeddeddeddeddeddeddeddeddeddeddeddeddeddeddeddeddeddeddeddeddeddeddeddeddeddeddeddeddeddeddeddeddeddeddeddeddeddeddeddeddeddeddeddeddeddeddeddƒeddÃeddăeddŃeddƃeddǃeddȃeddʃedd˃edd̃edd̓edd΃eddσeddуeddӃeddԃeddՃeddփedd؃eddكeddڃedd܃edd݃eddރedd߃eddeddeddeddeddeddeddeddeddeddeddeddeddeddeddeddeddeddeddeddeddeddeddeddeddeddeddeddeddedded d ed d ed d eddeddeddeddeddeddeddeddeddeddedded d!ed!d"ed"d#ed$d%ed%d&ed'd(ed)d*ed*d+ed,d-ed-d.ed.d/ed0d1ed1d2ed3d4ed4d5ed6d7ed8d9ed:d;ed<d=ed>d?ed?d@ed@dAedAdBedBdCedCdDedDdEedEdFedFdGedGdHedHdIedIdJedJdKedLdMedNdOedOdPedPdQedQdRedSdTedUdVedWdXdYZdZd[d\d]d^d_d`dadbdcdddedfdgdhgZe die Zeedjdke e!e "Z#e$e#Z%dldmdndogdpdmdndogdqgdpdldndogdrgdsdtgdudvdwgdxgdudvdwgdygdpdldmdogdzd{d|d}d~ddgdpdldmdngddd{dgdzd{dgddzddgdd}d~ddgdd|d~ddgdd|d}ddgdd|d}d~dgddzdd{d|d}d~dgdgdduddvdwgdgdgddudddvdwgdddddddvdwgddudrddvdwgdddgdduddvdwgddrdvdwgdgdddddduddddwg ddtgdddddduddddvg ddgddgddgddsgdgd(Z&ddddddddddddddddddddddddddddddddd Z'dddddddddddddÐdĐdhZ(dƐdǐdhZ)dɐdʐdːdhZ*e d̓Z+e$eZ,dZ-dS()BOM_UTF8 BOM_UTF16_BE BOM_UTF16_LE BOM_UTF32_BE BOM_UTF32_LE) OrderedDict)aliases) IGNORECASEcompile)DictListSetUnion) FREQUENCIESutf_8utf_7s+/v8s+/v9s+/v+s+/v/s+/v8-)gb18030s13utf_32utf_16 gcAiiiPiiipiii0iiiiPiiii@i`ipii i i i i i i i i i iiiiiiiiiiiiii i@i`iiiiiPiiii iiiiiiPiiiiiiiiii ip i i i!iP!i!i"i#i$i@$i`$i%i%i%i&i'i'i'i(i)i)i*i+i,i`,i,i-i0-i-i-i.i.i/i/i/i0i@0i0i1i01i1i1i1i1i2i3i4iMiNiiiФii@iii ii0i@iiii0i`iiii`iiii0ipiiiiiiiiiiPiii i0iPipiiiiii@iiiiiiii0iPiiiiiPiiii0ipiiii@i`iiii i i@ i i i i` i i i i i@ i` i i i iP i i i`iiiiiiPiiiiPiiiiiiiiii`iiii@iiiiPiiiiipiii`i i$i$iP%i0i04iDiFihi@jipjijikikioioioipiiiii0ipiiiiiiiiPii`iiiiii0iiii`iiii0iiiiiiPiiiiiiiiii@i iiii iiii(zControl characterz Basic LatinzLatin-1 SupplementzLatin Extended-AzLatin Extended-BzIPA ExtensionszSpacing Modifier LetterszCombining Diacritical MarkszGreek and CopticZCyrilliczCyrillic SupplementZArmenianZHebrewZArabicZSyriaczArabic SupplementZThaanaZNKoZ SamaritanZMandaiczSyriac SupplementzArabic Extended-AZ DevanagariZBengaliZGurmukhiZGujaratiZOriyaZTamilZTeluguZKannadaZ MalayalamZSinhalaZThaiZLaoZTibetanZMyanmarZGeorgianz Hangul JamoZEthiopiczEthiopic SupplementZCherokeez%Unified Canadian Aboriginal SyllabicsZOghamZRunicZTagalogZHanunooZBuhidZTagbanwaZKhmerZ Mongolianz.Unified Canadian Aboriginal Syllabics ExtendedZLimbuzTai Lez New Tai Luez Khmer SymbolsZBuginesezTai Thamz$Combining Diacritical Marks ExtendedZBalineseZ SundaneseZBatakZLepchazOl ChikizCyrillic Extended CzSundanese SupplementzVedic ExtensionszPhonetic ExtensionszPhonetic Extensions Supplementz&Combining Diacritical Marks SupplementzLatin Extended AdditionalzGreek ExtendedzGeneral PunctuationzSuperscripts and SubscriptszCurrency Symbolsz'Combining Diacritical Marks for SymbolszLetterlike Symbolsz Number FormsZArrowszMathematical OperatorszMiscellaneous TechnicalzControl PictureszOptical Character RecognitionzEnclosed Alphanumericsz Box DrawingzBlock ElementszGeometric ShapeszMiscellaneous SymbolsZDingbatsz$Miscellaneous Mathematical Symbols-AzSupplemental Arrows-AzBraille PatternszSupplemental Arrows-Bz$Miscellaneous Mathematical Symbols-Bz#Supplemental Mathematical Operatorsz Miscellaneous Symbols and ArrowsZ GlagoliticzLatin Extended-CZCopticzGeorgian SupplementZTifinaghzEthiopic ExtendedzCyrillic Extended-AzSupplemental PunctuationzCJK Radicals SupplementzKangxi Radicalsz"Ideographic Description CharacterszCJK Symbols and PunctuationZHiraganaZKatakanaZBopomofozHangul Compatibility JamoZKanbunzBopomofo Extendedz CJK StrokeszKatakana Phonetic ExtensionszEnclosed CJK Letters and MonthszCJK Compatibilityz"CJK Unified Ideographs Extension AzYijing Hexagram SymbolszCJK Unified Ideographsz Yi Syllablesz Yi RadicalsZLisuZVaizCyrillic Extended-BZBamumzModifier Tone LetterszLatin Extended-Dz Syloti NagrizCommon Indic Number FormszPhags-paZ SaurashtrazDevanagari ExtendedzKayah LiZRejangzHangul Jamo Extended-AZJavanesezMyanmar Extended-BZChamzMyanmar Extended-AzTai VietzMeetei Mayek ExtensionszEthiopic Extended-AzLatin Extended-EzCherokee Supplementz Meetei MayekzHangul SyllableszHangul Jamo Extended-BzHigh SurrogateszHigh Private Use SurrogateszLow SurrogateszPrivate Use AreazCJK Compatibility IdeographszAlphabetic Presentation FormszArabic Presentation Forms-AzVariation SelectorszVertical FormszCombining Half MarkszCJK Compatibility FormszSmall Form VariantszArabic Presentation Forms-BzHalfwidth and Fullwidth FormsZSpecialszLinear B SyllabaryzLinear B IdeogramszAegean NumberszAncient Greek NumberszAncient Symbolsz Phaistos DiscZLycianZCarianzCoptic Epact Numbersz Old ItalicZGothicz Old PermicZUgariticz Old PersianZDeseretZShavianZOsmanyaZOsageZElbasanzCaucasian AlbanianzLinear AzCypriot SyllabaryzImperial AramaicZ PalmyreneZ NabataeanZHatranZ PhoenicianZLydianzMeroitic HieroglyphszMeroitic CursiveZ KharoshthizOld South ArabianzOld North ArabianZ ManichaeanZAvestanzInscriptional ParthianzInscriptional PahlavizPsalter Pahlaviz Old Turkicz Old HungarianzRumi Numeral SymbolsZBrahmiZKaithiz Sora SompengZChakmaZMahajaniZSharadazSinhala Archaic NumbersZKhojkiZMultaniZ KhudawadiZGranthaZNewaZTirhutaZSiddhamZModizMongolian SupplementZTakriZAhomz Warang CitizZanabazar SquareZSoyomboz Pau Cin HauZ BhaiksukiZMarchenz Masaram GondiZ Cuneiformz!Cuneiform Numbers and PunctuationzEarly Dynastic CuneiformzEgyptian HieroglyphszAnatolian HieroglyphszBamum SupplementZMroz Bassa Vahz Pahawh HmongZMiaoz#Ideographic Symbols and PunctuationZTangutzTangut ComponentszKana SupplementzKana Extended-AZNushuZDuployanzShorthand Format ControlszByzantine Musical SymbolszMusical SymbolszAncient Greek Musical NotationzTai Xuan Jing SymbolszCounting Rod Numeralsz!Mathematical Alphanumeric SymbolszSutton SignWritingzGlagolitic Supplementz Mende KikakuiZAdlamz&Arabic Mathematical Alphabetic Symbolsz Mahjong Tilesz Domino Tilesz Playing Cardsz Enclosed Alphanumeric SupplementzEnclosed Ideographic Supplementz%Miscellaneous Symbols and PictographszEmoticons range(Emoji)zOrnamental DingbatszTransport and Map SymbolszAlchemical SymbolszGeometric Shapes ExtendedzSupplemental Arrows-Cz$Supplemental Symbols and Pictographsz"CJK Unified Ideographs Extension Bz"CJK Unified Ideographs Extension Cz"CJK Unified Ideographs Extension Dz"CJK Unified Ideographs Extension Ez"CJK Unified Ideographs Extension Fz'CJK Compatibility Ideographs SupplementTagszVariation Selectors SupplementZ SupplementZExtendedZ ExtensionsZModifierZMarksZ PunctuationZSymbolsZFormsZ OperatorsZ MiscellaneousZDrawingZBlockZShapesZ Supplementalrz^(?:(?:encoding)|(?:charset)|(?:coding))(?:[\:= ]{1,10})(?:[\"\']?)([a-zA-Z0-9\-_]+)(?:[\"\']?)cCs|ddko|dkS)NZ_codecF>mbcsrot_13tactis)endswith)xrL/opt/alt/python37/lib/python3.7/site-packages/charset_normalizer/constant.pyVsr!cp1026cp1140cp273cp500cp037cp866 iso8859_2kz1048ptcp154 iso8859_15 iso8859_9latin_1 iso8859_7 iso8859_13cp850cp858cp860cp861cp862cp863cp865cp437cp857cp1125 iso8859_14 iso8859_4tis_620cp1257 iso8859_10 iso8859_16 iso8859_3cp1252cp1254cp1250cp1253cp1258cp1251 mac_roman mac_turkish mac_iceland iso8859_11)(r&r"r9r#rCrFrArDrBr=r$r7r%r0r8r1r2r3r4r5r6r'r>rJr/r:r+r?r(r@r;r.r,r)r-rIrGrHr*r<z ISO-2022-KRz ISO-2022-JPzEUC-KRzTIS-620zUTF-32zEUC-JPzKOI8-Rz ISO-8859-1z ISO-8859-2z ISO-8859-5z ISO-8859-6z ISO-8859-7z ISO-8859-8zUTF-16ZIBM855Z MacCyrillicZGB2312ZGB18030ZCP932ZIBM866zutf-8z UTF-8-SIGZ SHIFT_JISZBig5z windows-1250z windows-1251z Windows-1252z windows-1253z windows-1255z windows-1256z Windows-1254ZCP949) iso2022_kr iso2022_jpeuc_krr<reuc_jpkoi8_r iso8859_1r( iso8859_5 iso8859_6r. iso8859_8rcp855 mac_cyrillicgb2312rcp932r'rZ utf_8_sig shift_jisbig5rCrFrArDcp1255cp1256rBcp949<>=:/&;{}[],|"-johabr\rMrYcp950 big5hkscshzz[0-9\W\n\r\t]+N).codecsrrrrr collectionsrZencodings.aliasesrrer r Z re_compiletypingr r r rZassetsrZENCODING_MARKSZTOO_SMALL_SEQUENCEintZTOO_BIG_SEQUENCEZUTF8_MAXIMAL_ALLOCATIONrangeZUNICODE_RANGES_COMBINEDZUNICODE_SECONDARY_RANGE_KEYWORDZRE_POSSIBLE_ENCODING_INDICATIONsortedfilterlistsetvaluesZIANA_SUPPORTEDlenZIANA_SUPPORTED_COUNTZIANA_SUPPORTED_SIMILARZCHARDET_CORRESPONDENCEZCOMMON_SAFE_ASCII_CHARACTERSZKO_NAMESZZH_NAMESZNOT_PRINTABLE_PATTERNZLANGUAGE_SUPPORTED_COUNTZTRACErrrr s                                                                 "       PK[]KlNo**__pycache__/api.cpython-37.pycnu[B bOO @sddlZddlmZmZddlmZmZmZmZyddl m Z Wne k rXe Z YnXddl mZmZmZmZddlmZmZmZmZddlmZdd lmZmZdd lmZmZmZm Z m!Z!m"Z"e#d Z$e%Z&e&'e(d de)e*e*e+ee ee e,e,ed ddZ-dee*e*e+ee ee e,e,ed ddZ.d e e*e*e+ee ee e,e,ed ddZ/d!e e*e*e+ee ee e,edddZ0dS)"N)basenamesplitext)BinaryIOListOptionalSet)PathLike)coherence_ratioencoding_languagesmb_encoding_languagesmerge_coherence_ratios)IANA_SUPPORTEDTOO_BIG_SEQUENCETOO_SMALL_SEQUENCETRACE) mess_ratio) CharsetMatchCharsetMatches)any_specified_encoding iana_nameidentify_sig_or_bom is_cp_similaris_multi_byte_encodingshould_strip_sig_or_bomcharset_normalizerz)%(asctime)s | %(levelname)s | %(message)s皙?TF) sequencessteps chunk_size threshold cp_isolation cp_exclusionpreemptive_behaviourexplainreturnc1 Cs t|ttfs tdt||r>tj}tt t t t |} | dkrt d|rvtt t |prtjtt|dddgdgS|dk rtt d d |d d |D}ng}|dk rtt d d |dd |D}ng}| ||krtt d||| d}| }|dkr:| ||kr:t| |}t |tk} t |tk} | rltt d| n| rtt d| g} |rt|nd} | dk r| | tt d| t}g}g}d}d}d}t}t|\}}|dk r| |tt dt ||| dd| kr.| dxn| tD]`}|rT||krTq:|rh||krhq:||krvq:||d}||k}|ot|}|dkr|stt d|q:y t|}Wn,t t!fk rtt d|w:YnXyr| r@|dkr@t"|dkr$|dtdn|t |td|dn&t"|dkrP|n|t |d|d}WnVt#t$fk r}z2t|t$stt d|t"|||w:Wdd}~XYnXd}x |D]}t%||rd}PqW|rtt d||q:t&|s dnt || t| |}|o<|dk o||tt d+||"t,|+d,d-d.|dd| gkr:|#s:t|||dg|},|| kr&|,}n|dkr6|,}n|,}q:tt d/|t,|+d,d-d.|sjt-|}-nt.|}-|-rtt d0|t"|-g}.|dkrx4|$D],}(t/|(d1|-rd2|-nd}/|.|/qWt0|.}0|0rtt d3|0||t|||+||0||| ddgkr\|+d1kr\t d4||rNtt t |t||gS||kr:t d5||rtt t |t||gSq:Wt |dk rP|s|s|rtt d6|rt d7|j1||nd|r|dk s | r| r|j2|j2k s |dk r6t d8||n| rPt d9||| rtt d:|3j1t |dn t d;| rtt t ||S)]szfrom_bytes..zacp_exclusion is set. use this flag for debugging purpose. limited list of encoding excluded : %s.cSsg|]}t|dqS)F)r)r*r+r,r,r-r.hsz^override steps (%i) and chunk_size (%i) as content does not fit (%i byte(s) given) parameters.r z>Trying to detect encoding from a tiny portion of ({}) byte(s).zIUsing lazy str decoding because the payload is quite large, ({}) byte(s).z@Detected declarative mark in sequence. Priority +1 given for %s.zIDetected a SIG or BOM mark on first %i byte(s). Priority +1 given for %s.ascii>utf_32utf_16z[Encoding %s wont be tested as-is because it require a BOM. Will try some sub-encoder LE/BE.z2Encoding %s does not provide an IncrementalDecodergA)encodingz9Code page %s does not fit given bytes sequence at ALL. %sTzW%s is deemed too similar to code page %s and was consider unsuited already. Continuing!zpCode page %s is a multi byte encoding table and it appear that at least one character was encoded using n-bytes.ignorestrict)errorszaLazyStr Loading: After MD chunk decode, code page %s does not fit given bytes sequence at ALL. %sgj@z^LazyStr Loading: After final lookup, code page %s does not fit given bytes sequence at ALL. %szc%s was excluded because of initial chaos probing. Gave up %i time(s). Computed mean chaos is %f %%.d)ndigitsz=%s passed initial chaos probing. Mean measured chaos is %f %%z&{} should target any language(s) of {}g?,z We detected language {} using {}z.Encoding detection: %s is most likely the one.zoEncoding detection: %s is most likely the one as we detected a BOM or SIG within the beginning of the sequence.zONothing got out of the detection process. Using ASCII/UTF-8/Specified fallback.z7Encoding detection: %s will be used as a fallback matchz:Encoding detection: utf_8 will be used as a fallback matchz:Encoding detection: ascii will be used as a fallback matchz]Encoding detection: Found %s as plausible (best-candidate) for content. With %i alternatives.z=Encoding detection: Unable to determine any suitable charset.)4 isinstance bytearraybytes TypeErrorformattypeloggerlevel addHandlerexplain_handlersetLevelrlendebug removeHandlerloggingWARNINGrrlogjoinintrrrappendsetrraddrrModuleNotFoundError ImportErrorstrUnicodeDecodeError LookupErrorrrangemaxdecodeminrsumroundr r r r r2Z fingerprintbest)1rr r!r"r#r$r%r&Zprevious_logger_levellengthZis_too_small_sequenceZis_too_large_sequenceZprioritized_encodingsZspecified_encodingZtestedZtested_but_hard_failureZtested_but_soft_failureZfallback_asciiZ fallback_u8Zfallback_specifiedresultsZ sig_encodingZ sig_payloadZ encoding_ianaZdecoded_payloadZbom_or_sig_availableZstrip_sig_or_bomZis_multi_byte_decodereZsimilar_soft_failure_testZencoding_soft_failedZr_Zmulti_byte_bonusZmax_chunk_gave_upZearly_stop_countZlazy_str_hard_failureZ md_chunksZ md_ratiosiZ cut_sequencechunkZchunk_partial_size_chkjZmean_mess_ratioZfallback_entryZtarget_languagesZ cd_ratiosZchunk_languagesZcd_ratios_mergedr,r,r- from_bytes%sZ                                                             rh) fpr r!r"r#r$r%r&r'c Cst||||||||S)z Same thing than the function from_bytes but using a file pointer that is already ready. Will not close the file pointer. )rhread)rir r!r"r#r$r%r&r,r,r-from_fpsrk) pathr r!r"r#r$r%r&r'c Cs,t|d}t||||||||SQRXdS)z Same thing than the function from_bytes but with one extra step. Opening and reading given file path in binary mode. Can raise IOError. rbN)openrk) rlr r!r"r#r$r%r&rir,r,r- from_paths ro)rlr r!r"r#r$r%r'c Cst|||||||}t|}tt|} t|dkrBtd||} | dd| j7<t dt | |d | d} | | WdQRX| S)zi Take a (text-based) file path and try to create another file next to it, this time using UTF-8. rz;Unable to normalize "{}", no encoding charset seems to fit.-z{}r)wbN)rorlistrrKIOErrorrDrar2rnrXreplacerQwriteoutput) rlr r!r"r#r$r%rcfilenameZtarget_extensionsresultrir,r,r- normalize7s*    ry)rrrNNTF)rrrNNTF)rrrNNTF)rrrNNT)1rNos.pathrrtypingrrrrosrrWrXZcdr r r r ZconstantrrrrZmdrZmodelsrrutilsrrrrrr getLoggerrF StreamHandlerrI setFormatter FormatterrBrRfloatboolrhrkroryr,r,r,r-sb    YPK[]\::&__pycache__/utils.cpython-37.opt-1.pycnu[B b\$@s,y ddlZWnek r(ddlZYnXddlZddlZddlmZddlmZddl m Z ddl m Z ddl mZmZmZmZmZddlmZdd lmZmZmZmZmZmZe ed eed d d Ze ed eed ddZe ed eeed ddZ e ed eed ddZ!eed ddZ"e ed eed ddZ#e ed eed ddZ$e ed eed ddZ%e ed eed ddZ&e ed eed ddZ'eed d d!Z(e ed eed d"d#Z)e ed eed d$d%Z*e ed eed d&d'Z+e ed eed d(d)Z,e ed eed d*d+Z-e e.ed eed,d-d.Z/dNe0e1eed0d1d2Z2e d3d eed4d5d6Z3e0eeee0fd7d8d9Z4eed:d;d<Z5dOeeed>d?d@Z6eeedAdBdCZ7eee8dDdEdFZ9eeedDdGdHZ:dIej;dJfee1eddKdLdMZ+;<>｜,TZ)isspacerr-)rr.rrr is_separator|s r=cCs||kS)N)islowerisupper)rrrris_case_variablesr@cCst|}|dkS)NZCo)rr-)rr.rrris_private_use_onlys rAcCs,yt|}Wntk r"dSXd|kS)NFZCJK)rrr)rcharacter_namerrris_cjks rCcCs,yt|}Wntk r"dSXd|kS)NFZHIRAGANA)rrr)rrBrrr is_hiraganas rDcCs,yt|}Wntk r"dSXd|kS)NFZKATAKANA)rrr)rrBrrr is_katakanas rEcCs,yt|}Wntk r"dSXd|kS)NFZHANGUL)rrr)rrBrrr is_hanguls rFcCs,yt|}Wntk r"dSXd|kS)NFZTHAI)rrr)rrBrrris_thais rG)r%rcstfddtDS)Nc3s|]}|kVqdS)Nr).0keyword)r%rr sz-is_unicode_range_secondary..)anyr)r%r)r%ris_unicode_range_secondarysrL)sequence search_zonercCst|tstt|}tt|dt||jddd}t|dkrHdSxJ|D]B}| dd}x,t D] \}}||kr|S||krl|SqlWqNWdS)zW Extract using ASCII-only decoder any specified encoding in the first n-bytes. Nr(ignore)errorsr-_) isinstancebytes TypeErrorlenrrmindecodelowerreplacerr$)rNrOZseq_lenZresultsZspecified_encodingencoding_alias encoding_ianarrrany_specified_encodings     r^)rrcCs |dkpttd|jtS)zQ Verify is a specific encoding is a multi byte one based on it IANA name > utf_16_be utf_8_sigutf_32utf_16 utf_16_le utf_32_beutf_7 utf_32_leutf_8z encodings.{}) issubclass importlib import_moduleformatrr )rrrris_multi_byte_encodings rm)rNrcCsJxDtD]<}t|}t|tr"|g}x|D]}||r(||fSq(WqWdS)z9 Identify and extract SIG/BOM in given sequence. )N)r rTrU startswith)rN iana_encodingZmarksZmarkrrridentify_sig_or_boms    rq)rprcCs|dkS)N>rbrcr)rprrrshould_strip_sig_or_bom srrT)cp_namestrictrcCsL|dd}x$tD]\}}|||gkr|SqW|rHtd||S)NrRrSz Unable to retrieve IANA for '{}')rZr[rr$rrl)rsrtr\r]rrr iana_names ru)decoded_sequencercCs8t}x(|D] }t|}|dkr"q ||q Wt|S)N)setr&addlist)rvZrangesrr/rrr range_scans rz) iana_name_a iana_name_brc Cst|st|rdStd|j}td|j}|dd}|dd}d}x6tdD]*}t|g}||||krZ|d7}qZW|dS) Ngz encodings.{}rP)rQrr )rmrjrkrlrrangerUrY) r{r|Z decoder_aZ decoder_bZid_aZid_bZcharacter_match_countiZ to_be_decodedrrr cp_similarity+s    rcCs|tko|t|kS)z Determine if two code page are at least 80% similar. IANA_SUPPORTED_SIMILAR dict was generated using the function cp_similarity. )r)r{r|rrr is_cp_similar@srZcharset_normalizerz)%(asctime)s | %(levelname)s | %(message)s)rlevel format_stringrcCs:t|}||t}|t|||dS)N)loggingZ getLoggerZsetLevelZ StreamHandlerZ setFormatter FormatterZ addHandler)rrrZloggerhandlerrrrset_logging_handlerKs   r)rM)T)=Z unicodedata2r ImportErrorrjrcodecsrZencodings.aliasesr functoolsrrertypingrrrr r Z_multibytecodecr Zconstantr rrrrrstrboolrr"r&r'r+r0r3r4r=r@rArCrDrErFrGrWrLrUr!r^rmrqrrrurzfloatrrINFOrrrrrsr                   PK[]`C"C"#__pycache__/cd.cpython-37.opt-1.pycnu[B bD+@sddlZddlmZddlmZmZddlmZddlm Z m Z m Z m Z ddl mZddlmZmZmZmZdd lmZdd lmZdd lmZmZmZmZmZee ed d dZee edddZ eee ed ddZ!eee ed ddZ"eedee e#e#fdddZ$d,e ee#e edddZ%ee ee&ddd Z'ee ed!d"d#Z(e eed$d%d&Z)ed'dd-ee&e eed)d*d+Z*dS).N)IncrementalDecoder)Counter OrderedDict) lru_cache)DictListOptionalTuple) FREQUENCIES)KO_NAMESLANGUAGE_SUPPORTED_COUNTTOO_SMALL_SEQUENCEZH_NAMES) is_suspiciously_successive_range)CoherenceMatches)is_accentuatedis_latinis_multi_byte_encodingis_unicode_range_secondary unicode_range) iana_namereturncst|rtdtd|j}|dd}idxltddD]^}|t|g}|r@t |}|dkrjq@t |d kr@|krd|<|d 7<d 7q@Wt fd d DS) zF Return associated unicode ranges in a single byte code page. z.Function not supported on multi-byte code pagez encodings.{}ignore)errorsr@NFr cs g|]}|dkr|qS)g333333?).0character_range)character_count seen_rangesrF/opt/alt/python37/lib/python3.7/site-packages/charset_normalizer/cd.py 2sz*encoding_unicode_range..) rIOError importlib import_moduleformatrrangedecodebytesrrsorted)rdecoderpichunkrr)r r!r"encoding_unicode_ranges(    r0) primary_rangercCsDg}x:tD].\}}x$|D]}t||kr||PqWqW|S)z> Return inferred languages used with a unicode range. )r itemsrappend)r1 languageslanguage characters characterrrr"unicode_range_languages9s    r8cCs>t|}d}x|D]}d|kr|}PqW|dkr6dgSt|S)z Single-byte encoding language association. Some code page are heavily linked to particular language(s). This function does the correspondence. NZLatinz Latin Based)r0r8)rZunicode_rangesr1Zspecified_rangerrr"encoding_languagesHs r9cCsb|ds&|ds&|ds&|dkr,dgS|ds>|tkrFddgS|d sX|tkr^d gSgS) z Multi-byte encoding language association. Some code page are heavily linked to particular language(s). This function does the correspondence. Zshift_ iso2022_jpZeuc_jcp932ZJapaneseZgbZChinesezClassical Chinese iso2022_krZKorean) startswithrr )rrrr"mb_encoding_languages\s   r>)maxsize)r5rcCsFd}d}x4t|D](}|s&t|r&d}|rt|dkrd}qW||fS)zg Determine main aspects from a supported language if it contains accents and if is pure Latin. FT)r rr)r5target_have_accentstarget_pure_latinr7rrr"get_target_featuresqs rBF)r6ignore_non_latinrc sg}tddD}xxtD]l\}}t|\}}|rB|dkrBq |dkrP|rPq t|}tfdd|D} | |} | dkr ||| fq Wt|ddd d }d d|DS) zE Return associated languages associated to given characters. css|]}t|VqdS)N)r)rr7rrr" sz%alphabet_languages..Fcsg|]}|kr|qSrr)rc)r6rr"r#sz&alphabet_languages..g?cSs|dS)Nr r)xrrr"z$alphabet_languages..T)keyreversecSsg|] }|dqS)rr)rZcompatible_languagerrr"r#s)anyr r2rBlenr3r+) r6rCr4Zsource_have_accentsr5Zlanguage_charactersr@rAr Zcharacter_match_countratior)r6r"alphabet_languagess    rN)r5ordered_charactersrcs6|tkrtd|d}x |D]}|t|kr6q"t|dt||}t|t||d}|d|||||dfdd|Dd}fdd|Dd}t|dkr|dkr|d 7}q"t|dkr|dkr|d 7}q"|t|d ks|t|d kr"|d 7}q"q"W|t|S) aN Determine if a ordered characters list (by occurrence from most appearance to rarest) match a particular language. The result is a ratio between 0. (absolutely no correspondence) and 1. (near perfect fit). Beware that is function is not strict on the match in order to ease the detection. (Meaning close match is 1.) z{} not availablerNcsg|] }|kqSrr)re)characters_beforerr"r#sz1characters_popularity_compare..Tcsg|] }|kqSrr)rrP)characters_afterrr"r#sr g?)r ValueErrorr'indexcountrL)r5rOZcharacter_approved_countr7Zcharacters_before_sourceZcharacters_after_sourceZbefore_match_countZafter_match_countr)rRrQr"characters_popularity_compares: rW)decoded_sequencercCst}x|D]}|dkrq t|}|dkr0q d}x |D]}t||dkr:|}Pq:W|dkrb|}||krx|||<q |||7<q Wt|S)a Given a decoded text sequence, return a list of str. Unicode range / alphabet separation. Ex. a text containing English/Latin with a bit a Hebrew will return two items in the resulting list; One containing the latin letters and the other hebrew. FN)risalpharrlowerlistvalues)rXZlayersr7rZlayer_target_rangeZdiscovered_rangerrr"alpha_unicode_splits(    r])resultsrcsptxD|D]<}x6|D].}|\}}|kr6|g|<q||qWq WfddD}t|ddddS)z This function merge results previously given by the function coherence_ratio. The return type is the same as coherence_ratio. cs.g|]&}|tt|t|dfqS)rS)roundsumrL)rr5)per_language_ratiosrr"r#sz*merge_coherence_ratios..cSs|dS)Nr r)rFrrr"rG"rHz(merge_coherence_ratios..T)rIrJ)rr3r+)r^resultZ sub_resultr5rMmerger)rar"merge_coherence_ratios s    rdi皙?)rX threshold lg_inclusionrcCsg}d}d}|dk r|dng}d|kr8d}|dxt|D]}t|}|} tdd| D} | tkrrqBd d | D} xZ|pt| |D]H} t| | } | |krqn| d kr|d 7}| | t | d f|dkrPqWqBWt |ddddS)z Detect ANY language that can be identified in given sequence. The sequence will be analysed by layers. A layer = Character extraction by alphabets/ranges. FrN,z Latin BasedTcss|]\}}|VqdS)Nr)rrEorrr"rD<sz"coherence_ratio..cSsg|] \}}|qSrr)rrErirrr"r#Asz#coherence_ratio..g?r rScSs|dS)Nr r)rFrrr"rGTrHz!coherence_ratio..)rIrJ) splitremover]r most_commonr`rrNrWr3r_r+)rXrfrgr^rCZsufficient_match_countZlg_inclusion_listZlayerZsequence_frequenciesrmr Zpopular_character_orderedr5rMrrr"coherence_ratio%s4    rn)F)reN)+r%codecsr collectionsrr functoolsrtypingrrrr Zassetsr Zconstantr r rrZmdrZmodelsrZutilsrrrrrstrr0r8r9r>boolrBrNfloatrWr]rdrnrrrr"s4      % #:' PK[]xV '__pycache__/legacy.cpython-37.opt-1.pycnu[B b8 @sddlZddlmZmZmZddlmZmZmZm Z ddl m Z ddl m Z mZeeeeeeeffddd ZGd d d e ZGd d d eZGdddeZGdddeZdS)N)DictOptionalUnion) from_bytesfrom_fp from_path normalize)CHARDET_CORRESPONDENCE) CharsetMatchCharsetMatches)byte_strreturncCst|ttfs tdt|t|tr2t|}t|}|dk rL|jnd}|dk rh|j dkrh|j nd}|dk r~d|j nd}|dk r|dkr|j r|d7}|t kr|nt |||dS) a chardet legacy method Detect the encoding of the given byte string. It should be mostly backward-compatible. Encoding name will match Chardet own writing whenever possible. (Not on encoding name unsupported by it) This function is deprecated and should be used to migrate your project easily, consult the documentation for further information. Not planned for removal. :param byte_str: The byte sequence to examine. z4Expected object of type bytes or bytearray, got: {0}NZUnknowng?utf_8Z_sig)encodinglanguage confidence) isinstance bytearraybytes TypeErrorformattyperZbestrrZchaosZbomr )r rrrrrJ/opt/alt/python37/lib/python3.7/site-packages/charset_normalizer/legacy.pydetect s     rc@s eZdZdS)CharsetNormalizerMatchN)__name__ __module__ __qualname__rrrrr0src@s<eZdZeddZeddZeddZeddZd S) CharsetNormalizerMatchescOstdtt||S)Nzkstaticmethod from_fp, from_bytes, from_path and normalize are deprecated and scheduled to be removed in 3.0)warningswarnDeprecationWarningr)argskwargsrrrr5sz CharsetNormalizerMatches.from_fpcOstdtt||S)Nzkstaticmethod from_fp, from_bytes, from_path and normalize are deprecated and scheduled to be removed in 3.0)r#r$r%r)r&r'rrrr>sz#CharsetNormalizerMatches.from_bytescOstdtt||S)Nzkstaticmethod from_fp, from_bytes, from_path and normalize are deprecated and scheduled to be removed in 3.0)r#r$r%r)r&r'rrrrGsz"CharsetNormalizerMatches.from_pathcOstdtt||S)Nzkstaticmethod from_fp, from_bytes, from_path and normalize are deprecated and scheduled to be removed in 3.0)r#r$r%r )r&r'rrrr Psz"CharsetNormalizerMatches.normalizeN)rr r! staticmethodrrrr rrrrr"4s r"c@s eZdZdS)CharsetDetectorN)rr r!rrrrr)Zsr)c@s eZdZdS) CharsetDoctorN)rr r!rrrrr*^sr*)r#typingrrrZapirrrr Zconstantr Zmodelsr r rstrfloatrrr"r)r*rrrrs $'&PK[].33'__pycache__/models.cpython-37.opt-1.pycnu[B b3@sddlZddlmZddlmZddlmZddlmZddl m Z ddl m Z m Z mZmZmZmZmZdd lmZmZdd lmZdd lmZmZmZGd d d ZGdddZeeefZ ee Z!GdddZ"dS)N)Counter)aliases)sha256)dumps)sub)AnyDictIteratorListOptionalTupleUnion)NOT_PRINTABLE_PATTERNTOO_BIG_SEQUENCE) mess_ratio) iana_nameis_multi_byte_encoding unicode_rangec@s eZdZdBeeeedeedddZe edddZ e edd d Z e ed d d Z e ed ddZe ed ddZe ed ddZed ddZed ddZdddddZe ed ddZe eed ddZe ed ddZe ed d d!Ze eed d"d#Ze ed d$d%Ze ed d&d'Ze ed d(d)Ze ed d*d+Ze ed d,d-Ze ed d.d/Z e edd d0d1Z!e ed d2d3Z"e eed d4d5Z#e eed d6d7Z$dd d8d9Z%dd d:d;Z&dCeed=d>d?Z'e ed d@dAZ(dS)D CharsetMatchNCoherenceMatches)payloadguessed_encodingmean_mess_ratiohas_sig_or_bom languagesdecoded_payloadcCsF||_||_||_||_||_d|_g|_d|_d|_d|_ ||_ dS)Ng) _payload _encoding_mean_mess_ratio _languages_has_sig_or_bom_unicode_ranges_leavesZ_mean_coherence_ratio_output_payload_output_encoding_string)selfrrrrrrr(J/opt/alt/python37/lib/python3.7/site-packages/charset_normalizer/models.py__init__s zCharsetMatch.__init__)otherreturncCs>t|ts&tdt|jt|j|j|jko<|j|jkS)Nz&__eq__ cannot be invoked on {} and {}.) isinstancer TypeErrorformatstr __class__encoding fingerprint)r'r+r(r(r)__eq__(s  zCharsetMatch.__eq__cCsvt|tstt|j|j}t|j|j}|dkrj|dkrj|dkr^|j|jkr^|j|jkS|j|jkS|j|jkS)zQ Implemented to make sorted available upon CharsetMatches items. g{Gz?g{Gz?g)r-r ValueErrorabschaos coherencemulti_byte_usage)r'r+Zchaos_differenceZcoherence_differencer(r(r)__lt__1s   zCharsetMatch.__lt__)r,cCsdtt|t|jS)Ng?)lenr0raw)r'r(r(r)r9DszCharsetMatch.multi_byte_usagecCstdttt|dS)z Check once again chaos in decoded text, except this time, with full content. Use with caution, this can be very slow. Notice: Will be removed in 3.0 z=chaos_secondary_pass is deprecated and will be removed in 3.0g?)warningswarnDeprecationWarningrr0)r'r(r(r)chaos_secondary_passHsz!CharsetMatch.chaos_secondary_passcCstdtdS)zy Coherence ratio on the first non-latin language detected if ANY. Notice: Will be removed in 3.0 zr?)r'r(r(r)coherence_non_latinUsz CharsetMatch.coherence_non_latincCs,tdtttdt|}t|S)z_ Word counter instance on decoded text. Notice: Will be removed in 3.0 z2w_counter is deprecated and will be removed in 3.0 ) r=r>r?rrr0lowerrsplit)r'Zstring_printable_onlyr(r(r) w_counteraszCharsetMatch.w_countercCs"|jdkrt|j|jd|_|jS)Nstrict)r&r0rr)r'r(r(r)__str__os zCharsetMatch.__str__cCsd|j|jS)Nz)r/r2r3)r'r(r(r)__repr__uszCharsetMatch.__repr__cCs8t|tr||kr"td|jd|_|j|dS)Nz;Unable to add instance <{}> as a submatch of a CharsetMatch)r-rr5r/r1r&r#append)r'r+r(r(r) add_submatchxs  zCharsetMatch.add_submatchcCs|jS)N)r)r'r(r(r)r2szCharsetMatch.encodingcCsHg}x>tD]2\}}|j|kr,||q|j|kr||qW|S)z Encoding name are known by many name, using this could help when searching for IBM855 when it's listed as CP855. )ritemsr2rI)r'Z also_known_asupr(r(r)encoding_aliasess   zCharsetMatch.encoding_aliasescCs|jS)N)r!)r'r(r(r)bomszCharsetMatch.bomcCs|jS)N)r!)r'r(r(r)byte_order_markszCharsetMatch.byte_order_markcCsdd|jDS)z Return the complete list of possible languages found in decoded sequence. Usually not really useful. Returned list may be empty even if 'language' property return something != 'Unknown'. cSsg|] }|dqS)rr().0er(r(r) sz*CharsetMatch.languages..)r )r'r(r(r)rszCharsetMatch.languagescCsp|jsbd|jkrdSddlm}m}t|jr8||jn||j}t|dksVd|krZdS|dS|jddS)z Most probable language found in decoded sequence. If none were detected or inferred, the property will return "Unknown". asciiZEnglishr)encoding_languagesmb_encoding_languagesz Latin BasedZUnknown)r could_be_from_charsetZcharset_normalizer.cdrUrVrr2r;)r'rUrVrr(r(r)languages  zCharsetMatch.languagecCs|jS)N)r)r'r(r(r)r7szCharsetMatch.chaoscCs|js dS|jddS)Ngrr)r )r'r(r(r)r8szCharsetMatch.coherencecCst|jdddS)Nd)ndigits)roundr7)r'r(r(r) percent_chaosszCharsetMatch.percent_chaoscCst|jdddS)NrYrZ)r[)r\r8)r'r(r(r)percent_coherenceszCharsetMatch.percent_coherencecCs|jS)z+ Original untouched bytes. )r)r'r(r(r)r<szCharsetMatch.rawcCs|jS)N)r#)r'r(r(r)submatchszCharsetMatch.submatchcCst|jdkS)Nr)r;r#)r'r(r(r) has_submatchszCharsetMatch.has_submatchcCs@|jdk r|jSddt|D}ttdd|D|_|jS)NcSsg|] }t|qSr()r)rQcharr(r(r)rSsz*CharsetMatch.alphabets..cSsh|] }|r|qSr(r()rQrr(r(r) sz)CharsetMatch.alphabets..)r"r0sortedlist)r'Zdetected_rangesr(r(r) alphabetss  zCharsetMatch.alphabetscCs|jgdd|jDS)z The complete list of encoding that output the exact SAME str result and therefore could be the originating encoding. This list does include the encoding available in property 'encoding'. cSsg|] }|jqSr()r2)rQmr(r(r)rSsz6CharsetMatch.could_be_from_charset..)rr#)r'r(r(r)rWsz"CharsetMatch.could_be_from_charsetcCs|S)z> Kept for BC reasons. Will be removed in 3.0. r()r'r(r(r)firstszCharsetMatch.firstcCs|S)z> Kept for BC reasons. Will be removed in 3.0. r()r'r(r(r)bestszCharsetMatch.bestutf_8)r2r,cCs2|jdks|j|kr,||_t||d|_|jS)z Method to get re-encoded bytes payload using given target encoding. Default to UTF-8. Any errors will be simply ignored by the encoder NOT replaced. Nreplace)r%r0encoder$)r'r2r(r(r)outputszCharsetMatch.outputcCst|S)zw Retrieve the unique SHA256 computed using the transformed (re-encoded) payload. Not the original one. )rrmZ hexdigest)r'r(r(r)r3 szCharsetMatch.fingerprint)N)rj))__name__ __module__ __qualname__bytesr0floatboolr r*objectr4r:propertyr9r@rArrErGrHrJr2r rNrOrPrrXr7r8r]r^r<r_r`rfrWrhrirmr3r(r(r(r)rsb        rc@seZdZdZdeedddZeedddZe e e fed d d Z e dd d Z edddZedd ddZeddddZeddddZdS)CharsetMatchesz Container with every CharsetMatch items ordered by default from most probable to the less one. Act like a list(iterable) but does not implements all related methods. N)resultscCs|r t|ng|_dS)N)rd_results)r'rwr(r(r)r*szCharsetMatches.__init__)r,ccs|jEdHdS)N)rx)r'r(r(r)__iter__szCharsetMatches.__iter__)itemr,cCsNt|tr|j|St|trFt|d}x|jD]}||jkr0|Sq0WtdS)z Retrieve a single item either by its position or encoding name (alias may be used here). Raise KeyError upon invalid index or encoding not present in results. FN)r-intrxr0rrWKeyError)r'rzresultr(r(r) __getitem__!s      zCharsetMatches.__getitem__cCs t|jS)N)r;rx)r'r(r(r)__len__/szCharsetMatches.__len__cCst|jdkS)Nr)r;rx)r'r(r(r)__bool__2szCharsetMatches.__bool__cCs~t|tstdt|jt|jtkrbx4|j D]*}|j |j kr4|j |j kr4| |dSq4W|j |t|j |_ dS)z~ Insert a single match. Will be inserted accordingly to preserve sort. Can be inserted as a submatch. z-Cannot append instance '{}' to CharsetMatchesN)r-rr5r/r0r1r;r<rrxr3r7rJrIrd)r'rzmatchr(r(r)rI5s    zCharsetMatches.appendrcCs|js dS|jdS)zQ Simply return the first match. Strict equivalent to matches[0]. Nr)rx)r'r(r(r)riIszCharsetMatches.bestcCs|S)zP Redundant method, call the method best(). Kept for BC reasons. )ri)r'r(r(r)rhQszCharsetMatches.first)N)rnrorp__doc__r rr*r ryr r{r0r~rrsrrIr rirhr(r(r(r)rvsrvc @sjeZdZeeeeeeeeeeeeeeeed ddZe e ee fdddZ edddZ d S) CliDetectionResult) pathr2rNalternative_encodingsrXrfrr7r8 unicode_path is_preferredc CsF||_| |_||_||_||_||_||_||_||_| |_ | |_ dS)N) rrr2rNrrXrfrr7r8r) r'rr2rNrrXrfrr7r8rrr(r(r)r*]szCliDetectionResult.__init__)r,c Cs2|j|j|j|j|j|j|j|j|j|j |j d S)N) rr2rNrrXrfrr7r8rr) rr2rNrrXrfrr7r8rr)r'r(r(r)__dict__wszCliDetectionResult.__dict__cCst|jdddS)NT)Z ensure_asciiindent)rr)r'r(r(r)to_jsonszCliDetectionResult.to_jsonN)rnrorpr0r r rsrrr*rurrrrr(r(r(r)r\sr)#r= collectionsrZencodings.aliasesrZhashlibrZjsonrrertypingrrr r r r r ZconstantrrZmdrZutilsrrrrrvr0rrZCoherenceMatchrrr(r(r(r)s      $  C PK[]U*)::#__pycache__/md.cpython-37.opt-1.pycnu[B bG@s<ddlmZddlmZmZddlmZmZddlm Z m Z m Z m Z m Z mZmZmZmZmZmZmZmZmZmZGdddZGdd d eZGd d d eZGd d d eZGdddeZGdddeZGdddeZGdddeZGdddeZ ee!ee!e"dddZ#eddd#e!e$e"e$dd d!Z%d"S)$) lru_cache)ListOptional)COMMON_SAFE_ASCII_CHARACTERSUNICODE_SECONDARY_RANGE_KEYWORD)is_accentuatedis_asciiis_case_variableis_cjk is_emoticon is_hangul is_hiragana is_katakanais_latinis_punctuation is_separator is_symbolis_thai remove_accent unicode_rangec@sPeZdZdZeedddZeddddZddd d Ze e dd d Z dS) MessDetectorPluginzy Base abstract class used for mess detection plugins. All detectors MUST extend and implement given methods. ) characterreturncCstdS)z@ Determine if given character should be fed in. N)NotImplementedError)selfrrF/opt/alt/python37/lib/python3.7/site-packages/charset_normalizer/md.pyeligibleszMessDetectorPlugin.eligibleNcCstdS)z The main routine to be executed upon character. Insert the logic in witch the text would be considered chaotic. N)r)rrrrrfeed$szMessDetectorPlugin.feed)rcCstdS)zB Permit to reset the plugin to the initial state. N)r)rrrrreset+szMessDetectorPlugin.resetcCstdS)z Compute the chaos ratio based on what your feed() has seen. Must NOT be lower than 0.; No restriction gt 0. N)r)rrrrratio1szMessDetectorPlugin.ratio) __name__ __module__ __qualname____doc__strboolrrr propertyfloatr!rrrrrs rc@sZeZdZddddZeedddZedddd Zddd d Ze e dd d Z dS) TooManySymbolOrPunctuationPluginN)rcCs"d|_d|_d|_d|_d|_dS)NrF)_punctuation_count _symbol_count_character_count_last_printable_charZ_frenzy_symbol_in_word)rrrr__init__;s z)TooManySymbolOrPunctuationPlugin.__init__)rrcCs|S)N) isprintable)rrrrrrCsz)TooManySymbolOrPunctuationPlugin.eligiblecCsp|jd7_||jkrf|tkrft|r8|jd7_n.|dkrft|rft|dkrf|jd7_||_dS)NrF) r-r.rrr+isdigitrr r,)rrrrrrFs   z%TooManySymbolOrPunctuationPlugin.feedcCsd|_d|_d|_dS)Nr)r+r-r,)rrrrr Xsz&TooManySymbolOrPunctuationPlugin.resetcCs0|jdkrdS|j|j|j}|dkr,|SdS)Nrgg333333?)r-r+r,)rZratio_of_punctuationrrrr!]s   z&TooManySymbolOrPunctuationPlugin.ratio) r"r#r$r/r&r'rrr r(r)r!rrrrr*:s r*c@sZeZdZddddZeedddZedddd Zddd d Ze e dd d Z dS)TooManyAccentuatedPluginN)rcCsd|_d|_dS)Nr)r-_accentuated_count)rrrrr/jsz!TooManyAccentuatedPlugin.__init__)rrcCs|S)N)isalpha)rrrrrrnsz!TooManyAccentuatedPlugin.eligiblecCs(|jd7_t|r$|jd7_dS)Nr)r-rr4)rrrrrrqszTooManyAccentuatedPlugin.feedcCsd|_d|_dS)Nr)r-r4)rrrrr wszTooManyAccentuatedPlugin.resetcCs*|jdkrdS|j|j}|dkr&|SdS)Nrggffffff?)r-r4)rZratio_of_accentuationrrrr!{s  zTooManyAccentuatedPlugin.ratio) r"r#r$r/r&r'rrr r(r)r!rrrrr3is r3c@sZeZdZddddZeedddZedddd Zddd d Ze e dd d Z dS)UnprintablePluginN)rcCsd|_d|_dS)Nr)_unprintable_countr-)rrrrr/szUnprintablePlugin.__init__)rrcCsdS)NTr)rrrrrrszUnprintablePlugin.eligiblecCs@|dkr.|dkr.|dkr.|jd7_|jd7_dS)NFr)isspacer0r7r-)rrrrrrs   zUnprintablePlugin.feedcCs d|_dS)Nr)r7)rrrrr szUnprintablePlugin.resetcCs|jdkrdS|jd|jS)Nrg)r-r7)rrrrr!s zUnprintablePlugin.ratio) r"r#r$r/r&r'rrr r(r)r!rrrrr6s  r6c@sZeZdZddddZeedddZedddd Zddd d Ze e dd d Z dS)SuspiciousDuplicateAccentPluginN)rcCsd|_d|_d|_dS)Nr)_successive_countr-_last_latin_character)rrrrr/sz(SuspiciousDuplicateAccentPlugin.__init__)rrcCs|ot|S)N)r5r)rrrrrrsz(SuspiciousDuplicateAccentPlugin.eligiblecCst|jd7_|jdk rjt|rjt|jrj|rJ|jrJ|jd7_t|t|jkrj|jd7_||_dS)Nr)r-r=risupperr<r)rrrrrrs  z$SuspiciousDuplicateAccentPlugin.feedcCsd|_d|_d|_dS)Nr)r<r-r=)rrrrr sz%SuspiciousDuplicateAccentPlugin.resetcCs|jdkrdS|jd|jS)Nrgr1)r-r<)rrrrr!s z%SuspiciousDuplicateAccentPlugin.ratio) r"r#r$r/r&r'rrr r(r)r!rrrrr;s r;c@sZeZdZddddZeedddZedddd Zddd d Ze e dd d Z dS)SuspiciousRangeN)rcCsd|_d|_d|_dS)Nr)"_suspicious_successive_range_countr-_last_printable_seen)rrrrr/szSuspiciousRange.__init__)rrcCs|S)N)r0)rrrrrrszSuspiciousRange.eligiblecCsx|jd7_|s&t|s&|tkr0d|_dS|jdkrD||_dSt|j}t|}t||rn|jd7_||_dS)Nr)r-r9rrrAr is_suspiciously_successive_ranger@)rrunicode_range_aunicode_range_brrrrs  zSuspiciousRange.feedcCsd|_d|_d|_dS)Nr)r-r@rA)rrrrr szSuspiciousRange.resetcCs.|jdkrdS|jd|j}|dkr*dS|S)Nrgr1g?)r-r@)rZratio_of_suspicious_range_usagerrrr!s zSuspiciousRange.ratio) r"r#r$r/r&r'rrr r(r)r!rrrrr?s r?c@sZeZdZddddZeedddZedddd Zddd d Ze e dd d Z dS)SuperWeirdWordPluginN)rcCs:d|_d|_d|_d|_d|_d|_d|_d|_d|_dS)NrF) _word_count_bad_word_count_foreign_long_count_is_current_word_bad_foreign_long_watchr-_bad_character_count_buffer_buffer_accent_count)rrrrr/szSuperWeirdWordPlugin.__init__)rrcCsdS)NTr)rrrrrr szSuperWeirdWordPlugin.eligiblecCs|rd|j|g|_t|r0|jd7_|jdkrt|dksNt|rt|dkrt|dkrt |dkrt |dkrt |dkrd|_dS|jsdS| st |st|r|jr|jd7_t|j}|j|7_|dkr:|j|dkrd|_t|jdr:|jdr:|jd7_d|_|dkr`|jr`|jd7_d|_|jr|jd7_|jt|j7_d|_d|_d|_d |_n6|d kr|dkrt|rd|_|j|7_dS) NrFrFTg(\?r>_<>~=-|)r5joinrMrrNrKrr r rrrr9rrrGlenr-rJr>rIrHrLr2r)rrZ buffer_lengthrrrrsR           zSuperWeirdWordPlugin.feedcCs4d|_d|_d|_d|_d|_d|_d|_d|_dS)NrFFr)rMrJrKrHrGr-rLrI)rrrrr DszSuperWeirdWordPlugin.resetcCs$|jdkr|jdkrdS|j|jS)N rg)rGrIrLr-)rrrrr!NszSuperWeirdWordPlugin.ratio) r"r#r$r/r&r'rrr r(r)r!rrrrrEs 6 rEc@s^eZdZdZddddZeedddZeddd d Zddd d Z e e dd dZ dS)CjkInvalidStopPluginu GB(Chinese) based encoding often render the stop incorrectly when the content does not fit and can be easily detected. Searching for the overuse of '丅' and '丄'. N)rcCsd|_d|_dS)Nr)_wrong_stop_count_cjk_character_count)rrrrr/\szCjkInvalidStopPlugin.__init__)rrcCsdS)NTr)rrrrrr`szCjkInvalidStopPlugin.eligiblecCs4|dkr|jd7_dSt|r0|jd7_dS)N>丅丄r)r]r r^)rrrrrrcs zCjkInvalidStopPlugin.feedcCsd|_d|_dS)Nr)r]r^)rrrrr jszCjkInvalidStopPlugin.resetcCs|jdkrdS|j|jS)Ng)r^r])rrrrr!ns zCjkInvalidStopPlugin.ratio) r"r#r$r%r/r&r'rrr r(r)r!rrrrr\Vsr\c@sZeZdZddddZeedddZedddd Zddd d Ze e dd d Z dS)ArchaicUpperLowerPluginN)rcCs.d|_d|_d|_d|_d|_d|_d|_dS)NFrT)_buf_character_count_since_last_sep_successive_upper_lower_count#_successive_upper_lower_count_finalr-_last_alpha_seen_current_ascii_only)rrrrr/vsz ArchaicUpperLowerPlugin.__init__)rrcCsdS)NTr)rrrrrrsz ArchaicUpperLowerPlugin.eligiblecCs$|ot|}|dk}|r|jdkr|jdkrV|dkrV|jdkrV|j|j7_d|_d|_d|_d|_|j d7_ d|_dS|jdkrt |dkrd|_|jdk r| r|j s| r|j r|jdkr|jd7_d|_qd|_nd|_|j d7_ |jd7_||_dS)NFr@rTr1) r5r rdr2rhrfrergrcr-r r>islower)rrZ is_concernedZ chunk_seprrrrs8      zArchaicUpperLowerPlugin.feedcCs.d|_d|_d|_d|_d|_d|_d|_dS)NrFT)r-rdrerfrgrcrh)rrrrr szArchaicUpperLowerPlugin.resetcCs|jdkrdS|j|jS)Nrg)r-rf)rrrrr!s zArchaicUpperLowerPlugin.ratio) r"r#r$r/r&r'rrr r(r)r!rrrrrbus  * rb)rCrDrcCs~|dks|dkrdS||kr dSd|kr4d|kr4dSd|ksDd|krHdSd|ksXd|krld|kshd|krldS|d|d}}x"|D]}|tkrq||krdSqW|dk|dk}}|s|rd |ksd |krdS|r|rdSd |ksd |kr"d |ksd |kr dS|d ks|d kr"dSd |ksJd |ksJ|d krz|d krzd |ks^d |krbdSd|ksvd|krzdSdS)za Determine if two Unicode range seen next to each other can be considered as suspicious. NTFZLatinZ EmoticonsZ Combining )HiraganaKatakanaZCJKZHangulz Basic Latin)rmrlZ PunctuationZForms)splitr)rCrDZkeywords_range_aZkeywords_range_bZelZrange_a_jp_charsZrange_b_jp_charsrrrrBsP    rBi)maxsize皙?F)decoded_sequencemaximum_thresholddebugrc CsddtD}t|d}d}|dkr0d}n|dkr>d}nd }x|t|d t|D]f\}}x |D]} | |rd| |qdW|d kr||d ks||dkrVtd d |D}||krVPqVW|rx|D]} t| j | j qWt |dS)zw Compute a mess ratio given a decoded bytes sequence. The maximum threshold does stop the computation earlier. cSsg|] }|qSrr).0Zmd_classrrr szmess_ratio..rgi iri rcss|] }|jVqdS)N)r!)rtdtrrr &szmess_ratio..) r__subclasses__rZziprangerrsumprint __class__r!round) rqrrrsZ detectorslengthZmean_mess_ratioZ!intermediary_mean_mess_ratio_calcrindexZdetectorryrrr mess_ratios*      rN)rpF)& functoolsrtypingrrZconstantrrZutilsrr r r r r rrrrrrrrrrr*r3r6r;r?rEr\rbr&r'rBr)rrrrrs" D"/%6ZM DPK[]ɗ"__pycache__/version.cpython-37.pycnu[B bP@sdZdZedZdS)z Expose version z2.0.12.N)__doc__ __version__splitVERSIONrrK/opt/alt/python37/lib/python3.7/site-packages/charset_normalizer/version.pysPK[]ɪ**$__pycache__/api.cpython-37.opt-1.pycnu[B bOO @sddlZddlmZmZddlmZmZmZmZyddl m Z Wne k rXe Z YnXddl mZmZmZmZddlmZmZmZmZddlmZdd lmZmZdd lmZmZmZm Z m!Z!m"Z"e#d Z$e%Z&e&'e(d de)e*e*e+ee ee e,e,ed ddZ-dee*e*e+ee ee e,e,ed ddZ.d e e*e*e+ee ee e,e,ed ddZ/d!e e*e*e+ee ee e,edddZ0dS)"N)basenamesplitext)BinaryIOListOptionalSet)PathLike)coherence_ratioencoding_languagesmb_encoding_languagesmerge_coherence_ratios)IANA_SUPPORTEDTOO_BIG_SEQUENCETOO_SMALL_SEQUENCETRACE) mess_ratio) CharsetMatchCharsetMatches)any_specified_encoding iana_nameidentify_sig_or_bom is_cp_similaris_multi_byte_encodingshould_strip_sig_or_bomZcharset_normalizerz)%(asctime)s | %(levelname)s | %(message)s皙?TF) sequencessteps chunk_size threshold cp_isolation cp_exclusionpreemptive_behaviourexplainreturnc1 Cs t|ttfs tdt||r>tj}tt t t t |} | dkrt d|rvtt t |prtjtt|dddgdgS|dk rtt d d |d d |D}ng}|dk rtt d d |dd |D}ng}| ||krtt d||| d}| }|dkr:| ||kr:t| |}t |tk} t |tk} | rltt d| n| rtt d| g} |rt|nd} | dk r| | tt d| t}g}g}d}d}d}t}t|\}}|dk r| |tt dt ||| dd| kr.| dxn| tD]`}|rT||krTq:|rh||krhq:||krvq:||d}||k}|ot|}|dkr|stt d|q:y t|}Wn,t t!fk rtt d|w:YnXyr| r@|dkr@t"|dkr$|dtdn|t |td|dn&t"|dkrP|n|t |d|d}WnVt#t$fk r}z2t|t$stt d|t"|||w:Wdd}~XYnXd}x |D]}t%||rd}PqW|rtt d||q:t&|s dnt || t| |}|o<|dk o||tt d+||"t,|+d,d-d.|dd| gkr:|#s:t|||dg|},|| kr&|,}n|dkr6|,}n|,}q:tt d/|t,|+d,d-d.|sjt-|}-nt.|}-|-rtt d0|t"|-g}.|dkrx4|$D],}(t/|(d1|-rd2|-nd}/|.|/qWt0|.}0|0rtt d3|0||t|||+||0||| ddgkr\|+d1kr\t d4||rNtt t |t||gS||kr:t d5||rtt t |t||gSq:Wt |dk rP|s|s|rtt d6|rt d7|j1||nd|r|dk s | r| r|j2|j2k s |dk r6t d8||n| rPt d9||| rtt d:|3j1t |dn t d;| rtt t ||S)]szfrom_bytes..zacp_exclusion is set. use this flag for debugging purpose. limited list of encoding excluded : %s.cSsg|]}t|dqS)F)r)r)r*r+r+r,r-hsz^override steps (%i) and chunk_size (%i) as content does not fit (%i byte(s) given) parameters.r z>Trying to detect encoding from a tiny portion of ({}) byte(s).zIUsing lazy str decoding because the payload is quite large, ({}) byte(s).z@Detected declarative mark in sequence. Priority +1 given for %s.zIDetected a SIG or BOM mark on first %i byte(s). Priority +1 given for %s.ascii>utf_32utf_16z[Encoding %s wont be tested as-is because it require a BOM. Will try some sub-encoder LE/BE.z2Encoding %s does not provide an IncrementalDecodergA)encodingz9Code page %s does not fit given bytes sequence at ALL. %sTzW%s is deemed too similar to code page %s and was consider unsuited already. Continuing!zpCode page %s is a multi byte encoding table and it appear that at least one character was encoded using n-bytes.ignorestrict)errorszaLazyStr Loading: After MD chunk decode, code page %s does not fit given bytes sequence at ALL. %sgj@z^LazyStr Loading: After final lookup, code page %s does not fit given bytes sequence at ALL. %szc%s was excluded because of initial chaos probing. Gave up %i time(s). Computed mean chaos is %f %%.d)Zndigitsz=%s passed initial chaos probing. Mean measured chaos is %f %%z&{} should target any language(s) of {}g?,z We detected language {} using {}z.Encoding detection: %s is most likely the one.zoEncoding detection: %s is most likely the one as we detected a BOM or SIG within the beginning of the sequence.zONothing got out of the detection process. Using ASCII/UTF-8/Specified fallback.z7Encoding detection: %s will be used as a fallback matchz:Encoding detection: utf_8 will be used as a fallback matchz:Encoding detection: ascii will be used as a fallback matchz]Encoding detection: Found %s as plausible (best-candidate) for content. With %i alternatives.z=Encoding detection: Unable to determine any suitable charset.)4 isinstance bytearraybytes TypeErrorformattypeloggerlevelZ addHandlerexplain_handlerZsetLevelrlendebugZ removeHandlerloggingZWARNINGrrlogjoinintrrrappendsetrraddrrModuleNotFoundError ImportErrorstrUnicodeDecodeError LookupErrorrrangemaxdecodeminrsumroundr r r r r1Z fingerprintbest)1rrr r!r"r#r$r%Zprevious_logger_levellengthZis_too_small_sequenceZis_too_large_sequenceZprioritized_encodingsZspecified_encodingZtestedZtested_but_hard_failureZtested_but_soft_failureZfallback_asciiZ fallback_u8Zfallback_specifiedresultsZ sig_encodingZ sig_payloadZ encoding_ianaZdecoded_payloadZbom_or_sig_availableZstrip_sig_or_bomZis_multi_byte_decodereZsimilar_soft_failure_testZencoding_soft_failedZr_Zmulti_byte_bonusZmax_chunk_gave_upZearly_stop_countZlazy_str_hard_failureZ md_chunksZ md_ratiosiZ cut_sequencechunkZchunk_partial_size_chkjZmean_mess_ratioZfallback_entryZtarget_languagesZ cd_ratiosZchunk_languagesZcd_ratios_mergedr+r+r, from_bytes%sZ                                                             rb) fprr r!r"r#r$r%r&c Cst||||||||S)z Same thing than the function from_bytes but using a file pointer that is already ready. Will not close the file pointer. )rbread)rcrr r!r"r#r$r%r+r+r,from_fpsre) pathrr r!r"r#r$r%r&c Cs,t|d}t||||||||SQRXdS)z Same thing than the function from_bytes but with one extra step. Opening and reading given file path in binary mode. Can raise IOError. rbN)openre) rfrr r!r"r#r$r%rcr+r+r, from_paths ri)rfrr r!r"r#r$r&c Cst|||||||}t|}tt|} t|dkrBtd||} | dd| j7<t dt | |d | d} | | WdQRX| S)zi Take a (text-based) file path and try to create another file next to it, this time using UTF-8. rz;Unable to normalize "{}", no encoding charset seems to fit.-z{}r(wbN)rirlistrrGIOErrorrBr[r1rhrRreplacerKwriteoutput) rfrr r!r"r#r$r]filenameZtarget_extensionsresultrcr+r+r, normalize7s*    rs)rrrNNTF)rrrNNTF)rrrNNTF)rrrNNT)1rIZos.pathrrtypingrrrrosrrQrRZcdr r r r ZconstantrrrrZmdrZmodelsrrZutilsrrrrrrZ getLoggerrDZ StreamHandlerrFZ setFormatter Formatterr@rLfloatboolrbrerirsr+r+r+r,sb    YPK[]9n)__pycache__/__init__.cpython-37.opt-1.pycnu[B b@sdZddlZddlmZmZmZmZddlmZm Z m Z m Z m Z ddl mZmZddlmZddlmZmZd Zed edS) u Charset-Normalizer ~~~~~~~~~~~~~~ The Real First Universal Charset Detector. A library that helps you read text from an unknown charset encoding. Motivated by chardet, This package is trying to resolve the issue by taking a new approach. All IANA character set names for which the Python core library provides codecs are supported. Basic usage: >>> from charset_normalizer import from_bytes >>> results = from_bytes('Bсеки човек има право на образование. Oбразованието!'.encode('utf_8')) >>> best_guess = results.best() >>> str(best_guess) 'Bсеки човек има право на образование. Oбразованието!' Others methods and usages are available - see the full documentation at . :copyright: (c) 2021 by Ahmed TAHRI :license: MIT, see LICENSE for more details. N) from_bytesfrom_fp from_path normalize)CharsetDetector CharsetDoctorCharsetNormalizerMatchCharsetNormalizerMatchesdetect) CharsetMatchCharsetMatches)set_logging_handler)VERSION __version__)rrrrr r r r r rrrrrZcharset_normalizer)__doc__ZloggingZapirrrrZlegacyrrr r r Zmodelsr r Zutilsrversionrr__all__Z getLoggerZ addHandlerZ NullHandlerrrL/opt/alt/python37/lib/python3.7/site-packages/charset_normalizer/__init__.pys PK[]Z233!__pycache__/models.cpython-37.pycnu[B b3@sddlZddlmZddlmZddlmZddlmZddl m Z ddl m Z m Z mZmZmZmZmZdd lmZmZdd lmZdd lmZmZmZGd d d ZGdddZeeefZ ee Z!GdddZ"dS)N)Counter)aliases)sha256)dumps)sub)AnyDictIteratorListOptionalTupleUnion)NOT_PRINTABLE_PATTERNTOO_BIG_SEQUENCE) mess_ratio) iana_nameis_multi_byte_encoding unicode_rangec@s eZdZdBeeeedeedddZe edddZ e edd d Z e ed d d Z e ed ddZe ed ddZe ed ddZed ddZed ddZdddddZe ed ddZe eed ddZe ed ddZe ed d d!Ze eed d"d#Ze ed d$d%Ze ed d&d'Ze ed d(d)Ze ed d*d+Ze ed d,d-Ze ed d.d/Z e edd d0d1Z!e ed d2d3Z"e eed d4d5Z#e eed d6d7Z$dd d8d9Z%dd d:d;Z&dCeed=d>d?Z'e ed d@dAZ(dS)D CharsetMatchNCoherenceMatches)payloadguessed_encodingmean_mess_ratiohas_sig_or_bom languagesdecoded_payloadcCsF||_||_||_||_||_d|_g|_d|_d|_d|_ ||_ dS)Ng) _payload _encoding_mean_mess_ratio _languages_has_sig_or_bom_unicode_ranges_leavesZ_mean_coherence_ratio_output_payload_output_encoding_string)selfrrrrrrr(J/opt/alt/python37/lib/python3.7/site-packages/charset_normalizer/models.py__init__s zCharsetMatch.__init__)otherreturncCs>t|ts&tdt|jt|j|j|jko<|j|jkS)Nz&__eq__ cannot be invoked on {} and {}.) isinstancer TypeErrorformatstr __class__encoding fingerprint)r'r+r(r(r)__eq__(s  zCharsetMatch.__eq__cCsvt|tstt|j|j}t|j|j}|dkrj|dkrj|dkr^|j|jkr^|j|jkS|j|jkS|j|jkS)zQ Implemented to make sorted available upon CharsetMatches items. g{Gz?g{Gz?g)r-r ValueErrorabschaos coherencemulti_byte_usage)r'r+Zchaos_differenceZcoherence_differencer(r(r)__lt__1s   zCharsetMatch.__lt__)r,cCsdtt|t|jS)Ng?)lenr0raw)r'r(r(r)r9DszCharsetMatch.multi_byte_usagecCstdttt|dS)z Check once again chaos in decoded text, except this time, with full content. Use with caution, this can be very slow. Notice: Will be removed in 3.0 z=chaos_secondary_pass is deprecated and will be removed in 3.0g?)warningswarnDeprecationWarningrr0)r'r(r(r)chaos_secondary_passHsz!CharsetMatch.chaos_secondary_passcCstdtdS)zy Coherence ratio on the first non-latin language detected if ANY. Notice: Will be removed in 3.0 zr?)r'r(r(r)coherence_non_latinUsz CharsetMatch.coherence_non_latincCs,tdtttdt|}t|S)z_ Word counter instance on decoded text. Notice: Will be removed in 3.0 z2w_counter is deprecated and will be removed in 3.0 ) r=r>r?rrr0lowerrsplit)r'Zstring_printable_onlyr(r(r) w_counteraszCharsetMatch.w_countercCs"|jdkrt|j|jd|_|jS)Nstrict)r&r0rr)r'r(r(r)__str__os zCharsetMatch.__str__cCsd|j|jS)Nz)r/r2r3)r'r(r(r)__repr__uszCharsetMatch.__repr__cCs8t|tr||kr"td|jd|_|j|dS)Nz;Unable to add instance <{}> as a submatch of a CharsetMatch)r-rr5r/r1r&r#append)r'r+r(r(r) add_submatchxs  zCharsetMatch.add_submatchcCs|jS)N)r)r'r(r(r)r2szCharsetMatch.encodingcCsHg}x>tD]2\}}|j|kr,||q|j|kr||qW|S)z Encoding name are known by many name, using this could help when searching for IBM855 when it's listed as CP855. )ritemsr2rI)r'Z also_known_asupr(r(r)encoding_aliasess   zCharsetMatch.encoding_aliasescCs|jS)N)r!)r'r(r(r)bomszCharsetMatch.bomcCs|jS)N)r!)r'r(r(r)byte_order_markszCharsetMatch.byte_order_markcCsdd|jDS)z Return the complete list of possible languages found in decoded sequence. Usually not really useful. Returned list may be empty even if 'language' property return something != 'Unknown'. cSsg|] }|dqS)rr().0er(r(r) sz*CharsetMatch.languages..)r )r'r(r(r)rszCharsetMatch.languagescCsp|jsbd|jkrdSddlm}m}t|jr8||jn||j}t|dksVd|krZdS|dS|jddS)z Most probable language found in decoded sequence. If none were detected or inferred, the property will return "Unknown". asciiZEnglishr)encoding_languagesmb_encoding_languagesz Latin BasedUnknown)r could_be_from_charsetZcharset_normalizer.cdrUrVrr2r;)r'rUrVrr(r(r)languages  zCharsetMatch.languagecCs|jS)N)r)r'r(r(r)r7szCharsetMatch.chaoscCs|js dS|jddS)Ngrr)r )r'r(r(r)r8szCharsetMatch.coherencecCst|jdddS)Nd)ndigits)roundr7)r'r(r(r) percent_chaosszCharsetMatch.percent_chaoscCst|jdddS)NrZr[)r\)r]r8)r'r(r(r)percent_coherenceszCharsetMatch.percent_coherencecCs|jS)z+ Original untouched bytes. )r)r'r(r(r)r<szCharsetMatch.rawcCs|jS)N)r#)r'r(r(r)submatchszCharsetMatch.submatchcCst|jdkS)Nr)r;r#)r'r(r(r) has_submatchszCharsetMatch.has_submatchcCs@|jdk r|jSddt|D}ttdd|D|_|jS)NcSsg|] }t|qSr()r)rQcharr(r(r)rSsz*CharsetMatch.alphabets..cSsh|] }|r|qSr(r()rQrr(r(r) sz)CharsetMatch.alphabets..)r"r0sortedlist)r'Zdetected_rangesr(r(r) alphabetss  zCharsetMatch.alphabetscCs|jgdd|jDS)z The complete list of encoding that output the exact SAME str result and therefore could be the originating encoding. This list does include the encoding available in property 'encoding'. cSsg|] }|jqSr()r2)rQmr(r(r)rSsz6CharsetMatch.could_be_from_charset..)rr#)r'r(r(r)rXsz"CharsetMatch.could_be_from_charsetcCs|S)z> Kept for BC reasons. Will be removed in 3.0. r()r'r(r(r)firstszCharsetMatch.firstcCs|S)z> Kept for BC reasons. Will be removed in 3.0. r()r'r(r(r)bestszCharsetMatch.bestutf_8)r2r,cCs2|jdks|j|kr,||_t||d|_|jS)z Method to get re-encoded bytes payload using given target encoding. Default to UTF-8. Any errors will be simply ignored by the encoder NOT replaced. Nreplace)r%r0encoder$)r'r2r(r(r)outputszCharsetMatch.outputcCst|S)zw Retrieve the unique SHA256 computed using the transformed (re-encoded) payload. Not the original one. )rrn hexdigest)r'r(r(r)r3 szCharsetMatch.fingerprint)N)rk))__name__ __module__ __qualname__bytesr0floatboolr r*objectr4r:propertyr9r@rArrErGrHrJr2r rNrOrPrrYr7r8r^r_r<r`rargrXrirjrnr3r(r(r(r)rsb        rc@seZdZdZdeedddZeedddZe e e fed d d Z e dd d Z edddZedd ddZeddddZeddddZdS)CharsetMatchesz Container with every CharsetMatch items ordered by default from most probable to the less one. Act like a list(iterable) but does not implements all related methods. N)resultscCs|r t|ng|_dS)N)re_results)r'ryr(r(r)r*szCharsetMatches.__init__)r,ccs|jEdHdS)N)rz)r'r(r(r)__iter__szCharsetMatches.__iter__)itemr,cCsNt|tr|j|St|trFt|d}x|jD]}||jkr0|Sq0WtdS)z Retrieve a single item either by its position or encoding name (alias may be used here). Raise KeyError upon invalid index or encoding not present in results. FN)r-intrzr0rrXKeyError)r'r|resultr(r(r) __getitem__!s      zCharsetMatches.__getitem__cCs t|jS)N)r;rz)r'r(r(r)__len__/szCharsetMatches.__len__cCst|jdkS)Nr)r;rz)r'r(r(r)__bool__2szCharsetMatches.__bool__cCs~t|tstdt|jt|jtkrbx4|j D]*}|j |j kr4|j |j kr4| |dSq4W|j |t|j |_ dS)z~ Insert a single match. Will be inserted accordingly to preserve sort. Can be inserted as a submatch. z-Cannot append instance '{}' to CharsetMatchesN)r-rr5r/r0r1r;r<rrzr3r7rJrIre)r'r|matchr(r(r)rI5s    zCharsetMatches.appendrcCs|js dS|jdS)zQ Simply return the first match. Strict equivalent to matches[0]. Nr)rz)r'r(r(r)rjIszCharsetMatches.bestcCs|S)zP Redundant method, call the method best(). Kept for BC reasons. )rj)r'r(r(r)riQszCharsetMatches.first)N)rprqrr__doc__r rr*r r{r r}r0rrrurrIr rjrir(r(r(r)rxsrxc @sjeZdZeeeeeeeeeeeeeeeed ddZe e ee fdddZ edddZ d S) CliDetectionResult) pathr2rNalternative_encodingsrYrgrr7r8 unicode_path is_preferredc CsF||_| |_||_||_||_||_||_||_||_| |_ | |_ dS)N) rrr2rNrrYrgrr7r8r) r'rr2rNrrYrgrr7r8rrr(r(r)r*]szCliDetectionResult.__init__)r,c Cs2|j|j|j|j|j|j|j|j|j|j |j d S)N) rr2rNrrYrgrr7r8rr) rr2rNrrYrgrr7r8rr)r'r(r(r)__dict__wszCliDetectionResult.__dict__cCst|jdddS)NT) ensure_asciiindent)rr)r'r(r(r)to_jsonszCliDetectionResult.to_jsonN)rprqrrr0r r rurtr*rwrrrrr(r(r(r)r\sr)#r= collectionsrZencodings.aliasesrhashlibrjsonrrertypingrrr r r r r ZconstantrrZmdrutilsrrrrrxr0rtZCoherenceMatchrrr(r(r(r)s      $  C PK[]f:: __pycache__/utils.cpython-37.pycnu[B b\$@s,y ddlZWnek r(ddlZYnXddlZddlZddlmZddlmZddl m Z ddl m Z ddl mZmZmZmZmZddlmZdd lmZmZmZmZmZmZe ed eed d d Ze ed eed ddZe ed eeed ddZ e ed eed ddZ!eed ddZ"e ed eed ddZ#e ed eed ddZ$e ed eed ddZ%e ed eed ddZ&e ed eed ddZ'eed d d!Z(e ed eed d"d#Z)e ed eed d$d%Z*e ed eed d&d'Z+e ed eed d(d)Z,e ed eed d*d+Z-e e.ed eed,d-d.Z/dNe0e1eed0d1d2Z2e d3d eed4d5d6Z3e0eeee0fd7d8d9Z4eed:d;d<Z5dOeeed>d?d@Z6eeedAdBdCZ7eee8dDdEdFZ9eeedDdGdHZ:dIej;dJfee1eddKdLdMZ<｜>;+,TZ)isspacerr.)rr/rrr is_separator|s r>cCs||kS)N)islowerisupper)rrrris_case_variablesrAcCst|}|dkS)NZCo)rr.)rr/rrris_private_use_onlys rBcCs,yt|}Wntk r"dSXd|kS)NFZCJK)rrr)rcharacter_namerrris_cjks rDcCs,yt|}Wntk r"dSXd|kS)NFZHIRAGANA)rrr)rrCrrr is_hiraganas rEcCs,yt|}Wntk r"dSXd|kS)NFZKATAKANA)rrr)rrCrrr is_katakanas rFcCs,yt|}Wntk r"dSXd|kS)NFZHANGUL)rrr)rrCrrr is_hanguls rGcCs,yt|}Wntk r"dSXd|kS)NFZTHAI)rrr)rrCrrris_thais rH)r&rcstfddtDS)Nc3s|]}|kVqdS)Nr).0keyword)r&rr sz-is_unicode_range_secondary..)anyr)r&r)r&ris_unicode_range_secondarysrM)sequence search_zonercCst|tstt|}tt|dt||jddd}t|dkrHdSxJ|D]B}| dd}x,t D] \}}||kr|S||krl|SqlWqNWdS)zW Extract using ASCII-only decoder any specified encoding in the first n-bytes. Nr)ignore)errorsr-_) isinstancebytes TypeErrorlenrrmindecodelowerreplacerr%)rOrPZseq_lenresultsZspecified_encodingencoding_alias encoding_ianarrrany_specified_encodings     r`)rrcCs |dkpttd|jtS)zQ Verify is a specific encoding is a multi byte one based on it IANA name > utf_8 utf_32_beutf_16 utf_32_leutf_7 utf_8_sigutf_32 utf_16_le utf_16_bez encodings.{}) issubclass importlib import_moduleformatrr )rrrris_multi_byte_encodings ro)rOrcCsJxDtD]<}t|}t|tr"|g}x|D]}||r(||fSq(WqWdS)z9 Identify and extract SIG/BOM in given sequence. )N)r rUrV startswith)rO iana_encodingZmarksZmarkrrridentify_sig_or_boms    rs)rrrcCs|dkS)N>rhrdr)rrrrrshould_strip_sig_or_bom srtT)cp_namestrictrcCsL|dd}x$tD]\}}|||gkr|SqW|rHtd||S)NrSrTz Unable to retrieve IANA for '{}')r[r\rr%rrn)rurvr^r_rrr iana_names rw)decoded_sequencercCs8t}x(|D] }t|}|dkr"q ||q Wt|S)N)setr'addlist)rxZrangesrr0rrr range_scans r|) iana_name_a iana_name_brc Cst|st|rdStd|j}td|j}|dd}|dd}d}x6tdD]*}t|g}||||krZ|d7}qZW|dS) Ngz encodings.{}rQ)rRrr )rorlrmrnrrangerVrZ) r}r~Z decoder_aZ decoder_bZid_aZid_bZcharacter_match_countiZ to_be_decodedrrr cp_similarity+s    rcCs|tko|t|kS)z Determine if two code page are at least 80% similar. IANA_SUPPORTED_SIMILAR dict was generated using the function cp_similarity. )r)r}r~rrr is_cp_similar@srcharset_normalizerz)%(asctime)s | %(levelname)s | %(message)s)rlevel format_stringrcCs:t|}||t}|t|||dS)N)logging getLoggersetLevel StreamHandler setFormatter Formatter addHandler)rrrloggerhandlerrrrset_logging_handlerKs   r)rN)T)= unicodedata2r ImportErrorrlrcodecsrZencodings.aliasesr functoolsrrertypingrrrr r Z_multibytecodecr Zconstantr rrrrrstrboolrr#r'r(r,r1r4r5r>rArBrDrErFrGrHrXrMrVr!r`rorsrtrwr|floatrrINFOrrrrrsr                   PK[]|)g !__pycache__/legacy.cpython-37.pycnu[B b8 @sddlZddlmZmZmZddlmZmZmZm Z ddl m Z ddl m Z mZeeeeeeeffddd ZGd d d e ZGd d d eZGdddeZGdddeZdS)N)DictOptionalUnion) from_bytesfrom_fp from_path normalize)CHARDET_CORRESPONDENCE) CharsetMatchCharsetMatches)byte_strreturncCst|ttfs tdt|t|tr2t|}t|}|dk rL|jnd}|dk rh|j dkrh|j nd}|dk r~d|j nd}|dk r|dkr|j r|d7}|t kr|nt |||dS) a chardet legacy method Detect the encoding of the given byte string. It should be mostly backward-compatible. Encoding name will match Chardet own writing whenever possible. (Not on encoding name unsupported by it) This function is deprecated and should be used to migrate your project easily, consult the documentation for further information. Not planned for removal. :param byte_str: The byte sequence to examine. z4Expected object of type bytes or bytearray, got: {0}NUnknowng?utf_8Z_sig)encodinglanguage confidence) isinstance bytearraybytes TypeErrorformattyperbestrrZchaosbomr )r rrrrrJ/opt/alt/python37/lib/python3.7/site-packages/charset_normalizer/legacy.pydetect s     r c@s eZdZdS)CharsetNormalizerMatchN)__name__ __module__ __qualname__rrrrr!0sr!c@s<eZdZeddZeddZeddZeddZd S) CharsetNormalizerMatchescOstdtt||S)Nzkstaticmethod from_fp, from_bytes, from_path and normalize are deprecated and scheduled to be removed in 3.0)warningswarnDeprecationWarningr)argskwargsrrrr5sz CharsetNormalizerMatches.from_fpcOstdtt||S)Nzkstaticmethod from_fp, from_bytes, from_path and normalize are deprecated and scheduled to be removed in 3.0)r&r'r(r)r)r*rrrr>sz#CharsetNormalizerMatches.from_bytescOstdtt||S)Nzkstaticmethod from_fp, from_bytes, from_path and normalize are deprecated and scheduled to be removed in 3.0)r&r'r(r)r)r*rrrrGsz"CharsetNormalizerMatches.from_pathcOstdtt||S)Nzkstaticmethod from_fp, from_bytes, from_path and normalize are deprecated and scheduled to be removed in 3.0)r&r'r(r )r)r*rrrr Psz"CharsetNormalizerMatches.normalizeN)r"r#r$ staticmethodrrrr rrrrr%4s r%c@s eZdZdS)CharsetDetectorN)r"r#r$rrrrr,Zsr,c@s eZdZdS) CharsetDoctorN)r"r#r$rrrrr-^sr-)r&typingrrrZapirrrr Zconstantr Zmodelsr r rstrfloatr r!r%r,r-rrrrs $'&PK[]R@`#__pycache__/__init__.cpython-37.pycnu[B b@sdZddlZddlmZmZmZmZddlmZm Z m Z m Z m Z ddl mZmZddlmZddlmZmZd Zed edS) u Charset-Normalizer ~~~~~~~~~~~~~~ The Real First Universal Charset Detector. A library that helps you read text from an unknown charset encoding. Motivated by chardet, This package is trying to resolve the issue by taking a new approach. All IANA character set names for which the Python core library provides codecs are supported. Basic usage: >>> from charset_normalizer import from_bytes >>> results = from_bytes('Bсеки човек има право на образование. Oбразованието!'.encode('utf_8')) >>> best_guess = results.best() >>> str(best_guess) 'Bсеки човек има право на образование. Oбразованието!' Others methods and usages are available - see the full documentation at . :copyright: (c) 2021 by Ahmed TAHRI :license: MIT, see LICENSE for more details. N) from_bytesfrom_fp from_path normalize)CharsetDetector CharsetDoctorCharsetNormalizerMatchCharsetNormalizerMatchesdetect) CharsetMatchCharsetMatches)set_logging_handler)VERSION __version__)rrrrr r r r r rrrrrcharset_normalizer)__doc__loggingZapirrrrZlegacyrrr r r Zmodelsr r utilsrversionrr__all__ getLogger addHandler NullHandlerrrL/opt/alt/python37/lib/python3.7/site-packages/charset_normalizer/__init__.pys PK[]>(__pycache__/version.cpython-37.opt-1.pycnu[B bP@sdZdZedZdS)z Expose version z2.0.12.N)__doc__ __version__splitZVERSIONrrK/opt/alt/python37/lib/python3.7/site-packages/charset_normalizer/version.pysPK[]t0assets/__pycache__/__init__.cpython-37.opt-1.pycnu[B bcC@sF ddlmZedddddddd d d d d dddddddddddddddgfddddd d ddd d dd dddddddddddddd dgfd!ddd dddd d ddd dddd"ddddd dd#dd$ddgfd%ddddd ddd d d dd dddddddddddddd&gfd'dddddd dd d dd ddddddddd dd$d#ddd(gfd)dddddd ddd ddddd dddd dd*ddd d+d,d-gfd.ddddd d dd d ddddddddddd-d dd/ddd0gfd1d2d3d4d5d6d7d8d9d:d;dd?d@dAdBdCdDdEdFdGdHdIdJdKgfdLdMdNdOdPdQdRdSdTdUdVdWdXdYdZd[d\d]d^d_d`dadbdcdddedfgfdgdddd dd d dddddd dddddd dhdd"did0dd/gfdjdddd dd dd d dddddd ddddddd dkdddgfdldmdndodpdqdrdsdtdudvdwdxdydzd{d|d}d~dddddddddddgfdd2d3d6dd5d9d:d8d4d7dd=d?dDdBdGdEdAdFdHdIdKdgfddd dddd ddd d ddddddddd dkddddddgfddddddd d dddddd ddd ddd d ddddddgfddd ddddddddd d d#dd ddddddd d0dddgfddddddd dd dd dd dddd/dd dd0ddddd"dgfddddd d ddd dddd0d"dddddd d dddd ddgfdddddddddddddddddddddddddddgfdddddd ddd d ddd ddddd dddddddddgfdddddd d ddd dd ddddddddddd dddidgfddddd dddd ddd d dddddddddddd ddgfdddddddddddddddddddddddddddgfdddddddddddddddddddddddddddgfddd ddddd d d ddddddddd ddkddddddgfdd3d5d2d4d6d9d7d@d8dd=d?d;dAdDdEddddddKdgfdddd dd ddddddd ddd dddddddddd+dgfddddddd d d ddddd ddddddd ddddddgfddddddd ddd d dd ddddd dddd0ddd/dd"gfdddddddddddddddddddddddddd gfd d3d5d2d4d6d8d9d7d:d;dd?d=dDdAdBd d@dEdHdKdFdJd dIgfd dddddd dd dddd dd dddddddd ddddgfdddddddddddddddddd d!d"d#d$d%d&d'd(d)gfd*dddd dd ddddd d dddddd ddd+dddd dgfd,ddddddd d d d d dddddddddddddddgfd-d.d/d0d1d2d3d4d5d6d7d8d9d:d;d<d=d>d?d@dAdBdCdDdEdFdGgfdHdIdJdKdLdMdNdOdPdQdRdSdTdUdVdWdXdYdZd[d\d]d^d_d`dadbgfdcdddedfdgdhdidjdkdldmdndodpdqdrdsdtdudvdwdxdydzd{gfd|d}d]d~dddndsddddtddddddpd}dddzddgfdd3dCd4d6d8d9d;dd>d7d=ddsl PK[]t*assets/__pycache__/__init__.cpython-37.pycnu[B bcC@sF ddlmZedddddddd d d d d dddddddddddddddgfddddd d ddd d dd dddddddddddddd dgfd!ddd dddd d ddd dddd"ddddd dd#dd$ddgfd%ddddd ddd d d dd dddddddddddddd&gfd'dddddd dd d dd ddddddddd dd$d#ddd(gfd)dddddd ddd ddddd dddd dd*ddd d+d,d-gfd.ddddd d dd d ddddddddddd-d dd/ddd0gfd1d2d3d4d5d6d7d8d9d:d;dd?d@dAdBdCdDdEdFdGdHdIdJdKgfdLdMdNdOdPdQdRdSdTdUdVdWdXdYdZd[d\d]d^d_d`dadbdcdddedfgfdgdddd dd d dddddd dddddd dhdd"did0dd/gfdjdddd dd dd d dddddd ddddddd dkdddgfdldmdndodpdqdrdsdtdudvdwdxdydzd{d|d}d~dddddddddddgfdd2d3d6dd5d9d:d8d4d7dd=d?dDdBdGdEdAdFdHdIdKdgfddd dddd ddd d ddddddddd dkddddddgfddddddd d dddddd ddd ddd d ddddddgfddd ddddddddd d d#dd ddddddd d0dddgfddddddd dd dd dd dddd/dd dd0ddddd"dgfddddd d ddd dddd0d"dddddd d dddd ddgfdddddddddddddddddddddddddddgfdddddd ddd d ddd ddddd dddddddddgfdddddd d ddd dd ddddddddddd dddidgfddddd dddd ddd d dddddddddddd ddgfdddddddddddddddddddddddddddgfdddddddddddddddddddddddddddgfddd ddddd d d ddddddddd ddkddddddgfdd3d5d2d4d6d9d7d@d8dd=d?d;dAdDdEddddddKdgfdddd dd ddddddd ddd dddddddddd+dgfddddddd d d ddddd ddddddd ddddddgfddddddd ddd d dd ddddd dddd0ddd/dd"gfdddddddddddddddddddddddddd gfd d3d5d2d4d6d8d9d7d:d;dd?d=dDdAdBd d@dEdHdKdFdJd dIgfd dddddd dd dddd dd dddddddd ddddgfdddddddddddddddddd d!d"d#d$d%d&d'd(d)gfd*dddd dd ddddd d dddddd ddd+dddd dgfd,ddddddd d d d d dddddddddddddddgfd-d.d/d0d1d2d3d4d5d6d7d8d9d:d;d<d=d>d?d@dAdBdCdDdEdFdGgfdHdIdJdKdLdMdNdOdPdQdRdSdTdUdVdWdXdYdZd[d\d]d^d_d`dadbgfdcdddedfdgdhdidjdkdldmdndodpdqdrdsdtdudvdwdxdydzd{gfd|d}d]d~dddndsddddtddddddpd}dddzddgfdd3dCd4d6d8d9d;dd>d7d=ddsl PK[]ûҍccassets/__init__.pynu[# -*- coding: utf_8 -*- from collections import OrderedDict FREQUENCIES = OrderedDict( [ ( "English", [ "e", "a", "t", "i", "o", "n", "s", "r", "h", "l", "d", "c", "u", "m", "f", "p", "g", "w", "y", "b", "v", "k", "x", "j", "z", "q", ], ), ( "German", [ "e", "n", "i", "r", "s", "t", "a", "d", "h", "u", "l", "g", "o", "c", "m", "b", "f", "k", "w", "z", "p", "v", "ü", "ä", "ö", "j", ], ), ( "French", [ "e", "a", "s", "n", "i", "t", "r", "l", "u", "o", "d", "c", "p", "m", "é", "v", "g", "f", "b", "h", "q", "à", "x", "è", "y", "j", ], ), ( "Dutch", [ "e", "n", "a", "i", "r", "t", "o", "d", "s", "l", "g", "h", "v", "m", "u", "k", "c", "p", "b", "w", "j", "z", "f", "y", "x", "ë", ], ), ( "Italian", [ "e", "i", "a", "o", "n", "l", "t", "r", "s", "c", "d", "u", "p", "m", "g", "v", "f", "b", "z", "h", "q", "è", "à", "k", "y", "ò", ], ), ( "Polish", [ "a", "i", "o", "e", "n", "r", "z", "w", "s", "c", "t", "k", "y", "d", "p", "m", "u", "l", "j", "ł", "g", "b", "h", "ą", "ę", "ó", ], ), ( "Spanish", [ "e", "a", "o", "n", "s", "r", "i", "l", "d", "t", "c", "u", "m", "p", "b", "g", "v", "f", "y", "ó", "h", "q", "í", "j", "z", "á", ], ), ( "Russian", [ "о", "а", "е", "и", "н", "с", "т", "р", "в", "л", "к", "м", "д", "п", "у", "г", "я", "ы", "з", "б", "й", "ь", "ч", "х", "ж", "ц", ], ), ( "Japanese", [ "の", "に", "る", "た", "は", "ー", "と", "し", "を", "で", "て", "が", "い", "ン", "れ", "な", "年", "ス", "っ", "ル", "か", "ら", "あ", "さ", "も", "り", ], ), ( "Portuguese", [ "a", "e", "o", "s", "i", "r", "d", "n", "t", "m", "u", "c", "l", "p", "g", "v", "b", "f", "h", "ã", "q", "é", "ç", "á", "z", "í", ], ), ( "Swedish", [ "e", "a", "n", "r", "t", "s", "i", "l", "d", "o", "m", "k", "g", "v", "h", "f", "u", "p", "ä", "c", "b", "ö", "å", "y", "j", "x", ], ), ( "Chinese", [ "的", "一", "是", "不", "了", "在", "人", "有", "我", "他", "这", "个", "们", "中", "来", "上", "大", "为", "和", "国", "地", "到", "以", "说", "时", "要", "就", "出", "会", ], ), ( "Ukrainian", [ "о", "а", "н", "і", "и", "р", "в", "т", "е", "с", "к", "л", "у", "д", "м", "п", "з", "я", "ь", "б", "г", "й", "ч", "х", "ц", "ї", ], ), ( "Norwegian", [ "e", "r", "n", "t", "a", "s", "i", "o", "l", "d", "g", "k", "m", "v", "f", "p", "u", "b", "h", "å", "y", "j", "ø", "c", "æ", "w", ], ), ( "Finnish", [ "a", "i", "n", "t", "e", "s", "l", "o", "u", "k", "ä", "m", "r", "v", "j", "h", "p", "y", "d", "ö", "g", "c", "b", "f", "w", "z", ], ), ( "Vietnamese", [ "n", "h", "t", "i", "c", "g", "a", "o", "u", "m", "l", "r", "à", "đ", "s", "e", "v", "p", "b", "y", "ư", "d", "á", "k", "ộ", "ế", ], ), ( "Czech", [ "o", "e", "a", "n", "t", "s", "i", "l", "v", "r", "k", "d", "u", "m", "p", "í", "c", "h", "z", "á", "y", "j", "b", "ě", "é", "ř", ], ), ( "Hungarian", [ "e", "a", "t", "l", "s", "n", "k", "r", "i", "o", "z", "á", "é", "g", "m", "b", "y", "v", "d", "h", "u", "p", "j", "ö", "f", "c", ], ), ( "Korean", [ "이", "다", "에", "의", "는", "로", "하", "을", "가", "고", "지", "서", "한", "은", "기", "으", "년", "대", "사", "시", "를", "리", "도", "인", "스", "일", ], ), ( "Indonesian", [ "a", "n", "e", "i", "r", "t", "u", "s", "d", "k", "m", "l", "g", "p", "b", "o", "h", "y", "j", "c", "w", "f", "v", "z", "x", "q", ], ), ( "Turkish", [ "a", "e", "i", "n", "r", "l", "ı", "k", "d", "t", "s", "m", "y", "u", "o", "b", "ü", "ş", "v", "g", "z", "h", "c", "p", "ç", "ğ", ], ), ( "Romanian", [ "e", "i", "a", "r", "n", "t", "u", "l", "o", "c", "s", "d", "p", "m", "ă", "f", "v", "î", "g", "b", "ș", "ț", "z", "h", "â", "j", ], ), ( "Farsi", [ "ا", "ی", "ر", "د", "ن", "ه", "و", "م", "ت", "ب", "س", "ل", "ک", "ش", "ز", "ف", "گ", "ع", "خ", "ق", "ج", "آ", "پ", "ح", "ط", "ص", ], ), ( "Arabic", [ "ا", "ل", "ي", "م", "و", "ن", "ر", "ت", "ب", "ة", "ع", "د", "س", "ف", "ه", "ك", "ق", "أ", "ح", "ج", "ش", "ط", "ص", "ى", "خ", "إ", ], ), ( "Danish", [ "e", "r", "n", "t", "a", "i", "s", "d", "l", "o", "g", "m", "k", "f", "v", "u", "b", "h", "p", "å", "y", "ø", "æ", "c", "j", "w", ], ), ( "Serbian", [ "а", "и", "о", "е", "н", "р", "с", "у", "т", "к", "ј", "в", "д", "м", "п", "л", "г", "з", "б", "a", "i", "e", "o", "n", "ц", "ш", ], ), ( "Lithuanian", [ "i", "a", "s", "o", "r", "e", "t", "n", "u", "k", "m", "l", "p", "v", "d", "j", "g", "ė", "b", "y", "ų", "š", "ž", "c", "ą", "į", ], ), ( "Slovene", [ "e", "a", "i", "o", "n", "r", "s", "l", "t", "j", "v", "k", "d", "p", "m", "u", "z", "b", "g", "h", "č", "c", "š", "ž", "f", "y", ], ), ( "Slovak", [ "o", "a", "e", "n", "i", "r", "v", "t", "s", "l", "k", "d", "m", "p", "u", "c", "h", "j", "b", "z", "á", "y", "ý", "í", "č", "é", ], ), ( "Hebrew", [ "י", "ו", "ה", "ל", "ר", "ב", "ת", "מ", "א", "ש", "נ", "ע", "ם", "ד", "ק", "ח", "פ", "ס", "כ", "ג", "ט", "צ", "ן", "ז", "ך", ], ), ( "Bulgarian", [ "а", "и", "о", "е", "н", "т", "р", "с", "в", "л", "к", "д", "п", "м", "з", "г", "я", "ъ", "у", "б", "ч", "ц", "й", "ж", "щ", "х", ], ), ( "Croatian", [ "a", "i", "o", "e", "n", "r", "j", "s", "t", "u", "k", "l", "v", "d", "m", "p", "g", "z", "b", "c", "č", "h", "š", "ž", "ć", "f", ], ), ( "Hindi", [ "क", "र", "स", "न", "त", "म", "ह", "प", "य", "ल", "व", "ज", "द", "ग", "ब", "श", "ट", "अ", "ए", "थ", "भ", "ड", "च", "ध", "ष", "इ", ], ), ( "Estonian", [ "a", "i", "e", "s", "t", "l", "u", "n", "o", "k", "r", "d", "m", "v", "g", "p", "j", "h", "ä", "b", "õ", "ü", "f", "c", "ö", "y", ], ), ( "Simple English", [ "e", "a", "t", "i", "o", "n", "s", "r", "h", "l", "d", "c", "m", "u", "f", "p", "g", "w", "b", "y", "v", "k", "j", "x", "z", "q", ], ), ( "Thai", [ "า", "น", "ร", "อ", "ก", "เ", "ง", "ม", "ย", "ล", "ว", "ด", "ท", "ส", "ต", "ะ", "ป", "บ", "ค", "ห", "แ", "จ", "พ", "ช", "ข", "ใ", ], ), ( "Greek", [ "α", "τ", "ο", "ι", "ε", "ν", "ρ", "σ", "κ", "η", "π", "ς", "υ", "μ", "λ", "ί", "ό", "ά", "γ", "έ", "δ", "ή", "ω", "χ", "θ", "ύ", ], ), ( "Tamil", [ "க", "த", "ப", "ட", "ர", "ம", "ல", "ன", "வ", "ற", "ய", "ள", "ச", "ந", "இ", "ண", "அ", "ஆ", "ழ", "ங", "எ", "உ", "ஒ", "ஸ", ], ), ( "Classical Chinese", [ "之", "年", "為", "也", "以", "一", "人", "其", "者", "國", "有", "二", "十", "於", "曰", "三", "不", "大", "而", "子", "中", "五", "四", ], ), ( "Kazakh", [ "а", "ы", "е", "н", "т", "р", "л", "і", "д", "с", "м", "қ", "к", "о", "б", "и", "у", "ғ", "ж", "ң", "з", "ш", "й", "п", "г", "ө", ], ), ] ) PK[]Y =33 models.pynu[import warnings from collections import Counter from encodings.aliases import aliases from hashlib import sha256 from json import dumps from re import sub from typing import Any, Dict, Iterator, List, Optional, Tuple, Union from .constant import NOT_PRINTABLE_PATTERN, TOO_BIG_SEQUENCE from .md import mess_ratio from .utils import iana_name, is_multi_byte_encoding, unicode_range class CharsetMatch: def __init__( self, payload: bytes, guessed_encoding: str, mean_mess_ratio: float, has_sig_or_bom: bool, languages: "CoherenceMatches", decoded_payload: Optional[str] = None, ): self._payload = payload # type: bytes self._encoding = guessed_encoding # type: str self._mean_mess_ratio = mean_mess_ratio # type: float self._languages = languages # type: CoherenceMatches self._has_sig_or_bom = has_sig_or_bom # type: bool self._unicode_ranges = None # type: Optional[List[str]] self._leaves = [] # type: List[CharsetMatch] self._mean_coherence_ratio = 0.0 # type: float self._output_payload = None # type: Optional[bytes] self._output_encoding = None # type: Optional[str] self._string = decoded_payload # type: Optional[str] def __eq__(self, other: object) -> bool: if not isinstance(other, CharsetMatch): raise TypeError( "__eq__ cannot be invoked on {} and {}.".format( str(other.__class__), str(self.__class__) ) ) return self.encoding == other.encoding and self.fingerprint == other.fingerprint def __lt__(self, other: object) -> bool: """ Implemented to make sorted available upon CharsetMatches items. """ if not isinstance(other, CharsetMatch): raise ValueError chaos_difference = abs(self.chaos - other.chaos) # type: float coherence_difference = abs(self.coherence - other.coherence) # type: float # Bellow 1% difference --> Use Coherence if chaos_difference < 0.01 and coherence_difference > 0.02: # When having a tough decision, use the result that decoded as many multi-byte as possible. if chaos_difference == 0.0 and self.coherence == other.coherence: return self.multi_byte_usage > other.multi_byte_usage return self.coherence > other.coherence return self.chaos < other.chaos @property def multi_byte_usage(self) -> float: return 1.0 - len(str(self)) / len(self.raw) @property def chaos_secondary_pass(self) -> float: """ Check once again chaos in decoded text, except this time, with full content. Use with caution, this can be very slow. Notice: Will be removed in 3.0 """ warnings.warn( "chaos_secondary_pass is deprecated and will be removed in 3.0", DeprecationWarning, ) return mess_ratio(str(self), 1.0) @property def coherence_non_latin(self) -> float: """ Coherence ratio on the first non-latin language detected if ANY. Notice: Will be removed in 3.0 """ warnings.warn( "coherence_non_latin is deprecated and will be removed in 3.0", DeprecationWarning, ) return 0.0 @property def w_counter(self) -> Counter: """ Word counter instance on decoded text. Notice: Will be removed in 3.0 """ warnings.warn( "w_counter is deprecated and will be removed in 3.0", DeprecationWarning ) string_printable_only = sub(NOT_PRINTABLE_PATTERN, " ", str(self).lower()) return Counter(string_printable_only.split()) def __str__(self) -> str: # Lazy Str Loading if self._string is None: self._string = str(self._payload, self._encoding, "strict") return self._string def __repr__(self) -> str: return "".format(self.encoding, self.fingerprint) def add_submatch(self, other: "CharsetMatch") -> None: if not isinstance(other, CharsetMatch) or other == self: raise ValueError( "Unable to add instance <{}> as a submatch of a CharsetMatch".format( other.__class__ ) ) other._string = None # Unload RAM usage; dirty trick. self._leaves.append(other) @property def encoding(self) -> str: return self._encoding @property def encoding_aliases(self) -> List[str]: """ Encoding name are known by many name, using this could help when searching for IBM855 when it's listed as CP855. """ also_known_as = [] # type: List[str] for u, p in aliases.items(): if self.encoding == u: also_known_as.append(p) elif self.encoding == p: also_known_as.append(u) return also_known_as @property def bom(self) -> bool: return self._has_sig_or_bom @property def byte_order_mark(self) -> bool: return self._has_sig_or_bom @property def languages(self) -> List[str]: """ Return the complete list of possible languages found in decoded sequence. Usually not really useful. Returned list may be empty even if 'language' property return something != 'Unknown'. """ return [e[0] for e in self._languages] @property def language(self) -> str: """ Most probable language found in decoded sequence. If none were detected or inferred, the property will return "Unknown". """ if not self._languages: # Trying to infer the language based on the given encoding # Its either English or we should not pronounce ourselves in certain cases. if "ascii" in self.could_be_from_charset: return "English" # doing it there to avoid circular import from charset_normalizer.cd import encoding_languages, mb_encoding_languages languages = ( mb_encoding_languages(self.encoding) if is_multi_byte_encoding(self.encoding) else encoding_languages(self.encoding) ) if len(languages) == 0 or "Latin Based" in languages: return "Unknown" return languages[0] return self._languages[0][0] @property def chaos(self) -> float: return self._mean_mess_ratio @property def coherence(self) -> float: if not self._languages: return 0.0 return self._languages[0][1] @property def percent_chaos(self) -> float: return round(self.chaos * 100, ndigits=3) @property def percent_coherence(self) -> float: return round(self.coherence * 100, ndigits=3) @property def raw(self) -> bytes: """ Original untouched bytes. """ return self._payload @property def submatch(self) -> List["CharsetMatch"]: return self._leaves @property def has_submatch(self) -> bool: return len(self._leaves) > 0 @property def alphabets(self) -> List[str]: if self._unicode_ranges is not None: return self._unicode_ranges # list detected ranges detected_ranges = [ unicode_range(char) for char in str(self) ] # type: List[Optional[str]] # filter and sort self._unicode_ranges = sorted(list({r for r in detected_ranges if r})) return self._unicode_ranges @property def could_be_from_charset(self) -> List[str]: """ The complete list of encoding that output the exact SAME str result and therefore could be the originating encoding. This list does include the encoding available in property 'encoding'. """ return [self._encoding] + [m.encoding for m in self._leaves] def first(self) -> "CharsetMatch": """ Kept for BC reasons. Will be removed in 3.0. """ return self def best(self) -> "CharsetMatch": """ Kept for BC reasons. Will be removed in 3.0. """ return self def output(self, encoding: str = "utf_8") -> bytes: """ Method to get re-encoded bytes payload using given target encoding. Default to UTF-8. Any errors will be simply ignored by the encoder NOT replaced. """ if self._output_encoding is None or self._output_encoding != encoding: self._output_encoding = encoding self._output_payload = str(self).encode(encoding, "replace") return self._output_payload # type: ignore @property def fingerprint(self) -> str: """ Retrieve the unique SHA256 computed using the transformed (re-encoded) payload. Not the original one. """ return sha256(self.output()).hexdigest() class CharsetMatches: """ Container with every CharsetMatch items ordered by default from most probable to the less one. Act like a list(iterable) but does not implements all related methods. """ def __init__(self, results: List[CharsetMatch] = None): self._results = sorted(results) if results else [] # type: List[CharsetMatch] def __iter__(self) -> Iterator[CharsetMatch]: yield from self._results def __getitem__(self, item: Union[int, str]) -> CharsetMatch: """ Retrieve a single item either by its position or encoding name (alias may be used here). Raise KeyError upon invalid index or encoding not present in results. """ if isinstance(item, int): return self._results[item] if isinstance(item, str): item = iana_name(item, False) for result in self._results: if item in result.could_be_from_charset: return result raise KeyError def __len__(self) -> int: return len(self._results) def __bool__(self) -> bool: return len(self._results) > 0 def append(self, item: CharsetMatch) -> None: """ Insert a single match. Will be inserted accordingly to preserve sort. Can be inserted as a submatch. """ if not isinstance(item, CharsetMatch): raise ValueError( "Cannot append instance '{}' to CharsetMatches".format( str(item.__class__) ) ) # We should disable the submatch factoring when the input file is too heavy (conserve RAM usage) if len(item.raw) <= TOO_BIG_SEQUENCE: for match in self._results: if match.fingerprint == item.fingerprint and match.chaos == item.chaos: match.add_submatch(item) return self._results.append(item) self._results = sorted(self._results) def best(self) -> Optional["CharsetMatch"]: """ Simply return the first match. Strict equivalent to matches[0]. """ if not self._results: return None return self._results[0] def first(self) -> Optional["CharsetMatch"]: """ Redundant method, call the method best(). Kept for BC reasons. """ return self.best() CoherenceMatch = Tuple[str, float] CoherenceMatches = List[CoherenceMatch] class CliDetectionResult: def __init__( self, path: str, encoding: Optional[str], encoding_aliases: List[str], alternative_encodings: List[str], language: str, alphabets: List[str], has_sig_or_bom: bool, chaos: float, coherence: float, unicode_path: Optional[str], is_preferred: bool, ): self.path = path # type: str self.unicode_path = unicode_path # type: Optional[str] self.encoding = encoding # type: Optional[str] self.encoding_aliases = encoding_aliases # type: List[str] self.alternative_encodings = alternative_encodings # type: List[str] self.language = language # type: str self.alphabets = alphabets # type: List[str] self.has_sig_or_bom = has_sig_or_bom # type: bool self.chaos = chaos # type: float self.coherence = coherence # type: float self.is_preferred = is_preferred # type: bool @property def __dict__(self) -> Dict[str, Any]: # type: ignore return { "path": self.path, "encoding": self.encoding, "encoding_aliases": self.encoding_aliases, "alternative_encodings": self.alternative_encodings, "language": self.language, "alphabets": self.alphabets, "has_sig_or_bom": self.has_sig_or_bom, "chaos": self.chaos, "coherence": self.coherence, "unicode_path": self.unicode_path, "is_preferred": self.is_preferred, } def to_json(self) -> str: return dumps(self.__dict__, ensure_ascii=True, indent=4) PK[]qW8 8 legacy.pynu[import warnings from typing import Dict, Optional, Union from .api import from_bytes, from_fp, from_path, normalize from .constant import CHARDET_CORRESPONDENCE from .models import CharsetMatch, CharsetMatches def detect(byte_str: bytes) -> Dict[str, Optional[Union[str, float]]]: """ chardet legacy method Detect the encoding of the given byte string. It should be mostly backward-compatible. Encoding name will match Chardet own writing whenever possible. (Not on encoding name unsupported by it) This function is deprecated and should be used to migrate your project easily, consult the documentation for further information. Not planned for removal. :param byte_str: The byte sequence to examine. """ if not isinstance(byte_str, (bytearray, bytes)): raise TypeError( # pragma: nocover "Expected object of type bytes or bytearray, got: " "{0}".format(type(byte_str)) ) if isinstance(byte_str, bytearray): byte_str = bytes(byte_str) r = from_bytes(byte_str).best() encoding = r.encoding if r is not None else None language = r.language if r is not None and r.language != "Unknown" else "" confidence = 1.0 - r.chaos if r is not None else None # Note: CharsetNormalizer does not return 'UTF-8-SIG' as the sig get stripped in the detection/normalization process # but chardet does return 'utf-8-sig' and it is a valid codec name. if r is not None and encoding == "utf_8" and r.bom: encoding += "_sig" return { "encoding": encoding if encoding not in CHARDET_CORRESPONDENCE else CHARDET_CORRESPONDENCE[encoding], "language": language, "confidence": confidence, } class CharsetNormalizerMatch(CharsetMatch): pass class CharsetNormalizerMatches(CharsetMatches): @staticmethod def from_fp(*args, **kwargs): # type: ignore warnings.warn( # pragma: nocover "staticmethod from_fp, from_bytes, from_path and normalize are deprecated " "and scheduled to be removed in 3.0", DeprecationWarning, ) return from_fp(*args, **kwargs) # pragma: nocover @staticmethod def from_bytes(*args, **kwargs): # type: ignore warnings.warn( # pragma: nocover "staticmethod from_fp, from_bytes, from_path and normalize are deprecated " "and scheduled to be removed in 3.0", DeprecationWarning, ) return from_bytes(*args, **kwargs) # pragma: nocover @staticmethod def from_path(*args, **kwargs): # type: ignore warnings.warn( # pragma: nocover "staticmethod from_fp, from_bytes, from_path and normalize are deprecated " "and scheduled to be removed in 3.0", DeprecationWarning, ) return from_path(*args, **kwargs) # pragma: nocover @staticmethod def normalize(*args, **kwargs): # type: ignore warnings.warn( # pragma: nocover "staticmethod from_fp, from_bytes, from_path and normalize are deprecated " "and scheduled to be removed in 3.0", DeprecationWarning, ) return normalize(*args, **kwargs) # pragma: nocover class CharsetDetector(CharsetNormalizerMatches): pass class CharsetDoctor(CharsetNormalizerMatches): pass PK[]IҍOOOOapi.pynu[import logging from os.path import basename, splitext from typing import BinaryIO, List, Optional, Set try: from os import PathLike except ImportError: # pragma: no cover PathLike = str # type: ignore from .cd import ( coherence_ratio, encoding_languages, mb_encoding_languages, merge_coherence_ratios, ) from .constant import IANA_SUPPORTED, TOO_BIG_SEQUENCE, TOO_SMALL_SEQUENCE, TRACE from .md import mess_ratio from .models import CharsetMatch, CharsetMatches from .utils import ( any_specified_encoding, iana_name, identify_sig_or_bom, is_cp_similar, is_multi_byte_encoding, should_strip_sig_or_bom, ) # Will most likely be controversial # logging.addLevelName(TRACE, "TRACE") logger = logging.getLogger("charset_normalizer") explain_handler = logging.StreamHandler() explain_handler.setFormatter( logging.Formatter("%(asctime)s | %(levelname)s | %(message)s") ) def from_bytes( sequences: bytes, steps: int = 5, chunk_size: int = 512, threshold: float = 0.2, cp_isolation: List[str] = None, cp_exclusion: List[str] = None, preemptive_behaviour: bool = True, explain: bool = False, ) -> CharsetMatches: """ Given a raw bytes sequence, return the best possibles charset usable to render str objects. If there is no results, it is a strong indicator that the source is binary/not text. By default, the process will extract 5 blocs of 512o each to assess the mess and coherence of a given sequence. And will give up a particular code page after 20% of measured mess. Those criteria are customizable at will. The preemptive behavior DOES NOT replace the traditional detection workflow, it prioritize a particular code page but never take it for granted. Can improve the performance. You may want to focus your attention to some code page or/and not others, use cp_isolation and cp_exclusion for that purpose. This function will strip the SIG in the payload/sequence every time except on UTF-16, UTF-32. By default the library does not setup any handler other than the NullHandler, if you choose to set the 'explain' toggle to True it will alter the logger configuration to add a StreamHandler that is suitable for debugging. Custom logging format and handler can be set manually. """ if not isinstance(sequences, (bytearray, bytes)): raise TypeError( "Expected object of type bytes or bytearray, got: {0}".format( type(sequences) ) ) if explain: previous_logger_level = logger.level # type: int logger.addHandler(explain_handler) logger.setLevel(TRACE) length = len(sequences) # type: int if length == 0: logger.debug("Encoding detection on empty bytes, assuming utf_8 intention.") if explain: logger.removeHandler(explain_handler) logger.setLevel(previous_logger_level or logging.WARNING) return CharsetMatches([CharsetMatch(sequences, "utf_8", 0.0, False, [], "")]) if cp_isolation is not None: logger.log( TRACE, "cp_isolation is set. use this flag for debugging purpose. " "limited list of encoding allowed : %s.", ", ".join(cp_isolation), ) cp_isolation = [iana_name(cp, False) for cp in cp_isolation] else: cp_isolation = [] if cp_exclusion is not None: logger.log( TRACE, "cp_exclusion is set. use this flag for debugging purpose. " "limited list of encoding excluded : %s.", ", ".join(cp_exclusion), ) cp_exclusion = [iana_name(cp, False) for cp in cp_exclusion] else: cp_exclusion = [] if length <= (chunk_size * steps): logger.log( TRACE, "override steps (%i) and chunk_size (%i) as content does not fit (%i byte(s) given) parameters.", steps, chunk_size, length, ) steps = 1 chunk_size = length if steps > 1 and length / steps < chunk_size: chunk_size = int(length / steps) is_too_small_sequence = len(sequences) < TOO_SMALL_SEQUENCE # type: bool is_too_large_sequence = len(sequences) >= TOO_BIG_SEQUENCE # type: bool if is_too_small_sequence: logger.log( TRACE, "Trying to detect encoding from a tiny portion of ({}) byte(s).".format( length ), ) elif is_too_large_sequence: logger.log( TRACE, "Using lazy str decoding because the payload is quite large, ({}) byte(s).".format( length ), ) prioritized_encodings = [] # type: List[str] specified_encoding = ( any_specified_encoding(sequences) if preemptive_behaviour else None ) # type: Optional[str] if specified_encoding is not None: prioritized_encodings.append(specified_encoding) logger.log( TRACE, "Detected declarative mark in sequence. Priority +1 given for %s.", specified_encoding, ) tested = set() # type: Set[str] tested_but_hard_failure = [] # type: List[str] tested_but_soft_failure = [] # type: List[str] fallback_ascii = None # type: Optional[CharsetMatch] fallback_u8 = None # type: Optional[CharsetMatch] fallback_specified = None # type: Optional[CharsetMatch] results = CharsetMatches() # type: CharsetMatches sig_encoding, sig_payload = identify_sig_or_bom(sequences) if sig_encoding is not None: prioritized_encodings.append(sig_encoding) logger.log( TRACE, "Detected a SIG or BOM mark on first %i byte(s). Priority +1 given for %s.", len(sig_payload), sig_encoding, ) prioritized_encodings.append("ascii") if "utf_8" not in prioritized_encodings: prioritized_encodings.append("utf_8") for encoding_iana in prioritized_encodings + IANA_SUPPORTED: if cp_isolation and encoding_iana not in cp_isolation: continue if cp_exclusion and encoding_iana in cp_exclusion: continue if encoding_iana in tested: continue tested.add(encoding_iana) decoded_payload = None # type: Optional[str] bom_or_sig_available = sig_encoding == encoding_iana # type: bool strip_sig_or_bom = bom_or_sig_available and should_strip_sig_or_bom( encoding_iana ) # type: bool if encoding_iana in {"utf_16", "utf_32"} and not bom_or_sig_available: logger.log( TRACE, "Encoding %s wont be tested as-is because it require a BOM. Will try some sub-encoder LE/BE.", encoding_iana, ) continue try: is_multi_byte_decoder = is_multi_byte_encoding(encoding_iana) # type: bool except (ModuleNotFoundError, ImportError): logger.log( TRACE, "Encoding %s does not provide an IncrementalDecoder", encoding_iana, ) continue try: if is_too_large_sequence and is_multi_byte_decoder is False: str( sequences[: int(50e4)] if strip_sig_or_bom is False else sequences[len(sig_payload) : int(50e4)], encoding=encoding_iana, ) else: decoded_payload = str( sequences if strip_sig_or_bom is False else sequences[len(sig_payload) :], encoding=encoding_iana, ) except (UnicodeDecodeError, LookupError) as e: if not isinstance(e, LookupError): logger.log( TRACE, "Code page %s does not fit given bytes sequence at ALL. %s", encoding_iana, str(e), ) tested_but_hard_failure.append(encoding_iana) continue similar_soft_failure_test = False # type: bool for encoding_soft_failed in tested_but_soft_failure: if is_cp_similar(encoding_iana, encoding_soft_failed): similar_soft_failure_test = True break if similar_soft_failure_test: logger.log( TRACE, "%s is deemed too similar to code page %s and was consider unsuited already. Continuing!", encoding_iana, encoding_soft_failed, ) continue r_ = range( 0 if not bom_or_sig_available else len(sig_payload), length, int(length / steps), ) multi_byte_bonus = ( is_multi_byte_decoder and decoded_payload is not None and len(decoded_payload) < length ) # type: bool if multi_byte_bonus: logger.log( TRACE, "Code page %s is a multi byte encoding table and it appear that at least one character " "was encoded using n-bytes.", encoding_iana, ) max_chunk_gave_up = int(len(r_) / 4) # type: int max_chunk_gave_up = max(max_chunk_gave_up, 2) early_stop_count = 0 # type: int lazy_str_hard_failure = False md_chunks = [] # type: List[str] md_ratios = [] for i in r_: if i + chunk_size > length + 8: continue cut_sequence = sequences[i : i + chunk_size] if bom_or_sig_available and strip_sig_or_bom is False: cut_sequence = sig_payload + cut_sequence try: chunk = cut_sequence.decode( encoding_iana, errors="ignore" if is_multi_byte_decoder else "strict", ) # type: str except UnicodeDecodeError as e: # Lazy str loading may have missed something there logger.log( TRACE, "LazyStr Loading: After MD chunk decode, code page %s does not fit given bytes sequence at ALL. %s", encoding_iana, str(e), ) early_stop_count = max_chunk_gave_up lazy_str_hard_failure = True break # multi-byte bad cutting detector and adjustment # not the cleanest way to perform that fix but clever enough for now. if is_multi_byte_decoder and i > 0 and sequences[i] >= 0x80: chunk_partial_size_chk = min(chunk_size, 16) # type: int if ( decoded_payload and chunk[:chunk_partial_size_chk] not in decoded_payload ): for j in range(i, i - 4, -1): cut_sequence = sequences[j : i + chunk_size] if bom_or_sig_available and strip_sig_or_bom is False: cut_sequence = sig_payload + cut_sequence chunk = cut_sequence.decode(encoding_iana, errors="ignore") if chunk[:chunk_partial_size_chk] in decoded_payload: break md_chunks.append(chunk) md_ratios.append(mess_ratio(chunk, threshold)) if md_ratios[-1] >= threshold: early_stop_count += 1 if (early_stop_count >= max_chunk_gave_up) or ( bom_or_sig_available and strip_sig_or_bom is False ): break # We might want to check the sequence again with the whole content # Only if initial MD tests passes if ( not lazy_str_hard_failure and is_too_large_sequence and not is_multi_byte_decoder ): try: sequences[int(50e3) :].decode(encoding_iana, errors="strict") except UnicodeDecodeError as e: logger.log( TRACE, "LazyStr Loading: After final lookup, code page %s does not fit given bytes sequence at ALL. %s", encoding_iana, str(e), ) tested_but_hard_failure.append(encoding_iana) continue mean_mess_ratio = ( sum(md_ratios) / len(md_ratios) if md_ratios else 0.0 ) # type: float if mean_mess_ratio >= threshold or early_stop_count >= max_chunk_gave_up: tested_but_soft_failure.append(encoding_iana) logger.log( TRACE, "%s was excluded because of initial chaos probing. Gave up %i time(s). " "Computed mean chaos is %f %%.", encoding_iana, early_stop_count, round(mean_mess_ratio * 100, ndigits=3), ) # Preparing those fallbacks in case we got nothing. if ( encoding_iana in ["ascii", "utf_8", specified_encoding] and not lazy_str_hard_failure ): fallback_entry = CharsetMatch( sequences, encoding_iana, threshold, False, [], decoded_payload ) if encoding_iana == specified_encoding: fallback_specified = fallback_entry elif encoding_iana == "ascii": fallback_ascii = fallback_entry else: fallback_u8 = fallback_entry continue logger.log( TRACE, "%s passed initial chaos probing. Mean measured chaos is %f %%", encoding_iana, round(mean_mess_ratio * 100, ndigits=3), ) if not is_multi_byte_decoder: target_languages = encoding_languages(encoding_iana) # type: List[str] else: target_languages = mb_encoding_languages(encoding_iana) if target_languages: logger.log( TRACE, "{} should target any language(s) of {}".format( encoding_iana, str(target_languages) ), ) cd_ratios = [] # We shall skip the CD when its about ASCII # Most of the time its not relevant to run "language-detection" on it. if encoding_iana != "ascii": for chunk in md_chunks: chunk_languages = coherence_ratio( chunk, 0.1, ",".join(target_languages) if target_languages else None ) cd_ratios.append(chunk_languages) cd_ratios_merged = merge_coherence_ratios(cd_ratios) if cd_ratios_merged: logger.log( TRACE, "We detected language {} using {}".format( cd_ratios_merged, encoding_iana ), ) results.append( CharsetMatch( sequences, encoding_iana, mean_mess_ratio, bom_or_sig_available, cd_ratios_merged, decoded_payload, ) ) if ( encoding_iana in [specified_encoding, "ascii", "utf_8"] and mean_mess_ratio < 0.1 ): logger.debug( "Encoding detection: %s is most likely the one.", encoding_iana ) if explain: logger.removeHandler(explain_handler) logger.setLevel(previous_logger_level) return CharsetMatches([results[encoding_iana]]) if encoding_iana == sig_encoding: logger.debug( "Encoding detection: %s is most likely the one as we detected a BOM or SIG within " "the beginning of the sequence.", encoding_iana, ) if explain: logger.removeHandler(explain_handler) logger.setLevel(previous_logger_level) return CharsetMatches([results[encoding_iana]]) if len(results) == 0: if fallback_u8 or fallback_ascii or fallback_specified: logger.log( TRACE, "Nothing got out of the detection process. Using ASCII/UTF-8/Specified fallback.", ) if fallback_specified: logger.debug( "Encoding detection: %s will be used as a fallback match", fallback_specified.encoding, ) results.append(fallback_specified) elif ( (fallback_u8 and fallback_ascii is None) or ( fallback_u8 and fallback_ascii and fallback_u8.fingerprint != fallback_ascii.fingerprint ) or (fallback_u8 is not None) ): logger.debug("Encoding detection: utf_8 will be used as a fallback match") results.append(fallback_u8) elif fallback_ascii: logger.debug("Encoding detection: ascii will be used as a fallback match") results.append(fallback_ascii) if results: logger.debug( "Encoding detection: Found %s as plausible (best-candidate) for content. With %i alternatives.", results.best().encoding, # type: ignore len(results) - 1, ) else: logger.debug("Encoding detection: Unable to determine any suitable charset.") if explain: logger.removeHandler(explain_handler) logger.setLevel(previous_logger_level) return results def from_fp( fp: BinaryIO, steps: int = 5, chunk_size: int = 512, threshold: float = 0.20, cp_isolation: List[str] = None, cp_exclusion: List[str] = None, preemptive_behaviour: bool = True, explain: bool = False, ) -> CharsetMatches: """ Same thing than the function from_bytes but using a file pointer that is already ready. Will not close the file pointer. """ return from_bytes( fp.read(), steps, chunk_size, threshold, cp_isolation, cp_exclusion, preemptive_behaviour, explain, ) def from_path( path: PathLike, steps: int = 5, chunk_size: int = 512, threshold: float = 0.20, cp_isolation: List[str] = None, cp_exclusion: List[str] = None, preemptive_behaviour: bool = True, explain: bool = False, ) -> CharsetMatches: """ Same thing than the function from_bytes but with one extra step. Opening and reading given file path in binary mode. Can raise IOError. """ with open(path, "rb") as fp: return from_fp( fp, steps, chunk_size, threshold, cp_isolation, cp_exclusion, preemptive_behaviour, explain, ) def normalize( path: PathLike, steps: int = 5, chunk_size: int = 512, threshold: float = 0.20, cp_isolation: List[str] = None, cp_exclusion: List[str] = None, preemptive_behaviour: bool = True, ) -> CharsetMatch: """ Take a (text-based) file path and try to create another file next to it, this time using UTF-8. """ results = from_path( path, steps, chunk_size, threshold, cp_isolation, cp_exclusion, preemptive_behaviour, ) filename = basename(path) target_extensions = list(splitext(filename)) if len(results) == 0: raise IOError( 'Unable to normalize "{}", no encoding charset seems to fit.'.format( filename ) ) result = results.best() target_extensions[0] += "-" + result.encoding # type: ignore with open( "{}".format(str(path).replace(filename, "".join(target_extensions))), "wb" ) as fp: fp.write(result.output()) # type: ignore return result # type: ignore PK[]oPP version.pynu[""" Expose version """ __version__ = "2.0.12" VERSION = __version__.split(".") PK[]py.typednu[PK[]3r)cli/__pycache__/normalizer.cpython-37.pycnu[B b$@sddlZddlZddlmZddlmZddlmZddlm Z ddl m Z ddl m Z ddlmZdeeed d d Zde eed ddZedkredS)N)dumps)abspath)python_version)List)from_fp)CliDetectionResult) __version__yes)questiondefaultreturncCsdddddd}|dkrd}n(|dkr,d}n|dkr:d }n td |xRtj||t}|dk rz|d krz||S||kr||Stjd qHWdS) aAsk a yes/no question via input() and return their answer. "question" is a string that is presented to the user. "default" is the presumed answer if the user just hits . It must be "yes" (the default), "no" or None (meaning an answer is required of the user). The "answer" return value is True for "yes" or False for "no". Credit goes to (c) https://stackoverflow.com/questions/3041986/apt-command-line-interface-like-yes-no-input TF)r yZyenonNz [y/n] r z [Y/n] rz [y/N] zinvalid default answer: '%s'z3Please respond with 'yes' or 'no' (or 'y' or 'n'). ) ValueErrorsysstdoutwriteinputlower)r r ZvalidpromptchoicerR/opt/alt/python37/lib/python3.7/site-packages/charset_normalizer/cli/normalizer.py query_yes_no s   r)argvr cstjdd}|jdtdddd|jdd d d d d d|jddd d ddd|jddd d ddd|jddd d ddd|jddd d ddd|jdd d d d!d"d|jd#d$d%d&td'd(d)|jd*d+d,ttd-d.||}|j d/kr|j d krt d0t j d1d2S|jd/kr:|j d kr:t d3t j d1d2S|jd4ksR|jd5krdt d6t j d1d2Sg}x|jD]t|j|jd7}|d8krt d9j|jd5krd:nd;t j d1|ttjd8ggdjDjjjjjd8d/ t |d2kr|j!rx^|D]VkrR|ttjjjfd?d>jDjjjjjd8d qRW|j d/krj"d@d/krt dAjt j d1j#d krr$qrj%dB}|j d kr4|&dCjj#d krp$n<|jd krpt'dDjdEd krpj#d krr$qryLtdFdB(||dG_)t*|dGj)dHdIdJ}|+t,Wd8QRXWnJt-k r}z*t t,|t j d1j#d kr$dKSd8}~XYnXj#d krr$qrW|j.d krdt t/t |d2krNdLd>|Dn|dGj0d/dMdNn.x,|jD]"t dO(fdPd>|DqlWdGS)Qz CLI assistant using ARGV and ArgumentParser :param argv: :return: 0 if everything is fine, anything else equal trouble zvThe Real First Universal Charset Detector. Discover originating encoding used on text file. Normalize text to unicode.) descriptionfilesrb+zFile(s) to be analysed)typenargshelpz-vz --verbose store_trueFverbosezjDisplay complementary information about file if any. Stdout will contain logs about the detection process.)actionr destr#z-az--with-alternative alternativeszIOutput complementary possibilities if any. Top-level JSON WILL be a list.z-nz --normalize normalizezLPermit to normalize input file. If not set, program does not write anything.z-mz --minimalminimalzBOnly output the charset detected to STDOUT. Disabling JSON output.z-rz --replacereplacezGReplace file when trying to normalize it instead of creating a new one.z-fz--forceforcezHReplace file without asking if you are sure, use this flag with caution.z-tz --thresholdstoreg? thresholdzUDefine a custom maximum amount of chaos allowed in decoded content. 0. <= chaos <= 1.)r&r r!r'r#z --versionversionz!Charset-Normalizer {} - Python {}z"Show version information and exit.)r&r/r#Tz.Use --replace in addition of --normalize only.)filez*Use --force in addition of --replace only.gg?z---threshold VALUE should be between 0. AND 1.)r.ZexplainNz4Unable to identify originating encoding for "{}". {}z-Maybe try increasing maximum amount of chaos.rUnknowncsg|]}|jkr|qSr)encoding).0cp) best_guessrr szcli_detect..csg|]}|jkr|qSr)r3)r4r5)elrrr7sutfzJ"{}" file does not need to be normalized, as it already came from unicode..z0Are you sure to normalize "{}" by replacing it ?rz./{}rwzutf-8)r3cSsg|] }|jqSr)__dict__)r4r8rrrr7 s) ensure_asciiindentz, cs&g|]}|jtjkr|jp dqS)Z undefined)pathrnamer3)r4r8)my_filerrr7s)1argparseArgumentParser add_argumentFileTypefloatformatrr parse_argsr+r)printrstderrr,r.rrr%bestrCappendrrr3Zencoding_aliasesZcould_be_from_charsetlanguageZ alphabetsbomZ percent_chaosZpercent_coherencelenr( startswithclosedclosesplitinsertrjoinZ unicode_pathopenrstrIOErrorr*rr>)rparserargsZx_matchesZo_fper)r6r8rDr cli_detect.sV                    $   ra__main__)r )N)rErjsonros.pathrplatformrtypingrcharset_normalizerrZcharset_normalizer.modelsrZcharset_normalizer.versionrrZboolrintra__name__rrrrs       !tPK[] >-cli/__pycache__/__init__.cpython-37.opt-1.pycnu[B b@sdS)NrrrP/opt/alt/python37/lib/python3.7/site-packages/charset_normalizer/cli/__init__.pyPK[] >'cli/__pycache__/__init__.cpython-37.pycnu[B b@sdS)NrrrP/opt/alt/python37/lib/python3.7/site-packages/charset_normalizer/cli/__init__.pyPK[]O/cli/__pycache__/normalizer.cpython-37.opt-1.pycnu[B b$@sddlZddlZddlmZddlmZddlmZddlm Z ddl m Z ddl m Z ddlmZdeeed d d Zde eed ddZedkredS)N)dumps)abspath)python_version)List)from_fp)CliDetectionResult) __version__yes)questiondefaultreturncCsdddddd}|dkrd}n(|dkr,d}n|dkr:d }n td |xRtj||t}|dk rz|d krz||S||kr||Stjd qHWdS) aAsk a yes/no question via input() and return their answer. "question" is a string that is presented to the user. "default" is the presumed answer if the user just hits . It must be "yes" (the default), "no" or None (meaning an answer is required of the user). The "answer" return value is True for "yes" or False for "no". Credit goes to (c) https://stackoverflow.com/questions/3041986/apt-command-line-interface-like-yes-no-input TF)r yZyenonNz [y/n] r z [Y/n] rz [y/N] zinvalid default answer: '%s'z3Please respond with 'yes' or 'no' (or 'y' or 'n'). ) ValueErrorsysstdoutwriteinputlower)r r ZvalidpromptZchoicerR/opt/alt/python37/lib/python3.7/site-packages/charset_normalizer/cli/normalizer.py query_yes_no s   r)argvr cstjdd}|jdtdddd|jdd d d d d d|jddd d ddd|jddd d ddd|jddd d ddd|jddd d ddd|jdd d d d!d"d|jd#d$d%d&td'd(d)|jd*d+d,ttd-d.||}|j d/kr|j d krt d0t j d1d2S|jd/kr:|j d kr:t d3t j d1d2S|jd4ksR|jd5krdt d6t j d1d2Sg}x|jD]t|j|jd7}|d8krt d9j|jd5krd:nd;t j d1|ttjd8ggdjDjjjjjd8d/ t |d2kr|j!rx^|D]VkrR|ttjjjfd?d>jDjjjjjd8d qRW|j d/krj"d@d/krt dAjt j d1j#d krr$qrj%dB}|j d kr4|&dCjj#d krp$n<|jd krpt'dDjdEd krpj#d krr$qryLtdFdB(||dG_)t*|dGj)dHdIdJ}|+t,Wd8QRXWnJt-k r}z*t t,|t j d1j#d kr$dKSd8}~XYnXj#d krr$qrW|j.d krdt t/t |d2krNdLd>|Dn|dGj0d/dMdNn.x,|jD]"t dO(fdPd>|DqlWdGS)Qz CLI assistant using ARGV and ArgumentParser :param argv: :return: 0 if everything is fine, anything else equal trouble zvThe Real First Universal Charset Detector. Discover originating encoding used on text file. Normalize text to unicode.)Z descriptionfilesrb+zFile(s) to be analysed)typenargshelpz-vz --verbose store_trueFverbosezjDisplay complementary information about file if any. Stdout will contain logs about the detection process.)actionr destr!z-az--with-alternative alternativeszIOutput complementary possibilities if any. Top-level JSON WILL be a list.z-nz --normalize normalizezLPermit to normalize input file. If not set, program does not write anything.z-mz --minimalminimalzBOnly output the charset detected to STDOUT. Disabling JSON output.z-rz --replacereplacezGReplace file when trying to normalize it instead of creating a new one.z-fz--forceforcezHReplace file without asking if you are sure, use this flag with caution.z-tz --thresholdZstoreg? thresholdzUDefine a custom maximum amount of chaos allowed in decoded content. 0. <= chaos <= 1.)r$r rr%r!z --versionversionz!Charset-Normalizer {} - Python {}z"Show version information and exit.)r$r,r!Tz.Use --replace in addition of --normalize only.)filez*Use --force in addition of --replace only.gg?z---threshold VALUE should be between 0. AND 1.)r+ZexplainNz4Unable to identify originating encoding for "{}". {}z-Maybe try increasing maximum amount of chaos.rZUnknowncsg|]}|jkr|qSr)encoding).0cp) best_guessrr szcli_detect..csg|]}|jkr|qSr)r/)r0r1)elrrr3sutfzJ"{}" file does not need to be normalized, as it already came from unicode..z0Are you sure to normalize "{}" by replacing it ?rz./{}rwzutf-8)r/cSsg|] }|jqSr)__dict__)r0r4rrrr3 s)Z ensure_asciiindentz, cs&g|]}|jtjkr|jp dqS)Z undefined)pathrnamer/)r0r4)my_filerrr3s)1argparseArgumentParser add_argumentZFileTypefloatformatrr parse_argsr)r'printrstderrr*r+rrr#Zbestr>appendrrr/Zencoding_aliasesZcould_be_from_charsetZlanguageZ alphabetsZbomZ percent_chaosZpercent_coherencelenr& startswithclosedclosesplitinsertrjoinZ unicode_pathopenrstrIOErrorr(rr:)rparserargsZx_ZmatchesZo_fper)r2r4r?r cli_detect.sV                    $   rW__main__)r )N)r@rZjsonrZos.pathrplatformrtypingrZcharset_normalizerrZcharset_normalizer.modelsrZcharset_normalizer.versionrrQboolrintrW__name__rrrrs       !tPK[]{J$$cli/normalizer.pynu[import argparse import sys from json import dumps from os.path import abspath from platform import python_version from typing import List from charset_normalizer import from_fp from charset_normalizer.models import CliDetectionResult from charset_normalizer.version import __version__ def query_yes_no(question: str, default: str = "yes") -> bool: """Ask a yes/no question via input() and return their answer. "question" is a string that is presented to the user. "default" is the presumed answer if the user just hits . It must be "yes" (the default), "no" or None (meaning an answer is required of the user). The "answer" return value is True for "yes" or False for "no". Credit goes to (c) https://stackoverflow.com/questions/3041986/apt-command-line-interface-like-yes-no-input """ valid = {"yes": True, "y": True, "ye": True, "no": False, "n": False} if default is None: prompt = " [y/n] " elif default == "yes": prompt = " [Y/n] " elif default == "no": prompt = " [y/N] " else: raise ValueError("invalid default answer: '%s'" % default) while True: sys.stdout.write(question + prompt) choice = input().lower() if default is not None and choice == "": return valid[default] elif choice in valid: return valid[choice] else: sys.stdout.write("Please respond with 'yes' or 'no' " "(or 'y' or 'n').\n") def cli_detect(argv: List[str] = None) -> int: """ CLI assistant using ARGV and ArgumentParser :param argv: :return: 0 if everything is fine, anything else equal trouble """ parser = argparse.ArgumentParser( description="The Real First Universal Charset Detector. " "Discover originating encoding used on text file. " "Normalize text to unicode." ) parser.add_argument( "files", type=argparse.FileType("rb"), nargs="+", help="File(s) to be analysed" ) parser.add_argument( "-v", "--verbose", action="store_true", default=False, dest="verbose", help="Display complementary information about file if any. " "Stdout will contain logs about the detection process.", ) parser.add_argument( "-a", "--with-alternative", action="store_true", default=False, dest="alternatives", help="Output complementary possibilities if any. Top-level JSON WILL be a list.", ) parser.add_argument( "-n", "--normalize", action="store_true", default=False, dest="normalize", help="Permit to normalize input file. If not set, program does not write anything.", ) parser.add_argument( "-m", "--minimal", action="store_true", default=False, dest="minimal", help="Only output the charset detected to STDOUT. Disabling JSON output.", ) parser.add_argument( "-r", "--replace", action="store_true", default=False, dest="replace", help="Replace file when trying to normalize it instead of creating a new one.", ) parser.add_argument( "-f", "--force", action="store_true", default=False, dest="force", help="Replace file without asking if you are sure, use this flag with caution.", ) parser.add_argument( "-t", "--threshold", action="store", default=0.1, type=float, dest="threshold", help="Define a custom maximum amount of chaos allowed in decoded content. 0. <= chaos <= 1.", ) parser.add_argument( "--version", action="version", version="Charset-Normalizer {} - Python {}".format( __version__, python_version() ), help="Show version information and exit.", ) args = parser.parse_args(argv) if args.replace is True and args.normalize is False: print("Use --replace in addition of --normalize only.", file=sys.stderr) return 1 if args.force is True and args.replace is False: print("Use --force in addition of --replace only.", file=sys.stderr) return 1 if args.threshold < 0.0 or args.threshold > 1.0: print("--threshold VALUE should be between 0. AND 1.", file=sys.stderr) return 1 x_ = [] for my_file in args.files: matches = from_fp(my_file, threshold=args.threshold, explain=args.verbose) best_guess = matches.best() if best_guess is None: print( 'Unable to identify originating encoding for "{}". {}'.format( my_file.name, "Maybe try increasing maximum amount of chaos." if args.threshold < 1.0 else "", ), file=sys.stderr, ) x_.append( CliDetectionResult( abspath(my_file.name), None, [], [], "Unknown", [], False, 1.0, 0.0, None, True, ) ) else: x_.append( CliDetectionResult( abspath(my_file.name), best_guess.encoding, best_guess.encoding_aliases, [ cp for cp in best_guess.could_be_from_charset if cp != best_guess.encoding ], best_guess.language, best_guess.alphabets, best_guess.bom, best_guess.percent_chaos, best_guess.percent_coherence, None, True, ) ) if len(matches) > 1 and args.alternatives: for el in matches: if el != best_guess: x_.append( CliDetectionResult( abspath(my_file.name), el.encoding, el.encoding_aliases, [ cp for cp in el.could_be_from_charset if cp != el.encoding ], el.language, el.alphabets, el.bom, el.percent_chaos, el.percent_coherence, None, False, ) ) if args.normalize is True: if best_guess.encoding.startswith("utf") is True: print( '"{}" file does not need to be normalized, as it already came from unicode.'.format( my_file.name ), file=sys.stderr, ) if my_file.closed is False: my_file.close() continue o_ = my_file.name.split(".") # type: List[str] if args.replace is False: o_.insert(-1, best_guess.encoding) if my_file.closed is False: my_file.close() elif ( args.force is False and query_yes_no( 'Are you sure to normalize "{}" by replacing it ?'.format( my_file.name ), "no", ) is False ): if my_file.closed is False: my_file.close() continue try: x_[0].unicode_path = abspath("./{}".format(".".join(o_))) with open(x_[0].unicode_path, "w", encoding="utf-8") as fp: fp.write(str(best_guess)) except IOError as e: print(str(e), file=sys.stderr) if my_file.closed is False: my_file.close() return 2 if my_file.closed is False: my_file.close() if args.minimal is False: print( dumps( [el.__dict__ for el in x_] if len(x_) > 1 else x_[0].__dict__, ensure_ascii=True, indent=4, ) ) else: for my_file in args.files: print( ", ".join( [ el.encoding or "undefined" for el in x_ if el.path == abspath(my_file.name) ] ) ) return 0 if __name__ == "__main__": cli_detect() PK[]cli/__init__.pynu[PK[]GGmd.pynu[from functools import lru_cache from typing import List, Optional from .constant import COMMON_SAFE_ASCII_CHARACTERS, UNICODE_SECONDARY_RANGE_KEYWORD from .utils import ( is_accentuated, is_ascii, is_case_variable, is_cjk, is_emoticon, is_hangul, is_hiragana, is_katakana, is_latin, is_punctuation, is_separator, is_symbol, is_thai, remove_accent, unicode_range, ) class MessDetectorPlugin: """ Base abstract class used for mess detection plugins. All detectors MUST extend and implement given methods. """ def eligible(self, character: str) -> bool: """ Determine if given character should be fed in. """ raise NotImplementedError # pragma: nocover def feed(self, character: str) -> None: """ The main routine to be executed upon character. Insert the logic in witch the text would be considered chaotic. """ raise NotImplementedError # pragma: nocover def reset(self) -> None: # pragma: no cover """ Permit to reset the plugin to the initial state. """ raise NotImplementedError @property def ratio(self) -> float: """ Compute the chaos ratio based on what your feed() has seen. Must NOT be lower than 0.; No restriction gt 0. """ raise NotImplementedError # pragma: nocover class TooManySymbolOrPunctuationPlugin(MessDetectorPlugin): def __init__(self) -> None: self._punctuation_count = 0 # type: int self._symbol_count = 0 # type: int self._character_count = 0 # type: int self._last_printable_char = None # type: Optional[str] self._frenzy_symbol_in_word = False # type: bool def eligible(self, character: str) -> bool: return character.isprintable() def feed(self, character: str) -> None: self._character_count += 1 if ( character != self._last_printable_char and character not in COMMON_SAFE_ASCII_CHARACTERS ): if is_punctuation(character): self._punctuation_count += 1 elif ( character.isdigit() is False and is_symbol(character) and is_emoticon(character) is False ): self._symbol_count += 2 self._last_printable_char = character def reset(self) -> None: # pragma: no cover self._punctuation_count = 0 self._character_count = 0 self._symbol_count = 0 @property def ratio(self) -> float: if self._character_count == 0: return 0.0 ratio_of_punctuation = ( self._punctuation_count + self._symbol_count ) / self._character_count # type: float return ratio_of_punctuation if ratio_of_punctuation >= 0.3 else 0.0 class TooManyAccentuatedPlugin(MessDetectorPlugin): def __init__(self) -> None: self._character_count = 0 # type: int self._accentuated_count = 0 # type: int def eligible(self, character: str) -> bool: return character.isalpha() def feed(self, character: str) -> None: self._character_count += 1 if is_accentuated(character): self._accentuated_count += 1 def reset(self) -> None: # pragma: no cover self._character_count = 0 self._accentuated_count = 0 @property def ratio(self) -> float: if self._character_count == 0: return 0.0 ratio_of_accentuation = ( self._accentuated_count / self._character_count ) # type: float return ratio_of_accentuation if ratio_of_accentuation >= 0.35 else 0.0 class UnprintablePlugin(MessDetectorPlugin): def __init__(self) -> None: self._unprintable_count = 0 # type: int self._character_count = 0 # type: int def eligible(self, character: str) -> bool: return True def feed(self, character: str) -> None: if ( character.isspace() is False # includes \n \t \r \v and character.isprintable() is False and character != "\x1A" # Why? Its the ASCII substitute character. ): self._unprintable_count += 1 self._character_count += 1 def reset(self) -> None: # pragma: no cover self._unprintable_count = 0 @property def ratio(self) -> float: if self._character_count == 0: return 0.0 return (self._unprintable_count * 8) / self._character_count class SuspiciousDuplicateAccentPlugin(MessDetectorPlugin): def __init__(self) -> None: self._successive_count = 0 # type: int self._character_count = 0 # type: int self._last_latin_character = None # type: Optional[str] def eligible(self, character: str) -> bool: return character.isalpha() and is_latin(character) def feed(self, character: str) -> None: self._character_count += 1 if ( self._last_latin_character is not None and is_accentuated(character) and is_accentuated(self._last_latin_character) ): if character.isupper() and self._last_latin_character.isupper(): self._successive_count += 1 # Worse if its the same char duplicated with different accent. if remove_accent(character) == remove_accent(self._last_latin_character): self._successive_count += 1 self._last_latin_character = character def reset(self) -> None: # pragma: no cover self._successive_count = 0 self._character_count = 0 self._last_latin_character = None @property def ratio(self) -> float: if self._character_count == 0: return 0.0 return (self._successive_count * 2) / self._character_count class SuspiciousRange(MessDetectorPlugin): def __init__(self) -> None: self._suspicious_successive_range_count = 0 # type: int self._character_count = 0 # type: int self._last_printable_seen = None # type: Optional[str] def eligible(self, character: str) -> bool: return character.isprintable() def feed(self, character: str) -> None: self._character_count += 1 if ( character.isspace() or is_punctuation(character) or character in COMMON_SAFE_ASCII_CHARACTERS ): self._last_printable_seen = None return if self._last_printable_seen is None: self._last_printable_seen = character return unicode_range_a = unicode_range( self._last_printable_seen ) # type: Optional[str] unicode_range_b = unicode_range(character) # type: Optional[str] if is_suspiciously_successive_range(unicode_range_a, unicode_range_b): self._suspicious_successive_range_count += 1 self._last_printable_seen = character def reset(self) -> None: # pragma: no cover self._character_count = 0 self._suspicious_successive_range_count = 0 self._last_printable_seen = None @property def ratio(self) -> float: if self._character_count == 0: return 0.0 ratio_of_suspicious_range_usage = ( self._suspicious_successive_range_count * 2 ) / self._character_count # type: float if ratio_of_suspicious_range_usage < 0.1: return 0.0 return ratio_of_suspicious_range_usage class SuperWeirdWordPlugin(MessDetectorPlugin): def __init__(self) -> None: self._word_count = 0 # type: int self._bad_word_count = 0 # type: int self._foreign_long_count = 0 # type: int self._is_current_word_bad = False # type: bool self._foreign_long_watch = False # type: bool self._character_count = 0 # type: int self._bad_character_count = 0 # type: int self._buffer = "" # type: str self._buffer_accent_count = 0 # type: int def eligible(self, character: str) -> bool: return True def feed(self, character: str) -> None: if character.isalpha(): self._buffer = "".join([self._buffer, character]) if is_accentuated(character): self._buffer_accent_count += 1 if ( self._foreign_long_watch is False and (is_latin(character) is False or is_accentuated(character)) and is_cjk(character) is False and is_hangul(character) is False and is_katakana(character) is False and is_hiragana(character) is False and is_thai(character) is False ): self._foreign_long_watch = True return if not self._buffer: return if ( character.isspace() or is_punctuation(character) or is_separator(character) ) and self._buffer: self._word_count += 1 buffer_length = len(self._buffer) # type: int self._character_count += buffer_length if buffer_length >= 4: if self._buffer_accent_count / buffer_length > 0.34: self._is_current_word_bad = True # Word/Buffer ending with a upper case accentuated letter are so rare, # that we will consider them all as suspicious. Same weight as foreign_long suspicious. if is_accentuated(self._buffer[-1]) and self._buffer[-1].isupper(): self._foreign_long_count += 1 self._is_current_word_bad = True if buffer_length >= 24 and self._foreign_long_watch: self._foreign_long_count += 1 self._is_current_word_bad = True if self._is_current_word_bad: self._bad_word_count += 1 self._bad_character_count += len(self._buffer) self._is_current_word_bad = False self._foreign_long_watch = False self._buffer = "" self._buffer_accent_count = 0 elif ( character not in {"<", ">", "-", "=", "~", "|", "_"} and character.isdigit() is False and is_symbol(character) ): self._is_current_word_bad = True self._buffer += character def reset(self) -> None: # pragma: no cover self._buffer = "" self._is_current_word_bad = False self._foreign_long_watch = False self._bad_word_count = 0 self._word_count = 0 self._character_count = 0 self._bad_character_count = 0 self._foreign_long_count = 0 @property def ratio(self) -> float: if self._word_count <= 10 and self._foreign_long_count == 0: return 0.0 return self._bad_character_count / self._character_count class CjkInvalidStopPlugin(MessDetectorPlugin): """ GB(Chinese) based encoding often render the stop incorrectly when the content does not fit and can be easily detected. Searching for the overuse of '丅' and '丄'. """ def __init__(self) -> None: self._wrong_stop_count = 0 # type: int self._cjk_character_count = 0 # type: int def eligible(self, character: str) -> bool: return True def feed(self, character: str) -> None: if character in {"丅", "丄"}: self._wrong_stop_count += 1 return if is_cjk(character): self._cjk_character_count += 1 def reset(self) -> None: # pragma: no cover self._wrong_stop_count = 0 self._cjk_character_count = 0 @property def ratio(self) -> float: if self._cjk_character_count < 16: return 0.0 return self._wrong_stop_count / self._cjk_character_count class ArchaicUpperLowerPlugin(MessDetectorPlugin): def __init__(self) -> None: self._buf = False # type: bool self._character_count_since_last_sep = 0 # type: int self._successive_upper_lower_count = 0 # type: int self._successive_upper_lower_count_final = 0 # type: int self._character_count = 0 # type: int self._last_alpha_seen = None # type: Optional[str] self._current_ascii_only = True # type: bool def eligible(self, character: str) -> bool: return True def feed(self, character: str) -> None: is_concerned = character.isalpha() and is_case_variable(character) chunk_sep = is_concerned is False if chunk_sep and self._character_count_since_last_sep > 0: if ( self._character_count_since_last_sep <= 64 and character.isdigit() is False and self._current_ascii_only is False ): self._successive_upper_lower_count_final += ( self._successive_upper_lower_count ) self._successive_upper_lower_count = 0 self._character_count_since_last_sep = 0 self._last_alpha_seen = None self._buf = False self._character_count += 1 self._current_ascii_only = True return if self._current_ascii_only is True and is_ascii(character) is False: self._current_ascii_only = False if self._last_alpha_seen is not None: if (character.isupper() and self._last_alpha_seen.islower()) or ( character.islower() and self._last_alpha_seen.isupper() ): if self._buf is True: self._successive_upper_lower_count += 2 self._buf = False else: self._buf = True else: self._buf = False self._character_count += 1 self._character_count_since_last_sep += 1 self._last_alpha_seen = character def reset(self) -> None: # pragma: no cover self._character_count = 0 self._character_count_since_last_sep = 0 self._successive_upper_lower_count = 0 self._successive_upper_lower_count_final = 0 self._last_alpha_seen = None self._buf = False self._current_ascii_only = True @property def ratio(self) -> float: if self._character_count == 0: return 0.0 return self._successive_upper_lower_count_final / self._character_count def is_suspiciously_successive_range( unicode_range_a: Optional[str], unicode_range_b: Optional[str] ) -> bool: """ Determine if two Unicode range seen next to each other can be considered as suspicious. """ if unicode_range_a is None or unicode_range_b is None: return True if unicode_range_a == unicode_range_b: return False if "Latin" in unicode_range_a and "Latin" in unicode_range_b: return False if "Emoticons" in unicode_range_a or "Emoticons" in unicode_range_b: return False # Latin characters can be accompanied with a combining diacritical mark # eg. Vietnamese. if ("Latin" in unicode_range_a or "Latin" in unicode_range_b) and ( "Combining" in unicode_range_a or "Combining" in unicode_range_b ): return False keywords_range_a, keywords_range_b = unicode_range_a.split( " " ), unicode_range_b.split(" ") for el in keywords_range_a: if el in UNICODE_SECONDARY_RANGE_KEYWORD: continue if el in keywords_range_b: return False # Japanese Exception range_a_jp_chars, range_b_jp_chars = ( unicode_range_a in ( "Hiragana", "Katakana", ), unicode_range_b in ("Hiragana", "Katakana"), ) if (range_a_jp_chars or range_b_jp_chars) and ( "CJK" in unicode_range_a or "CJK" in unicode_range_b ): return False if range_a_jp_chars and range_b_jp_chars: return False if "Hangul" in unicode_range_a or "Hangul" in unicode_range_b: if "CJK" in unicode_range_a or "CJK" in unicode_range_b: return False if unicode_range_a == "Basic Latin" or unicode_range_b == "Basic Latin": return False # Chinese/Japanese use dedicated range for punctuation and/or separators. if ("CJK" in unicode_range_a or "CJK" in unicode_range_b) or ( unicode_range_a in ["Katakana", "Hiragana"] and unicode_range_b in ["Katakana", "Hiragana"] ): if "Punctuation" in unicode_range_a or "Punctuation" in unicode_range_b: return False if "Forms" in unicode_range_a or "Forms" in unicode_range_b: return False return True @lru_cache(maxsize=2048) def mess_ratio( decoded_sequence: str, maximum_threshold: float = 0.2, debug: bool = False ) -> float: """ Compute a mess ratio given a decoded bytes sequence. The maximum threshold does stop the computation earlier. """ detectors = [ md_class() for md_class in MessDetectorPlugin.__subclasses__() ] # type: List[MessDetectorPlugin] length = len(decoded_sequence) + 1 # type: int mean_mess_ratio = 0.0 # type: float if length < 512: intermediary_mean_mess_ratio_calc = 32 # type: int elif length <= 1024: intermediary_mean_mess_ratio_calc = 64 else: intermediary_mean_mess_ratio_calc = 128 for character, index in zip(decoded_sequence + "\n", range(length)): for detector in detectors: if detector.eligible(character): detector.feed(character) if ( index > 0 and index % intermediary_mean_mess_ratio_calc == 0 ) or index == length - 1: mean_mess_ratio = sum(dt.ratio for dt in detectors) if mean_mess_ratio >= maximum_threshold: break if debug: for dt in detectors: # pragma: nocover print(dt.__class__, dt.ratio) return round(mean_mess_ratio, 3) PK[]QҚ\$\$utils.pynu[try: import unicodedata2 as unicodedata except ImportError: import unicodedata # type: ignore[no-redef] import importlib import logging from codecs import IncrementalDecoder from encodings.aliases import aliases from functools import lru_cache from re import findall from typing import List, Optional, Set, Tuple, Union from _multibytecodec import MultibyteIncrementalDecoder # type: ignore from .constant import ( ENCODING_MARKS, IANA_SUPPORTED_SIMILAR, RE_POSSIBLE_ENCODING_INDICATION, UNICODE_RANGES_COMBINED, UNICODE_SECONDARY_RANGE_KEYWORD, UTF8_MAXIMAL_ALLOCATION, ) @lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION) def is_accentuated(character: str) -> bool: try: description = unicodedata.name(character) # type: str except ValueError: return False return ( "WITH GRAVE" in description or "WITH ACUTE" in description or "WITH CEDILLA" in description or "WITH DIAERESIS" in description or "WITH CIRCUMFLEX" in description or "WITH TILDE" in description ) @lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION) def remove_accent(character: str) -> str: decomposed = unicodedata.decomposition(character) # type: str if not decomposed: return character codes = decomposed.split(" ") # type: List[str] return chr(int(codes[0], 16)) @lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION) def unicode_range(character: str) -> Optional[str]: """ Retrieve the Unicode range official name from a single character. """ character_ord = ord(character) # type: int for range_name, ord_range in UNICODE_RANGES_COMBINED.items(): if character_ord in ord_range: return range_name return None @lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION) def is_latin(character: str) -> bool: try: description = unicodedata.name(character) # type: str except ValueError: return False return "LATIN" in description def is_ascii(character: str) -> bool: try: character.encode("ascii") except UnicodeEncodeError: return False return True @lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION) def is_punctuation(character: str) -> bool: character_category = unicodedata.category(character) # type: str if "P" in character_category: return True character_range = unicode_range(character) # type: Optional[str] if character_range is None: return False return "Punctuation" in character_range @lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION) def is_symbol(character: str) -> bool: character_category = unicodedata.category(character) # type: str if "S" in character_category or "N" in character_category: return True character_range = unicode_range(character) # type: Optional[str] if character_range is None: return False return "Forms" in character_range @lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION) def is_emoticon(character: str) -> bool: character_range = unicode_range(character) # type: Optional[str] if character_range is None: return False return "Emoticons" in character_range @lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION) def is_separator(character: str) -> bool: if character.isspace() or character in {"｜", "+", ",", ";", "<", ">"}: return True character_category = unicodedata.category(character) # type: str return "Z" in character_category @lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION) def is_case_variable(character: str) -> bool: return character.islower() != character.isupper() def is_private_use_only(character: str) -> bool: character_category = unicodedata.category(character) # type: str return character_category == "Co" @lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION) def is_cjk(character: str) -> bool: try: character_name = unicodedata.name(character) except ValueError: return False return "CJK" in character_name @lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION) def is_hiragana(character: str) -> bool: try: character_name = unicodedata.name(character) except ValueError: return False return "HIRAGANA" in character_name @lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION) def is_katakana(character: str) -> bool: try: character_name = unicodedata.name(character) except ValueError: return False return "KATAKANA" in character_name @lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION) def is_hangul(character: str) -> bool: try: character_name = unicodedata.name(character) except ValueError: return False return "HANGUL" in character_name @lru_cache(maxsize=UTF8_MAXIMAL_ALLOCATION) def is_thai(character: str) -> bool: try: character_name = unicodedata.name(character) except ValueError: return False return "THAI" in character_name @lru_cache(maxsize=len(UNICODE_RANGES_COMBINED)) def is_unicode_range_secondary(range_name: str) -> bool: return any(keyword in range_name for keyword in UNICODE_SECONDARY_RANGE_KEYWORD) def any_specified_encoding(sequence: bytes, search_zone: int = 4096) -> Optional[str]: """ Extract using ASCII-only decoder any specified encoding in the first n-bytes. """ if not isinstance(sequence, bytes): raise TypeError seq_len = len(sequence) # type: int results = findall( RE_POSSIBLE_ENCODING_INDICATION, sequence[: min(seq_len, search_zone)].decode("ascii", errors="ignore"), ) # type: List[str] if len(results) == 0: return None for specified_encoding in results: specified_encoding = specified_encoding.lower().replace("-", "_") for encoding_alias, encoding_iana in aliases.items(): if encoding_alias == specified_encoding: return encoding_iana if encoding_iana == specified_encoding: return encoding_iana return None @lru_cache(maxsize=128) def is_multi_byte_encoding(name: str) -> bool: """ Verify is a specific encoding is a multi byte one based on it IANA name """ return name in { "utf_8", "utf_8_sig", "utf_16", "utf_16_be", "utf_16_le", "utf_32", "utf_32_le", "utf_32_be", "utf_7", } or issubclass( importlib.import_module("encodings.{}".format(name)).IncrementalDecoder, # type: ignore MultibyteIncrementalDecoder, ) def identify_sig_or_bom(sequence: bytes) -> Tuple[Optional[str], bytes]: """ Identify and extract SIG/BOM in given sequence. """ for iana_encoding in ENCODING_MARKS: marks = ENCODING_MARKS[iana_encoding] # type: Union[bytes, List[bytes]] if isinstance(marks, bytes): marks = [marks] for mark in marks: if sequence.startswith(mark): return iana_encoding, mark return None, b"" def should_strip_sig_or_bom(iana_encoding: str) -> bool: return iana_encoding not in {"utf_16", "utf_32"} def iana_name(cp_name: str, strict: bool = True) -> str: cp_name = cp_name.lower().replace("-", "_") for encoding_alias, encoding_iana in aliases.items(): if cp_name in [encoding_alias, encoding_iana]: return encoding_iana if strict: raise ValueError("Unable to retrieve IANA for '{}'".format(cp_name)) return cp_name def range_scan(decoded_sequence: str) -> List[str]: ranges = set() # type: Set[str] for character in decoded_sequence: character_range = unicode_range(character) # type: Optional[str] if character_range is None: continue ranges.add(character_range) return list(ranges) def cp_similarity(iana_name_a: str, iana_name_b: str) -> float: if is_multi_byte_encoding(iana_name_a) or is_multi_byte_encoding(iana_name_b): return 0.0 decoder_a = importlib.import_module("encodings.{}".format(iana_name_a)).IncrementalDecoder # type: ignore decoder_b = importlib.import_module("encodings.{}".format(iana_name_b)).IncrementalDecoder # type: ignore id_a = decoder_a(errors="ignore") # type: IncrementalDecoder id_b = decoder_b(errors="ignore") # type: IncrementalDecoder character_match_count = 0 # type: int for i in range(255): to_be_decoded = bytes([i]) # type: bytes if id_a.decode(to_be_decoded) == id_b.decode(to_be_decoded): character_match_count += 1 return character_match_count / 254 def is_cp_similar(iana_name_a: str, iana_name_b: str) -> bool: """ Determine if two code page are at least 80% similar. IANA_SUPPORTED_SIMILAR dict was generated using the function cp_similarity. """ return ( iana_name_a in IANA_SUPPORTED_SIMILAR and iana_name_b in IANA_SUPPORTED_SIMILAR[iana_name_a] ) def set_logging_handler( name: str = "charset_normalizer", level: int = logging.INFO, format_string: str = "%(asctime)s | %(levelname)s | %(message)s", ) -> None: logger = logging.getLogger(name) logger.setLevel(level) handler = logging.StreamHandler() handler.setFormatter(logging.Formatter(format_string)) logger.addHandler(handler) PK[] __init__.pynu[# -*- coding: utf_8 -*- """ Charset-Normalizer ~~~~~~~~~~~~~~ The Real First Universal Charset Detector. A library that helps you read text from an unknown charset encoding. Motivated by chardet, This package is trying to resolve the issue by taking a new approach. All IANA character set names for which the Python core library provides codecs are supported. Basic usage: >>> from charset_normalizer import from_bytes >>> results = from_bytes('Bсеки човек има право на образование. Oбразованието!'.encode('utf_8')) >>> best_guess = results.best() >>> str(best_guess) 'Bсеки човек има право на образование. Oбразованието!' Others methods and usages are available - see the full documentation at . :copyright: (c) 2021 by Ahmed TAHRI :license: MIT, see LICENSE for more details. """ import logging from .api import from_bytes, from_fp, from_path, normalize from .legacy import ( CharsetDetector, CharsetDoctor, CharsetNormalizerMatch, CharsetNormalizerMatches, detect, ) from .models import CharsetMatch, CharsetMatches from .utils import set_logging_handler from .version import VERSION, __version__ __all__ = ( "from_fp", "from_path", "from_bytes", "normalize", "detect", "CharsetMatch", "CharsetMatches", "CharsetNormalizerMatch", "CharsetNormalizerMatches", "CharsetDetector", "CharsetDoctor", "__version__", "VERSION", "set_logging_handler", ) # Attach a NullHandler to the top level logger by default # https://docs.python.org/3.3/howto/logging.html#configuring-logging-for-a-library logging.getLogger("charset_normalizer").addHandler(logging.NullHandler()) PK[]FsKK constant.pynu[from codecs import BOM_UTF8, BOM_UTF16_BE, BOM_UTF16_LE, BOM_UTF32_BE, BOM_UTF32_LE from collections import OrderedDict from encodings.aliases import aliases from re import IGNORECASE, compile as re_compile from typing import Dict, List, Set, Union from .assets import FREQUENCIES # Contain for each eligible encoding a list of/item bytes SIG/BOM ENCODING_MARKS = OrderedDict( [ ("utf_8", BOM_UTF8), ( "utf_7", [ b"\x2b\x2f\x76\x38", b"\x2b\x2f\x76\x39", b"\x2b\x2f\x76\x2b", b"\x2b\x2f\x76\x2f", b"\x2b\x2f\x76\x38\x2d", ], ), ("gb18030", b"\x84\x31\x95\x33"), ("utf_32", [BOM_UTF32_BE, BOM_UTF32_LE]), ("utf_16", [BOM_UTF16_BE, BOM_UTF16_LE]), ] ) # type: Dict[str, Union[bytes, List[bytes]]] TOO_SMALL_SEQUENCE = 32 # type: int TOO_BIG_SEQUENCE = int(10e6) # type: int UTF8_MAXIMAL_ALLOCATION = 1112064 # type: int UNICODE_RANGES_COMBINED = { "Control character": range(31 + 1), "Basic Latin": range(32, 127 + 1), "Latin-1 Supplement": range(128, 255 + 1), "Latin Extended-A": range(256, 383 + 1), "Latin Extended-B": range(384, 591 + 1), "IPA Extensions": range(592, 687 + 1), "Spacing Modifier Letters": range(688, 767 + 1), "Combining Diacritical Marks": range(768, 879 + 1), "Greek and Coptic": range(880, 1023 + 1), "Cyrillic": range(1024, 1279 + 1), "Cyrillic Supplement": range(1280, 1327 + 1), "Armenian": range(1328, 1423 + 1), "Hebrew": range(1424, 1535 + 1), "Arabic": range(1536, 1791 + 1), "Syriac": range(1792, 1871 + 1), "Arabic Supplement": range(1872, 1919 + 1), "Thaana": range(1920, 1983 + 1), "NKo": range(1984, 2047 + 1), "Samaritan": range(2048, 2111 + 1), "Mandaic": range(2112, 2143 + 1), "Syriac Supplement": range(2144, 2159 + 1), "Arabic Extended-A": range(2208, 2303 + 1), "Devanagari": range(2304, 2431 + 1), "Bengali": range(2432, 2559 + 1), "Gurmukhi": range(2560, 2687 + 1), "Gujarati": range(2688, 2815 + 1), "Oriya": range(2816, 2943 + 1), "Tamil": range(2944, 3071 + 1), "Telugu": range(3072, 3199 + 1), "Kannada": range(3200, 3327 + 1), "Malayalam": range(3328, 3455 + 1), "Sinhala": range(3456, 3583 + 1), "Thai": range(3584, 3711 + 1), "Lao": range(3712, 3839 + 1), "Tibetan": range(3840, 4095 + 1), "Myanmar": range(4096, 4255 + 1), "Georgian": range(4256, 4351 + 1), "Hangul Jamo": range(4352, 4607 + 1), "Ethiopic": range(4608, 4991 + 1), "Ethiopic Supplement": range(4992, 5023 + 1), "Cherokee": range(5024, 5119 + 1), "Unified Canadian Aboriginal Syllabics": range(5120, 5759 + 1), "Ogham": range(5760, 5791 + 1), "Runic": range(5792, 5887 + 1), "Tagalog": range(5888, 5919 + 1), "Hanunoo": range(5920, 5951 + 1), "Buhid": range(5952, 5983 + 1), "Tagbanwa": range(5984, 6015 + 1), "Khmer": range(6016, 6143 + 1), "Mongolian": range(6144, 6319 + 1), "Unified Canadian Aboriginal Syllabics Extended": range(6320, 6399 + 1), "Limbu": range(6400, 6479 + 1), "Tai Le": range(6480, 6527 + 1), "New Tai Lue": range(6528, 6623 + 1), "Khmer Symbols": range(6624, 6655 + 1), "Buginese": range(6656, 6687 + 1), "Tai Tham": range(6688, 6831 + 1), "Combining Diacritical Marks Extended": range(6832, 6911 + 1), "Balinese": range(6912, 7039 + 1), "Sundanese": range(7040, 7103 + 1), "Batak": range(7104, 7167 + 1), "Lepcha": range(7168, 7247 + 1), "Ol Chiki": range(7248, 7295 + 1), "Cyrillic Extended C": range(7296, 7311 + 1), "Sundanese Supplement": range(7360, 7375 + 1), "Vedic Extensions": range(7376, 7423 + 1), "Phonetic Extensions": range(7424, 7551 + 1), "Phonetic Extensions Supplement": range(7552, 7615 + 1), "Combining Diacritical Marks Supplement": range(7616, 7679 + 1), "Latin Extended Additional": range(7680, 7935 + 1), "Greek Extended": range(7936, 8191 + 1), "General Punctuation": range(8192, 8303 + 1), "Superscripts and Subscripts": range(8304, 8351 + 1), "Currency Symbols": range(8352, 8399 + 1), "Combining Diacritical Marks for Symbols": range(8400, 8447 + 1), "Letterlike Symbols": range(8448, 8527 + 1), "Number Forms": range(8528, 8591 + 1), "Arrows": range(8592, 8703 + 1), "Mathematical Operators": range(8704, 8959 + 1), "Miscellaneous Technical": range(8960, 9215 + 1), "Control Pictures": range(9216, 9279 + 1), "Optical Character Recognition": range(9280, 9311 + 1), "Enclosed Alphanumerics": range(9312, 9471 + 1), "Box Drawing": range(9472, 9599 + 1), "Block Elements": range(9600, 9631 + 1), "Geometric Shapes": range(9632, 9727 + 1), "Miscellaneous Symbols": range(9728, 9983 + 1), "Dingbats": range(9984, 10175 + 1), "Miscellaneous Mathematical Symbols-A": range(10176, 10223 + 1), "Supplemental Arrows-A": range(10224, 10239 + 1), "Braille Patterns": range(10240, 10495 + 1), "Supplemental Arrows-B": range(10496, 10623 + 1), "Miscellaneous Mathematical Symbols-B": range(10624, 10751 + 1), "Supplemental Mathematical Operators": range(10752, 11007 + 1), "Miscellaneous Symbols and Arrows": range(11008, 11263 + 1), "Glagolitic": range(11264, 11359 + 1), "Latin Extended-C": range(11360, 11391 + 1), "Coptic": range(11392, 11519 + 1), "Georgian Supplement": range(11520, 11567 + 1), "Tifinagh": range(11568, 11647 + 1), "Ethiopic Extended": range(11648, 11743 + 1), "Cyrillic Extended-A": range(11744, 11775 + 1), "Supplemental Punctuation": range(11776, 11903 + 1), "CJK Radicals Supplement": range(11904, 12031 + 1), "Kangxi Radicals": range(12032, 12255 + 1), "Ideographic Description Characters": range(12272, 12287 + 1), "CJK Symbols and Punctuation": range(12288, 12351 + 1), "Hiragana": range(12352, 12447 + 1), "Katakana": range(12448, 12543 + 1), "Bopomofo": range(12544, 12591 + 1), "Hangul Compatibility Jamo": range(12592, 12687 + 1), "Kanbun": range(12688, 12703 + 1), "Bopomofo Extended": range(12704, 12735 + 1), "CJK Strokes": range(12736, 12783 + 1), "Katakana Phonetic Extensions": range(12784, 12799 + 1), "Enclosed CJK Letters and Months": range(12800, 13055 + 1), "CJK Compatibility": range(13056, 13311 + 1), "CJK Unified Ideographs Extension A": range(13312, 19903 + 1), "Yijing Hexagram Symbols": range(19904, 19967 + 1), "CJK Unified Ideographs": range(19968, 40959 + 1), "Yi Syllables": range(40960, 42127 + 1), "Yi Radicals": range(42128, 42191 + 1), "Lisu": range(42192, 42239 + 1), "Vai": range(42240, 42559 + 1), "Cyrillic Extended-B": range(42560, 42655 + 1), "Bamum": range(42656, 42751 + 1), "Modifier Tone Letters": range(42752, 42783 + 1), "Latin Extended-D": range(42784, 43007 + 1), "Syloti Nagri": range(43008, 43055 + 1), "Common Indic Number Forms": range(43056, 43071 + 1), "Phags-pa": range(43072, 43135 + 1), "Saurashtra": range(43136, 43231 + 1), "Devanagari Extended": range(43232, 43263 + 1), "Kayah Li": range(43264, 43311 + 1), "Rejang": range(43312, 43359 + 1), "Hangul Jamo Extended-A": range(43360, 43391 + 1), "Javanese": range(43392, 43487 + 1), "Myanmar Extended-B": range(43488, 43519 + 1), "Cham": range(43520, 43615 + 1), "Myanmar Extended-A": range(43616, 43647 + 1), "Tai Viet": range(43648, 43743 + 1), "Meetei Mayek Extensions": range(43744, 43775 + 1), "Ethiopic Extended-A": range(43776, 43823 + 1), "Latin Extended-E": range(43824, 43887 + 1), "Cherokee Supplement": range(43888, 43967 + 1), "Meetei Mayek": range(43968, 44031 + 1), "Hangul Syllables": range(44032, 55215 + 1), "Hangul Jamo Extended-B": range(55216, 55295 + 1), "High Surrogates": range(55296, 56191 + 1), "High Private Use Surrogates": range(56192, 56319 + 1), "Low Surrogates": range(56320, 57343 + 1), "Private Use Area": range(57344, 63743 + 1), "CJK Compatibility Ideographs": range(63744, 64255 + 1), "Alphabetic Presentation Forms": range(64256, 64335 + 1), "Arabic Presentation Forms-A": range(64336, 65023 + 1), "Variation Selectors": range(65024, 65039 + 1), "Vertical Forms": range(65040, 65055 + 1), "Combining Half Marks": range(65056, 65071 + 1), "CJK Compatibility Forms": range(65072, 65103 + 1), "Small Form Variants": range(65104, 65135 + 1), "Arabic Presentation Forms-B": range(65136, 65279 + 1), "Halfwidth and Fullwidth Forms": range(65280, 65519 + 1), "Specials": range(65520, 65535 + 1), "Linear B Syllabary": range(65536, 65663 + 1), "Linear B Ideograms": range(65664, 65791 + 1), "Aegean Numbers": range(65792, 65855 + 1), "Ancient Greek Numbers": range(65856, 65935 + 1), "Ancient Symbols": range(65936, 65999 + 1), "Phaistos Disc": range(66000, 66047 + 1), "Lycian": range(66176, 66207 + 1), "Carian": range(66208, 66271 + 1), "Coptic Epact Numbers": range(66272, 66303 + 1), "Old Italic": range(66304, 66351 + 1), "Gothic": range(66352, 66383 + 1), "Old Permic": range(66384, 66431 + 1), "Ugaritic": range(66432, 66463 + 1), "Old Persian": range(66464, 66527 + 1), "Deseret": range(66560, 66639 + 1), "Shavian": range(66640, 66687 + 1), "Osmanya": range(66688, 66735 + 1), "Osage": range(66736, 66815 + 1), "Elbasan": range(66816, 66863 + 1), "Caucasian Albanian": range(66864, 66927 + 1), "Linear A": range(67072, 67455 + 1), "Cypriot Syllabary": range(67584, 67647 + 1), "Imperial Aramaic": range(67648, 67679 + 1), "Palmyrene": range(67680, 67711 + 1), "Nabataean": range(67712, 67759 + 1), "Hatran": range(67808, 67839 + 1), "Phoenician": range(67840, 67871 + 1), "Lydian": range(67872, 67903 + 1), "Meroitic Hieroglyphs": range(67968, 67999 + 1), "Meroitic Cursive": range(68000, 68095 + 1), "Kharoshthi": range(68096, 68191 + 1), "Old South Arabian": range(68192, 68223 + 1), "Old North Arabian": range(68224, 68255 + 1), "Manichaean": range(68288, 68351 + 1), "Avestan": range(68352, 68415 + 1), "Inscriptional Parthian": range(68416, 68447 + 1), "Inscriptional Pahlavi": range(68448, 68479 + 1), "Psalter Pahlavi": range(68480, 68527 + 1), "Old Turkic": range(68608, 68687 + 1), "Old Hungarian": range(68736, 68863 + 1), "Rumi Numeral Symbols": range(69216, 69247 + 1), "Brahmi": range(69632, 69759 + 1), "Kaithi": range(69760, 69839 + 1), "Sora Sompeng": range(69840, 69887 + 1), "Chakma": range(69888, 69967 + 1), "Mahajani": range(69968, 70015 + 1), "Sharada": range(70016, 70111 + 1), "Sinhala Archaic Numbers": range(70112, 70143 + 1), "Khojki": range(70144, 70223 + 1), "Multani": range(70272, 70319 + 1), "Khudawadi": range(70320, 70399 + 1), "Grantha": range(70400, 70527 + 1), "Newa": range(70656, 70783 + 1), "Tirhuta": range(70784, 70879 + 1), "Siddham": range(71040, 71167 + 1), "Modi": range(71168, 71263 + 1), "Mongolian Supplement": range(71264, 71295 + 1), "Takri": range(71296, 71375 + 1), "Ahom": range(71424, 71487 + 1), "Warang Citi": range(71840, 71935 + 1), "Zanabazar Square": range(72192, 72271 + 1), "Soyombo": range(72272, 72367 + 1), "Pau Cin Hau": range(72384, 72447 + 1), "Bhaiksuki": range(72704, 72815 + 1), "Marchen": range(72816, 72895 + 1), "Masaram Gondi": range(72960, 73055 + 1), "Cuneiform": range(73728, 74751 + 1), "Cuneiform Numbers and Punctuation": range(74752, 74879 + 1), "Early Dynastic Cuneiform": range(74880, 75087 + 1), "Egyptian Hieroglyphs": range(77824, 78895 + 1), "Anatolian Hieroglyphs": range(82944, 83583 + 1), "Bamum Supplement": range(92160, 92735 + 1), "Mro": range(92736, 92783 + 1), "Bassa Vah": range(92880, 92927 + 1), "Pahawh Hmong": range(92928, 93071 + 1), "Miao": range(93952, 94111 + 1), "Ideographic Symbols and Punctuation": range(94176, 94207 + 1), "Tangut": range(94208, 100351 + 1), "Tangut Components": range(100352, 101119 + 1), "Kana Supplement": range(110592, 110847 + 1), "Kana Extended-A": range(110848, 110895 + 1), "Nushu": range(110960, 111359 + 1), "Duployan": range(113664, 113823 + 1), "Shorthand Format Controls": range(113824, 113839 + 1), "Byzantine Musical Symbols": range(118784, 119039 + 1), "Musical Symbols": range(119040, 119295 + 1), "Ancient Greek Musical Notation": range(119296, 119375 + 1), "Tai Xuan Jing Symbols": range(119552, 119647 + 1), "Counting Rod Numerals": range(119648, 119679 + 1), "Mathematical Alphanumeric Symbols": range(119808, 120831 + 1), "Sutton SignWriting": range(120832, 121519 + 1), "Glagolitic Supplement": range(122880, 122927 + 1), "Mende Kikakui": range(124928, 125151 + 1), "Adlam": range(125184, 125279 + 1), "Arabic Mathematical Alphabetic Symbols": range(126464, 126719 + 1), "Mahjong Tiles": range(126976, 127023 + 1), "Domino Tiles": range(127024, 127135 + 1), "Playing Cards": range(127136, 127231 + 1), "Enclosed Alphanumeric Supplement": range(127232, 127487 + 1), "Enclosed Ideographic Supplement": range(127488, 127743 + 1), "Miscellaneous Symbols and Pictographs": range(127744, 128511 + 1), "Emoticons range(Emoji)": range(128512, 128591 + 1), "Ornamental Dingbats": range(128592, 128639 + 1), "Transport and Map Symbols": range(128640, 128767 + 1), "Alchemical Symbols": range(128768, 128895 + 1), "Geometric Shapes Extended": range(128896, 129023 + 1), "Supplemental Arrows-C": range(129024, 129279 + 1), "Supplemental Symbols and Pictographs": range(129280, 129535 + 1), "CJK Unified Ideographs Extension B": range(131072, 173791 + 1), "CJK Unified Ideographs Extension C": range(173824, 177983 + 1), "CJK Unified Ideographs Extension D": range(177984, 178207 + 1), "CJK Unified Ideographs Extension E": range(178208, 183983 + 1), "CJK Unified Ideographs Extension F": range(183984, 191471 + 1), "CJK Compatibility Ideographs Supplement": range(194560, 195103 + 1), "Tags": range(917504, 917631 + 1), "Variation Selectors Supplement": range(917760, 917999 + 1), } # type: Dict[str, range] UNICODE_SECONDARY_RANGE_KEYWORD = [ "Supplement", "Extended", "Extensions", "Modifier", "Marks", "Punctuation", "Symbols", "Forms", "Operators", "Miscellaneous", "Drawing", "Block", "Shapes", "Supplemental", "Tags", ] # type: List[str] RE_POSSIBLE_ENCODING_INDICATION = re_compile( r"(?:(?:encoding)|(?:charset)|(?:coding))(?:[\:= ]{1,10})(?:[\"\']?)([a-zA-Z0-9\-_]+)(?:[\"\']?)", IGNORECASE, ) IANA_SUPPORTED = sorted( filter( lambda x: x.endswith("_codec") is False and x not in {"rot_13", "tactis", "mbcs"}, list(set(aliases.values())), ) ) # type: List[str] IANA_SUPPORTED_COUNT = len(IANA_SUPPORTED) # type: int # pre-computed code page that are similar using the function cp_similarity. IANA_SUPPORTED_SIMILAR = { "cp037": ["cp1026", "cp1140", "cp273", "cp500"], "cp1026": ["cp037", "cp1140", "cp273", "cp500"], "cp1125": ["cp866"], "cp1140": ["cp037", "cp1026", "cp273", "cp500"], "cp1250": ["iso8859_2"], "cp1251": ["kz1048", "ptcp154"], "cp1252": ["iso8859_15", "iso8859_9", "latin_1"], "cp1253": ["iso8859_7"], "cp1254": ["iso8859_15", "iso8859_9", "latin_1"], "cp1257": ["iso8859_13"], "cp273": ["cp037", "cp1026", "cp1140", "cp500"], "cp437": ["cp850", "cp858", "cp860", "cp861", "cp862", "cp863", "cp865"], "cp500": ["cp037", "cp1026", "cp1140", "cp273"], "cp850": ["cp437", "cp857", "cp858", "cp865"], "cp857": ["cp850", "cp858", "cp865"], "cp858": ["cp437", "cp850", "cp857", "cp865"], "cp860": ["cp437", "cp861", "cp862", "cp863", "cp865"], "cp861": ["cp437", "cp860", "cp862", "cp863", "cp865"], "cp862": ["cp437", "cp860", "cp861", "cp863", "cp865"], "cp863": ["cp437", "cp860", "cp861", "cp862", "cp865"], "cp865": ["cp437", "cp850", "cp857", "cp858", "cp860", "cp861", "cp862", "cp863"], "cp866": ["cp1125"], "iso8859_10": ["iso8859_14", "iso8859_15", "iso8859_4", "iso8859_9", "latin_1"], "iso8859_11": ["tis_620"], "iso8859_13": ["cp1257"], "iso8859_14": [ "iso8859_10", "iso8859_15", "iso8859_16", "iso8859_3", "iso8859_9", "latin_1", ], "iso8859_15": [ "cp1252", "cp1254", "iso8859_10", "iso8859_14", "iso8859_16", "iso8859_3", "iso8859_9", "latin_1", ], "iso8859_16": [ "iso8859_14", "iso8859_15", "iso8859_2", "iso8859_3", "iso8859_9", "latin_1", ], "iso8859_2": ["cp1250", "iso8859_16", "iso8859_4"], "iso8859_3": ["iso8859_14", "iso8859_15", "iso8859_16", "iso8859_9", "latin_1"], "iso8859_4": ["iso8859_10", "iso8859_2", "iso8859_9", "latin_1"], "iso8859_7": ["cp1253"], "iso8859_9": [ "cp1252", "cp1254", "cp1258", "iso8859_10", "iso8859_14", "iso8859_15", "iso8859_16", "iso8859_3", "iso8859_4", "latin_1", ], "kz1048": ["cp1251", "ptcp154"], "latin_1": [ "cp1252", "cp1254", "cp1258", "iso8859_10", "iso8859_14", "iso8859_15", "iso8859_16", "iso8859_3", "iso8859_4", "iso8859_9", ], "mac_iceland": ["mac_roman", "mac_turkish"], "mac_roman": ["mac_iceland", "mac_turkish"], "mac_turkish": ["mac_iceland", "mac_roman"], "ptcp154": ["cp1251", "kz1048"], "tis_620": ["iso8859_11"], } # type: Dict[str, List[str]] CHARDET_CORRESPONDENCE = { "iso2022_kr": "ISO-2022-KR", "iso2022_jp": "ISO-2022-JP", "euc_kr": "EUC-KR", "tis_620": "TIS-620", "utf_32": "UTF-32", "euc_jp": "EUC-JP", "koi8_r": "KOI8-R", "iso8859_1": "ISO-8859-1", "iso8859_2": "ISO-8859-2", "iso8859_5": "ISO-8859-5", "iso8859_6": "ISO-8859-6", "iso8859_7": "ISO-8859-7", "iso8859_8": "ISO-8859-8", "utf_16": "UTF-16", "cp855": "IBM855", "mac_cyrillic": "MacCyrillic", "gb2312": "GB2312", "gb18030": "GB18030", "cp932": "CP932", "cp866": "IBM866", "utf_8": "utf-8", "utf_8_sig": "UTF-8-SIG", "shift_jis": "SHIFT_JIS", "big5": "Big5", "cp1250": "windows-1250", "cp1251": "windows-1251", "cp1252": "Windows-1252", "cp1253": "windows-1253", "cp1255": "windows-1255", "cp1256": "windows-1256", "cp1254": "Windows-1254", "cp949": "CP949", } # type: Dict[str, str] COMMON_SAFE_ASCII_CHARACTERS = { "<", ">", "=", ":", "/", "&", ";", "{", "}", "[", "]", ",", "|", '"', "-", } # type: Set[str] KO_NAMES = {"johab", "cp949", "euc_kr"} # type: Set[str] ZH_NAMES = {"big5", "cp950", "big5hkscs", "hz"} # type: Set[str] NOT_PRINTABLE_PATTERN = re_compile(r"[0-9\W\n\r\t]+") LANGUAGE_SUPPORTED_COUNT = len(FREQUENCIES) # type: int # Logging LEVEL bellow DEBUG TRACE = 5 # type: int PK[]YtOD+D+cd.pynu[PK[]}T::y+__pycache__/md.cpython-37.pycnu[PK[]cC"C"of__pycache__/cd.cpython-37.pycnu[PK[]Bt355#__pycache__/constant.cpython-37.pycnu[PK[]ׂ55)3__pycache__/constant.cpython-37.opt-1.pycnu[PK[]KlNo**m__pycache__/api.cpython-37.pycnu[PK[]\::&e __pycache__/utils.cpython-37.opt-1.pycnu[PK[]`C"C"#?__pycache__/cd.cpython-37.opt-1.pycnu[PK[]xV 'b__pycache__/legacy.cpython-37.opt-1.pycnu[PK[].33'o__pycache__/models.cpython-37.opt-1.pycnu[PK[]U*)::#__pycache__/md.cpython-37.opt-1.pycnu[PK[]ɗ"__pycache__/version.cpython-37.pycnu[PK[]ɪ**$__pycache__/api.cpython-37.opt-1.pycnu[PK[]9n) __pycache__/__init__.cpython-37.opt-1.pycnu[PK[]Z233!__pycache__/models.cpython-37.pycnu[PK[]f:: E__pycache__/utils.cpython-37.pycnu[PK[]|)g !9e__pycache__/legacy.cpython-37.pycnu[PK[]R@`#qr__pycache__/__init__.cpython-37.pycnu[PK[]>(gy__pycache__/version.cpython-37.opt-1.pycnu[PK[]t0zassets/__pycache__/__init__.cpython-37.opt-1.pycnu[PK[]t*2assets/__pycache__/__init__.cpython-37.pycnu[PK[]ûҍccassets/__init__.pynu[PK[]Y =33 vmodels.pynu[PK[]qW8 8 Qlegacy.pynu[PK[]IҍOOOO_api.pynu[PK[]oPP version.pynu[PK[]&py.typednu[PK[]3r)^cli/__pycache__/normalizer.cpython-37.pycnu[PK[] >->cli/__pycache__/__init__.cpython-37.opt-1.pycnu[PK[] >'Bcli/__pycache__/__init__.cpython-37.pycnu[PK[]O/@cli/__pycache__/normalizer.cpython-37.opt-1.pycnu[PK[]{J$$&cli/normalizer.pynu[PK[]cli/__init__.pynu[PK[]GG:md.pynu[PK[]QҚ\$\$~Mutils.pynu[PK[] r__init__.pynu[PK[]FsKK Kyconstant.pynu[PK%%D