ó
ýYc           @   s  d  d l  m Z d  d l m Z d  d l m Z m Z m Z e e ƒ Z	 e ƒ  Z
 e
 j e ƒ e	 j e ƒ e	 j e
 ƒ d Z d „  Z d „  Z d „  Z e d k rd	 d
 g Z d d g Z d d g Z e e g Z i d d 6d d 6d d 6Z e e e e ƒ Z x e D] Z e GHq	Wn  d S(   iÿÿÿÿ(   t   calculate_importance_score(   t   max_idf_key(   t	   getLoggert   StreamHandlert   DEBUGg      à?c         C   s#  g  } t  |  ƒ } xÈ t d | ƒ D]· } x® t | d | ƒ D]™ } | | k r< | | k r< t | | | | | ƒ } | t k rÕ t | | | ƒ } t | | | ƒ }	 | |	 k rÂ | j | ƒ qÒ | j | ƒ qÕ q< q< Wq" Wg  }
 x9 t d t  |  ƒ ƒ D]" } | | k rù |
 j | ƒ qù qù W|
 S(   Ni    i   (   t   lent   ranget   calculate_cosine_similarityt   similarity_limitR    t   append(   t	   sentencest   word_list_listt   idf_mapt   delete_flaged_idt   sentences_lengtht   it   jt
   similarityt   importance_it   importance_jt   ans_sentence_index(    (    sa   /virtual/develop/public_html/auto_summary/initial_app/arrange_summary/DuplicateSentenceDeleter.pyt   delete_duplicate_sentence   s"    c         C   sm  t  t |  | ƒ ƒ } t |  ƒ } t | ƒ } d } d } d } x| D]}	 d }
 |	 | k rr t | |	 ƒ }
 n t | t ƒ }
 |	 |  k rñ |	 | k rñ | t |
 | |	 d ƒ 7} | t |
 | |	 d ƒ 7} | |
 | |	 |
 | |	 7} qG |	 |  k r'|	 | k r'| t |
 | |	 d ƒ 7} qG |	 |  k rG |	 | k rG | t |
 | |	 d ƒ 7} qG qG W| | | S(   Ng        i   (   t   listt   sett   generate_tft   floatR   t   pow(   t   words1t   words2R   t	   all_wordst   word_a_tf_mapt   word_b_tf_mapt   cosine_distancet   At   Bt   wordt   idf(    (    sa   /virtual/develop/public_html/auto_summary/initial_app/arrange_summary/DuplicateSentenceDeleter.pyR   2   s(    !"c         C   sY   i  } xL |  D]D } | j  | ƒ rG | j i t | | ƒ d | 6ƒ q d | | <q W| S(   Ni   (   t   has_keyt   updatet   int(   t   wordst   tf_mapR#   (    (    sa   /virtual/develop/public_html/auto_summary/initial_app/arrange_summary/DuplicateSentenceDeleter.pyR   Q   s    %t   __main__s!   ä»Šæ—¥ã®å¤©æ°—ãŒã„ã„ã§ã™ã­s!   ä»Šæ—¥ã®å¤©æ°—ã¯æ™´ã‚Œã§ã™ã­s   ä»Šæ—¥s   å¤©æ°—s   æ™´ã‚Œgffffffæ?g333333ã?N(   t   SentenceImportanceCalculatorR    t   WordImportanceGeneratorR   t   loggingR   R   R   t   __name__t   loggert   handlert   setLevelt
   addHandlerR   R   R   R   R
   t   word_at   word_bt	   word_listR   t   ans_sentencest   sen(    (    (    sa   /virtual/develop/public_html/auto_summary/initial_app/arrange_summary/DuplicateSentenceDeleter.pyt   <module>   s.   				
