
    j(                     V    d Z ddlZddlZddlZddlmZmZmZ d	dZd
dZ	d Z
d Zd Zy)u   商品页字段抽取：产出与「抓取与对比记录表」列名对齐的清洗后字段。
原始值保留在 价格证据/原文类字段；数值字段按站点清洗器归一。
    N)SITE_CONFIGSCLEANERSclean_histogramc                     	 | j                  |      j                  }|j                         r |j                  |      j	                         S d S # t
        $ r Y y w xY w)Ntimeout)locatorfirstcount
inner_textstrip	Exception)pageselr   els       /home/user/monitor/extract.py_tr      sT    \\#$$9;r}}W}-335MM s   A
A A 	AAc                     | r| j                  d      s| S t        j                  dd| d| j                  d      d         S )uO   把 m.media-amazon.com 图片 URL 规格化为固定 token，便于逐轮比较httpz\._[^_]+_\.jpg$.z_.jpg r   )
startswithresubsplit)utokens     r   _norm_img_urlr      s=    ALL((66$%&6QHH    c           
         	 | j                  d      }t        |j                               D ]  }|j                  |      j	                         xs d}	 t        j                  |      }|j                  d      xs |j                  d      }|sdg }|j                         D ]  \  }}|D ]  }	t        |	t              s|	j                  d      xs |	j                  d      }
|	j                  d      xs d}|	j                  d      xs |	j                  d	      xs d}|j                  ||
||d
         |s|c S  	 y# t        $ r Y $w xY w# t        $ r Y yw xY w)uE   从 twister 的 a-state JSON 解析变体：子ASIN|维度=值|状态z)#twister_feature_div script[data-a-state] sortedDimValuesForAllDimsdimensionValuesDisplayDatadefaultAsinasindimensionValueStatedimensionValueDisplayTextdisplayValue)   维度   子ASIN   属性   状态N)r	   ranger   nthtext_contentjsonloadsr   getitems
isinstancedictappend)r   scriptsirawdatadimsr3   dimvalsvr%   statedisps                r   parse_variants_from_staterA      s_   ,,JKw}}' 	#A++a.--/52Czz# 8878bDHHEa<bDE!ZZ\ d	T dA%a.55/@155=DEE"78>BE55!<=\~AV\Z\DLLCDD\a!bcdd Ul!	#&     sC   AE E)CE =E E 	EE EE 	E E c                    t        | d      xs d}dD ]  }|j                  |      }|dkD  s|d| }  t        j                  dt        j                        }g }|j                  d      D ]J  }dj                  |j                               }|s%|j                  |      s7|j                  |dd	        L d
j                  |      dd S )u]   购买列文本，截断到'经常一起买/相关商品'分界，保留促销关键词行。z
#centerColr!   )zFrequently bought togetheru   Häufig zusammen gekauftzProducts related to this itemu   Ähnliche ArtikelzCompare with similar itemsVergleichenr   Nu   (Limited time deal|Angebot des Tages|Befristetes Angebot|Deal of the Day|Lightning Deal|Coupon|Save\s*[£€]|Spare\w*\s*[£€]|percent savings|%?\s*Rabatt|\d+%\s*off)
r   x   z | i  )	r   findr   compileIr   joinsearchr6   )r   txtstopidxkwkeeplines          r   collect_promo_linesrQ   -   s    
T<
 
&BCt hhtn7ds)C	
 
  @  BD  BF  BF  
GBD		$ $xx

%BIIdOKKTc
#$ ::dDS!!r   c                 T   t         |   }t        |   }i }||d<   |d   |d<   dj                  t        | d      xs dj	                               xs d |d<   | j
                  |d<   t        j                  d	| j
                        }|r|j                  d
      nd |d<   | j                         }t        j                  d|      }|r|j                  d
      nd |d<   t        | d      xs t        | d      }t        | d      xs t        | d      }t        | d      }	 |d   |      |d<    |d   |      |d<    |d   |	      |d<   t        j                  d|      }|r |d   |j                  d
            nd |d<   dj                  d d| d| d|	 fD              |d<   t        | d      |d <   |d    rd!|d    xs d"z   d#z   nd$|d%<   g }
d&D ]  \  }}	 | j                  |      }|j                         rM|
j                  | d'|j                  j                         r|j                  j                         sd(nd)        n|
j                  | d*        dj                  |
      |d,<   dj                  t        | d-      xs dj	                               xs d |d.<   dj                  t        | d/      xs t        | d0      xs dj	                               d d1 xs d |d2<   t#        |       }|xs d |d3<   t        j                  d4|      }|r|j                  d
      nd |d5<   t        j                  d6|      }|r |d   |j                  d
            nd |d7<   t        j                  d8|t        j$                        }|r |d9   |j                  d
            nd |d:<   	 | j                  d;      j                         r,| j                  d;      j                  j'                  d<d=>      nd } |d?   |xs t        | d@            |dA<    |dB   t        | dC            |dD<   t        | dE      }t)        ||      |dF<   	 | j+                  dG       t-        j.                  dH       t        | dI      }|rtdj                  |j	                               d dJ |dK<   t        j                  dL|      xs t        j                  dM|      }|r |dB   |j                  d
            nd |dN<   	 | j                  dO      j                  }t1        |j'                  dP      xs |j'                  dQ            |dR<   	 | j                  dS      }g dT}}t3        |j                               D ]  }|j5                  |      j'                  dQ      }|rdU|v sdV|v r.dW}	 |j5                  |      j                  dX      }|j                         r|j7                  dY>      nd}t9        t        j                  dZ|            }|r|d
z  }|j                  t1        |              ||d[<   t;        |      |d\<   |r|nd |d]<   	 | j                  d^      }t3        |j                               D cg c]+  }t1        |j5                  |      j'                  dQ            - }}|D cg c]#  }|s|j=                  d_      sdU|vsd`|vs"|% }}||da<   	 | j                  db      }t?               g }!} t3        |j                               D ]n  }dj                  |j5                  |      jA                         j	                               }"|"sB|"dck7  sH|"| vsM| jC                  |"       |!j                  |"       p ddj                  |!d de       xs d |df<   g }#	 | j                  dg      }$t3        |$j                               D ]  }dj                  |$j5                  |      jA                         j	                               }%t        j                  dh|%t        j$                        set        jD                  di|%      }#|%d dj |dk<    n |#rGdl|#dT   dT    dm|#dT   d
   jG                          |dn<   dj                  do |#d
d  D              xs d |dp<   	 | j                  dq      }&t3        |&j                               D cg c]>  }dj                  |&j5                  |      jA                         j	                               @ c}d dr |ds<   tI        |       }'|'sg }'	 | j                  dt      }$t3        |$j                               D ]  }|$j5                  |      j'                  du      xs  |$j5                  |      j'                  dv      }(|$j5                  |      j'                  dw      xs d})|'j                  |(dx|)v rdyndz|)v rd{nd|d}        	 |'xs g D *cg c]B  }*|*d~    d|*jK                  dd       d|*jK                  dd       d|*jK                  dd       D c}*|d<   t;        |d         |d<   |jK                  d      rdnd|d<   |S # t         $ r |
j                  | d+       Y w xY w# t         $ r d }Y w xY w# t         $ r Y w xY w# t         $ r	 d |dR<   Y jw xY w# t         $ r Y w xY w# t         $ r Y w xY wc c}w c c}w # t         $ r Y w xY w# t         $ r Y \w xY w# t         $ r Y w xY wc c}w # t         $ r Y w xY w# t         $ r Y Vw xY wc c}*w )Nu   实际站点currencyu   实际币种r   z#productTitler!   u   标题u   实际页面链接z/dp/([A-Z0-9]{10})   u
   实际ASINz#"parentAsin"\s*:\s*"([A-Z0-9]{10})"u   父ASINz+#corePriceDisplay_div .a-price .a-offscreenzspan.a-price .a-offscreenz2#corePriceDisplay_div .priceBlockStrikePriceStringz&span.a-price.a-text-price .a-offscreenz.basisPrice .a-offscreenpriceu   当前展示价格u   历史参考价u   单位价格u%   (?:UVP|RRP)[:\s]*[£€]?\s*([\d.,]+)u   建议零售价z; c              3   X   K   | ]"  }|s|j                  d d      d   dk7  s| $ yw):rT   NoneN)r   ).0xs     r   	<genexpr>z"extract_product.<locals>.<genexpr>V   s[       "Zvw|}  }D  }D  EH  JK  }L  MN  }O  SY  }Y!  "Zs   ***u   展示:u   划线:u   单位:u   价格证据z#sellerProfileTriggerIdu   当前销售店铺u   有featured offer(?)u   无featured offeru   Buy Box状态))z#add-to-cart-buttonzAdd to basket)z#buy-now-buttonzBuy NowrW   u   可见启用u   隐藏或禁用u
   :未找到u
   :未检查u   购买按钮状态z#availabilityu   可售状态z#deliveryMessagez#mir-layout-DELIVERY_BLOCK   u   配送信息原文u   Coupon及促销原文zX(Limited time deal|Angebot des Tages|Befristetes Angebot|Deal of the Day|Lightning Deal)u   限时优惠标识u,   (?:Save|Sparen|Spare)\s*[£€]?\s*([\d.,]+)u   优惠券金额z2(\d+)\s*(?:percent savings|% off|% Rabatt|Prozent)numberu   优惠券折扣百分比z#acrPopovertitlei  r   ratingz$span[data-hook='rating-out-of-text']u   综合评分r   z#acrCustomerReviewTextu   评分人数z#histogramTableu   _星级百分比zi() => document.querySelector('#reviewsMedley, #cm-cr-dp-review-list, #customerReviews')?.scrollIntoView()g      ?z0[data-hook="cr-filter-info-review-rating-count"]   u   _评论过滤行原文z,([\d.,]+)\s+(?:with reviews|mit Rezensionen)z
^([\d.,]+)u   评论总数z#landingImagezdata-old-hiressrcu	   主图URLz#altImages li.item imgr   spriteztransparent-pixelFzxpath=ancestor::li[1]i   zAvideo-block-injected|a-button-toggle-video|videoCount|video-countu   副图URL列表u   副图数量u   视频展示数量z"#aplus_feature_div img, #aplus imgr   z
grey-pixelu   A+图片URL列表z&#wayfinding-breadcrumbs_feature_div liu   ›z >    u   所属类目路径zo#detailBulletsWrapper_feature_div li, #productDetails_detailBullets_sections1 li, #detailBullets_feature_div liz'Best\s*Sellers?\s*Rank|Bestseller-?Rangz.(?:#|Nr\.?)\s*([\d.,]+)\s+in\s+([^\n(;]{2,60})   u
   _BSR原文# in u   大类排名c              3   N   K   | ]  \  }}d | d|j                            yw)rg   rh   N)r   )rY   ncs      r   r[   z"extract_product.<locals>.<genexpr>   s'     %SA!D&<%Ss   #%u   小类排名z$#feature-bullets li span.a-list-item   u   卖点列表z#li[data-asin], li[data-defaultasin]z	data-asinzdata-defaultasinclassselectedSELECTEDunavailableUNAVAILABLE	AVAILABLE)r*   r,   r*   |r)   =r+   r,   u   变体清单u   变体数量u   正常商品页u   错误页面u   页面访问状态)&r   r   rI   r   r   urlr   rJ   groupcontentr	   r   r6   r
   
is_visibleis_disabledr   rQ   rH   get_attributer   evaluatetimesleepr   r-   r.   
inner_htmlboollenr   setr   addfindallr   rA   r2   )+r   sitecfgcldmhtml	raw_price
raw_strikeraw_unitbtnsr   namebblobltrthist_rawfimmmainimgsurlsvidsr8   r   is_vidlili_htmlaplausbcseenpathrZ   rankslisrK   fbvsaclsr>   s+                                              r   extract_productr   =   sN   
t
C	$B
AAnJAn ((Bt_5;BBDEMAhK"hhA
		'2A$%aggaj4AlO<<>D
		8$?A!"1771:AiL 4FGp2dToKpIDNO  FSUVZ  ]E  TFJ$23H)bk)4A&2g;z2A#7H-An
		:DAA67;2g;qwwqz2TA		  "Z	{.CwzlE[_fgofp]q-r  "Z  ZAn !'@AA\]^r\s.!4H2I2PSQTWW  zMAoD_ -	T	-S!AwwytfA8J8J8LUVU\U\UhUhUjn  qB  &C  D  EtfJ/0- #iioA"T?";"Ar!H!H!JKStAn!hh41C(D(t4QmHn(trt'{'{'}~  @D  AD  E  M  IMA t$D!%A	npt	uB-/bhhqkTA
		A4HA67;2g;qwwqz2TA
		GrttTA@ALBxL$<tA !W[WcWcdqWrWxWxWzT\\-(..<<Wd<S  AE %8R%[2d4Z+[\An#7Bt-E$FGAn$)*H+D(;A	  B  	C

3HI*-((288:*>t*DA&'JBOoSUS\S\]jlnSoB<>7BHHQK 8DAn
||O,22&t'9'9:J'K'htOaOabgOhi+||45dtzz|$ 	0A))%0AA)<)A8FXXa[(()@A8:
"---4bii(lnuvw tqyt++mA./	0  $
I.*.$D
ll?@GLSYY[GYZ!}SWWQZ55e<=ZZlQq||F';PQ@QVbjkVkqll!$

	\\BCUBdrxxz" 	,A--/5578AQ%ZATMT[[^	, #(**T"1X"6">$
 E	ll  M  Nsyy{# 	A((3771:00288:;CyyCS"$$O

#TVYZ"%ds),	 a}Dq!1B1B1D0EF. II%Sqr%SS[W[.\\@AOTUWU]U]U_O`a!SXXbffQi&:&:&<&B&B&DEabecef.
 
#4	(B	,,DEC399;' SGGAJ,,[9iSWWQZ=U=UVh=iggaj..w7=2		asAR:jw{~j~Yf  EP  R  SS uwt|z|~noAiL>155"+=*>ahr@R?SSTUVUZUZ[cdfUgThi~AnAn-.An 4555?/AHq  	-KK4&
+,	-&      +     [l      b   		~sJ  Bi'Ai/ B(j -Aj 9Aj6 A#j&;A j6 <,k (0kk k&k8k=kkk A7k  k  k  ?k  Bk0 "!k0 ,l 9Al <	l B5l Al%i,+i,/i>=i>	jjj#"j#&	j3/j6 2j33j6 6	kk
k 	kk 	k-,k-0	k=<k= l 	ll	l"!l")i	  )_SL1500_)__doc__r0   r   r|   sitesr   r   r   r   r   rA   rQ   r    r   r   <module>r      s3      9 9I
2" ^r   