o th@sJddlZddlZddlZddlZddlZddlZddlZddlZddlm Z m Z m Z ddl m Z ddlmZddlmZddlmZddlmZddlmZmZmZmZmZmZmZmZmZdd l m!Z!m"Z"m#Z#m$Z$ddl%Z%dd l%m&Z&dd l'm(Z(m)Z)m*Z*e+e,Z-d Z.eed ge/fZ0ddZ1ddZ2de3fddZ4d[de5de6de7ddfddZ8de5ddfddZ9ded fd!d"Z:Gd#d$d$Z;Gd%dde;ZGd)d d e?Z@d*d+ZAd,e5de6fd-d.ZBdd/d0e@d1e0dee6fd2d3ZCdddd4ddddd5d5ddd6d5d7d6d8d1e0d9e/de>fd:d;ZD d\de5de3d?ejEd@ee6def dAdBZF C Dd]dff dGdHZGGdIdJdJeZHe6dKde-jIdddddd7d5dCdL dMe6dee6dNedOeedPee6d1e0dQeeee6ge6fdRe5dSe/dTe6fdUdVZJGdWdXdXZK d\dYdZZLdS)^N)ThreadPoolExecutor TimeoutError as_completed) parsedate)partial) NOT_FOUND)count)create_default_context) AnyCallableIteratorListMapping NamedTupleOptionalTupleUnion)quoteurlparseurlsplit urlunparse) exceptions) performanceutilversionREDACTEDUrlErrorcCs<tt|dd}|ds|dr|d|d<d|d<t|S)Nhttp)scheme)listrr)url parsed_urlr%6/usr/lib/python3/dist-packages/cloudinit/url_helper.py _cleanurl1s  r'cGs$dd}|}|D]}|||}q|S)NcSsNtt|}|d}|r|ds|d7}|tt|dd7}||d<t|S)Nr /z/:)safe)r"rendswithrstrr)r#add_on url_parsedpathr%r%r&combine_single<s z#combine_url..combine_singler%)baseadd_onsr/r#r,r%r%r& combine_url;s   r2returnc Cs^tjdtjdtjdtjdtdi}|t|}|s-t|t r&|j r&|j }|St dd}|S)z4helper for read_ftps to map return codes to a number,iXiz9Unexpected exception type while connecting to ftp server.i) ftplib error_reply error_temp error_perm error_protoEOFErrorgettype isinstanceOSErrorerrnoLOGwarning)excftp_error_codescoder%r%r&"ftp_get_return_code_from_exceptionKs rG@r#timeoutkwargs FtpResponsec Kst|}|jstdtd|dt_}|jpd}|jpd}d|jkrzt j t d}t d|||j|j||p;d d Wn!t jy`}zt|} td |d | d | d|d|d}~wwt d|z|j||jpoddt d|Wnt jy}zt dtddd|d|d}~wwzNz,t d|j|jd|j|jdt||WWt d|WdSt jy}zt|} td|d | d | d|d|d}~wwt d|wzwzTt } t d||| j|j||pd d t d|| j||jpddt d|j| jd|j|jdt||WWt d| WdSt jyh}zt|} td|d | d | d|d|d}~wwt d| w1swwYdS)azconnect to URL using ftp over TLS and read a file when using strict mode (ftps://), raise exception in event of failure when not using strict mode (ftp://), fall back to using unencrypted ftp url: string containing the desination to read a file from. The url is parsed with urllib.urlsplit to identify username, password, host, path, and port in the following format: ftps://[username:password@]host[:port]/[path] host is the only required component timeout: maximum time for the connection to take kwargs: unused, for compatibility with read_url returns: UrlResponse zInvalid url providedNcauserFheadersr# anonymousftps)contextz3Attempting to connect to %s via port [%s] over tls.rH)hostportrIz1Reading file from server over tls failed for url z []z"Attempting to login with user [%s]r!)userpasswdzCreating a secure connectionzAttempted to connect to an insecure ftp server but used a scheme of ftps://, which is not allowed. Use ftp:// to allow connecting to insecure ftp servers.r6zReading file: %szRETR )callbackzClosing connectionz,Reading file from ftp server failed for url z(Attempting to connect to %s via port %s.)rhostnamerrioBytesIOrTusernamerr7FTP_TLSr rBdebugconnect all_errorsrGloginpasswordprot_pr:rCr. retrbinarywriterKgetvaluecloseFTP) r#rIrJ url_partsbufferrTrVftp_tlserFftpr%r%r& read_ftpsbs          <    ] rnr. FileResponsec Ks|dr td|z t|}t||WSty+}z t|td|d|d}~wt y@}z t||j d|d|d}~ww)zmread a binary file and return a FileResponse matches function signature with read_ftps and read_url dataz'Unable to post data to file resource %sNrL) r=rBrCrload_binary_fileroFileNotFoundErrorrrIOErrorrA)r.rJcontentsrlr%r%r& _read_files    ru)ro UrlResponserKc Ks|}zt|}Wnty}zt||d|d}~ww|j}|dks-|r6d|dkr6t|jfi|S|dvrBt|fi|S|dvrNt|fi|St d|t|fi|S) a0Wrapper function around readurl to allow passing a file path as url. When url is not a local file path, passthrough any kwargs to readurl. In the case of parameter passthrough to readurl, default values for some parameters. See: call-signature of readurl in this module for param docs. )rMr#Nfiler(r)rmrQ)rhttpszAttempting unknown protocol %s) lstripr ValueErrorrrrur.rnreadurlrBrC)r#rJparsedrlrr%r%r&read_file_or_urls    r}c@s&eZdZd ddZddZddZdS) StringResponsecCs||_i|_||_||_dSN)rFrNrtr#selfrtr#rFr%r%r&__init__s zStringResponse.__init__cOs |jdkS)NrrFrargsrJr%r%r&ok zStringResponse.okcCs |jdS)Nzutf-8)rtdecoderr%r%r&__str__s zStringResponse.__str__Nr)__name__ __module__ __qualname__rrrr%r%r%r&r~s  r~cs(eZdZddedeffdd ZZS)rorrtr#cstj|||ddS)Nrsuperrr __class__r%r&r!szFileResponse.__init__rrrrbytesr+r __classcell__r%r%rr&ro s cs&eZdZdedeffdd ZZS)rKrtr#cst||dSrr)rrtr#rr%r&r&szFtpResponse.__init__rr%r%rr&rK%sc@seZdZdejfddZedefddZede fddZ dde fd d Z ed d Z edefddZddZ ddeede deefddZdS)rvresponsecCs ||_dSr) _response)rrr%r%r&r+rzUrlResponse.__init__r3cCs|jjdurdS|jjS)N)rcontentrr%r%r&rt.s zUrlResponse.contentscC|jjSr)rr#rr%r%r&r#4zUrlResponse.urlFcCs.d}|rd}d|jkr|krdSdSdS)Nr4r5rTFr)r redirects_okupperr%r%r&r8szUrlResponse.okcCrr)rrNrr%r%r&rNArzUrlResponse.headerscCrr)r status_coderr%r%r&rFErzUrlResponse.codecCrr)rtextrr%r%r&rIszUrlResponse.__str__r chunk_sizedecode_unicodeccs|j||EdHdS)atIterates over the response data. When stream=True is set on the request, this avoids reading the content at once into memory for large responses. :param chunk_size: Number of bytes it should read into memory. :param decode_unicode: If True, content will be decoded using the best available encoding based on the response. N)r iter_content)rrrr%r%r&rLs zUrlResponse.iter_contentN)F)rF)rrrrequestsResponserpropertyrrtr+r#boolrrNintrFrrr rr%r%r%r&rv*s( rvc @s:eZdZ   ddedeedeedeefddZdS) rNrMrFrNr#cCs8t|t|||_||_|durin||_||_dSr)rsrr+rMrFrNr#)rrMrFrNr#r%r%r&r\s  zUrlError.__init__)NNN) rrrr rrrr+rr%r%r%r&r[scCsi}t|j}|dkr@|r@d|vr|dr|d|d<nd|d<d|vr4d|vr4|d|dg|d<|Sd|vr@t|d|d<|S)Nrxca_certsverifyT cert_filekey_filecert)rrr+)r# ssl_detailsssl_argsrr%r%r& _get_ssl_argsjs  r retry_afterc Cszt|}W|StyV}zCzt|}|std|tt|t}Wnty8td|d}Ynw|dkrKtdd}WYd}~|SWYd}~|Sd}~ww)a<Parse a Retry-After header value into an integer. : param retry_after: The value of the Retry-After header. https://www.rfc-editor.org/rfc/rfc9110.html#section-10.2.3 https://www.rfc-editor.org/rfc/rfc2616#section-3.3 : return: The number of seconds to wait before retrying the request. z(Failed to parse Retry-After header valuezGFailed to parse Retry-After header value: %s. Waiting 1 second instead.rrzBRetry-After header value is in the past. Waiting 1 second instead.N)floatrzrtimemktimerBinfo)rto_waitrD time_tupler%r%r&_get_retry_after|s<   r exception_cberrorrcCs`|r||rdS|jr*|jdkr*td|j|jr#t|jddStddS|s.dS|)aHandle exceptions raised during request processing. If we have no exception callback or the callback handled the error or we got a 503, return with an optional timeout so the request can be retried. Otherwise, raise the error. :param error: The exception raised during the request. :param response: The response object. :param exception_cb: Callable to handle the exception. :return: Optional time to wait before retrying the request. NizNEndpoint returned a 503 error. HTTP endpoint is overloaded. Retrying URL (%s).z Retry-After1z7Unable to introspect response header. Waiting 1 second.r)rFrBrCr#rNrr=r)rrr%r%r& _handle_errors  rrTFr!)rprIretries sec_betweenrN headers_cbheaders_redactr check_statusallow_redirectsrsessioninfinite log_req_resprequest_methodstreamrc st|}||d}|t||| |d<|s|rdnd}||d<|dur8t|tr/||d<n tt|d|d<|dur>g}d }|rKtt|d d }d t }|durZ| }ni}|rb||d <|durhd }| durpt } t D]}|r|||}d |vr||d <||d<i}|D]1\}|d krq|dkr|rfdd|D}|rt ||<|D]}t|||<qq||<qz3|rtd|| rdn|||| jdi|}| r|td||jt|j|d t|WStjy}zt||d|d}~wtjy)}zt||jj|jj|d}|}WYd}~n"d}~wtjyF}zt||d}|}d}WYd}~nd}~wwt || d}|s_| pX|d |k}|s_|||pc|}|dkrw|rrtd|t!"|qst#d)a`Wrapper around requests.Session to read the url and retry if necessary :param url: Mandatory url to request. :param data: Optional form data to post the URL. Will set request_method to 'POST' if present. :param timeout: Timeout in seconds to wait for a response. May be a tuple if specifying (connection timeout, read timeout). :param retries: Number of times to retry on exception if exception_cb is None or exception_cb returns True for the exception caught. Default is to fail with 0 retries on exception. :param sec_between: Default 1: amount of seconds passed to time.sleep between retries. None or -1 means don't sleep. :param headers: Optional dict of headers to send during request :param headers_cb: Optional callable returning a dict of values to send as headers during request :param headers_redact: Optional list of header names to redact from the log :param ssl_details: Optional dict providing key_file, ca_certs, and cert_file keys for use on in ssl connections. :param check_status: Optional boolean set True to raise when HTTPError occurs. Default: True. :param allow_redirects: Optional boolean passed straight to Session.request as 'allow_redirects'. Default: True. :param exception_cb: Optional callable to handle exception and returns True if retries are permitted. :param session: Optional exiting requests.Session instance to reuse. :param infinite: Bool, set True to retry indefinitely. Default: False. :param log_req_resp: Set False to turn off verbose debug messages. :param request_method: String passed as 'method' to Session.request. Typically GET, or POST. Default: POST if data is provided, GET otherwise. :param stream: if False, the response content will be immediately downloaded. )r#rrPOSTGETmethodNrIrrz Cloud-Init/%srpz User-AgentrNcsg|] }|r|qSr%)r=).0kvr%r& *szreadurl..z'[%s/%s] open '%s' with %s configurationrz(Read from %s (%s, %sb) after %s attemptsr#rFrNr#r1Please wait %s seconds while we wait to try againz"This path should be unreachable...r%)$r'updaterr?tuplemaxrrrversion_stringcopyrSessionritemsdeepcopyrrBr^requestraise_for_statusrlenrrvrSSLErrorr HTTPErrorrrNRequestExceptionrrsleep RuntimeError) r#rprIrrrNrrrrrrrrrrrreq_args manual_tries user_agentifiltered_req_argsrmatched_headerskeyrrl url_errorraised_exceptionresponse_sleep_time will_retry sleep_timer%rr&r{s5              r{func.addreventdelaycCs|r |j|dr dS|||S)z Execute func with optional delayrINwaitrrrIrrr%r%r&_run_func_with_delayss  r333333? addresses stagger_delayc s.d}d}d}g}ttt|dz|zWfddt|D}t|dD],} || }| } | r>| }||q*| }|rV ||fWWj ddSq*|rbt d|||t d |td tyt d d |d tt|YnwWj dd||fSj ddw) aexecute multiple callbacks in parallel Run blocking func against two different addresses staggered with a delay. The first call to return successfully is returned from this function and remaining unfinished calls are cancelled if they have not yet started N) max_workersc s,i|]\}}jt||d|qS)r)submitr)rrrexecutorris_donerrIr%r& s zdual_stack..rFrz.default_sleep_timerr start_timercSs0|tddfvr dS|dkpt|||kS)z4Check if time is up based on start time and max waitrNFr)rr monotonic)rr#rr%r%r&timeups zwait_for_url..timeuprr#c Ssd}d}|r|sd}tt|}||fSz|jWn&tjjyA}zt||jj|jj |d}|t |fWYd}~Sd}~ww|j sZd|j }tt||j |j |d}||fS| sod|j }tt||j |j |d}||fS)z?Map requests response code/contents to internal "UrlError" typer!NzRequest timed outrzempty response [%s]zbad status code [%s])rrzrrrrrrrrNr+rtrFr)rr#reasonurl_excrlr%r%r&handle_url_responsesH   z)wait_for_url..handle_url_response url_reader_cburlsexc_cblog_cbc sd}d}z||\}}||\}}|st||ddWSWn/ty5} z d| }| }WYd} ~ nd} ~ wtyL} z d| }| }WYd} ~ nd} ~ wwtt|} r[dnd} d|pet|d d | | |f} || tddt|trt||d d Sdd S) z:Execute request, handle response, optionally log exceptionr!N)rzrequest error [%s]zunexpected error [%s]z%ss unlimitedzCalling '%s' failed [%s/%s]: %sr#rr#rr) rr Exceptionrrr$getattrr?r) r)r*r#r+r,r&r#rr'rl time_taken max_wait_str status_msg)r(rr%r&read_url_handle_exceptionsDsD   z0wait_for_url..read_url_handle_exceptionsrIcs$t|durin||ddS)NF)rNrrIrr)r{)r#rI)rrrr%r& read_url_cbpsz!wait_for_url..read_url_cbc sdtffdd }g}D]H}t}dkr>s>|r(tddddSdur>r>||kr>t|||||||}|jrM|S|jrV||jq|r]t|nd} tdd| dS)z|iterate over list of urls, request each one and handle responses and thrown exceptions individually per url r#cs||fSrr%r)r5rIr%r&url_reader_serialz@wait_for_url..read_url_serial..url_reader_serialrNr.) r+rr$rrrrr r) r#rIr+r,r6 wait_timesr#nowoutr)loop_nrmust_try_againr5r4r%r*rr&read_url_serialzs0     z%wait_for_url..read_url_serialcs tt|d}||||S)zpass list of urls to dual_stack which sends requests in parallel handle response and exceptions of the first endpoint to respond )rrI)rr)r#rIr+r,url_reader_parallel)rr5r4r*r%r&read_url_parallels z'wait_for_url..read_url_parallelz3sleep_time and sleep_time_cb are mutually exclusiveNTFrr)FN)r)rrrrvr+rrr r rr ExceptionCallbackrrr$rzrr#rtrrrBr^)r*rrIrrrrrrrrrr"r=r?r# do_read_urlcalculate_sleep_timerrespcurrent_sleep_time current_timer%) rr(rrr;rr<r5r4rrr%r*r& wait_for_urls/ ), $      rFc@sheZdZ     dddZddZddZd d Zd d Zd dZddZ ddZ ddZ ddZ dS)OauthUrlHelperN/run/oauth_skew.jsoncCst||_|pd|_||_||_||_d|_d|_|j|j|jf}t|s'd|_nt|s/t d| }|p6i|_ dS)Nr!Tr FzBall or none of token_key, token_secret, or consumer_key can be set) consumer_keyconsumer_secret token_key token_secretskew_data_file _do_oauthskew_change_limitanyallrzread_skew_file skew_data)rrIrKrLrJrMrequiredoldr%r%r&rs  zOauthUrlHelper.__init__c Cs|jrGtj|jrGtd|j,t|jdd}t|WdWdS1s2wYWddS1sBwYdS)NzReading rmode) rMosr.isfilerTimedopenjsonload)rfpr%r%r&rRs:zOauthUrlHelper.read_skew_filec Cs|jsdS|}|duri}|||<td|j0t|jdd}|t|Wdn1s7wYWddSWddS1sOwYdS)NzWriting wrW)rMrRrr[r\rer]dumps)rrSvaluecurr_r%r%r&update_skew_filesPzOauthUrlHelper.update_skew_filec Cst|tr|jdks|jdksdSd|jvrtd|jdS|jd}z tt|}Wnt yG}ztd||WYd}~dSd}~wwt |t}t |j j }|j|d}t|||jkrs|||td||||j|<dS)Niidatez$Missing header 'date' in %s responsez#Failed to convert datetime '%s': %srz$Setting oauth clockskew for %s to %d)r?rrFrNrBrCrrrr/rrr#netlocrSr=absrOrd)rrre remote_timerlskewrSold_skewr%r%r&rs.     zOauthUrlHelper.exception_cbcCsZ|jsiSd}t|j}|jr||jvrtt|j|}t||j|j|j |j |dS)N)r#rIrKrLrJ timestamp) rNrrfrSrr oauth_headersrIrKrLrJ)rr#rkrSr%r%r&r-s zOauthUrlHelper.headers_cbcCs:t|j|d|d<t|j|d|d<||i|S)Nrr)r _headers_cbr= _exception_cb)r wrapped_funcrrJr%r%r&_wrapped?s  zOauthUrlHelper._wrappedcO|t||Sr)rprFrr%r%r&rFHr7zOauthUrlHelper.wait_for_urlcOrqr)rpr{rr%r%r&r{Kr7zOauthUrlHelper.readurlc Cs>d}z|r||}W|||SW|||S||w)NTr)rextra_exception_cbrretr%r%r&rnNs    zOauthUrlHelper._exception_cbcCs$i}|r||}||||Sr)rr)rextra_headers_cbr#rNr%r%r&rmWs zOauthUrlHelper._headers_cb)NNNNrH) rrrrrRrdrrrprFr{rnrmr%r%r%r&rGs     rGc Csrzddlm}Wnty}ztd|d}~ww|r!t|}nd}|j|||||j|d}||\} } } | S)Nrzoauth support is not available) client_secretresource_owner_keyresource_owner_secretsignature_methodrk)oauthlib.oauth1oauth1 ImportErrorNotImplementedErrorr+ClientSIGNATURE_PLAINTEXTsign) r#rIrKrLrJrkrzrlclient_urisigned_headers_bodyr%r%r&rl_s&  rl)rHr)rr)Mrr7rZr]loggingrYrrconcurrent.futuresrrr email.utilsr functoolsr http.clientr itertoolsrsslr typingr r r r rrrrr urllib.parserrrrrr cloudinitrrr getLoggerrrBrrr@r'r2rrGr+rdictrnrur}r~rorKrvrsrrrrr{rrrrr^rFrGrlr%r%r%r&s      ,     1' &  6    S