o i@sddlZddlZddlZddlZddlZddlZddlZddlmZm Z ddl m Z m Z m Z mZmZmZmZmZddlmZmZmZmZmZmZmZmZmZddlmZddlm Z ddl!m"Z"m#Z#ddl$m%Z%dd l&m'Z'dd l(m)Z)dd l*m+Z+d Z,d Z-dZ.dZ/e,e-e.gZ0dZ1dZ2dZ3dZ4dZ5dZ6dZ7dZ8e9e:Z;dddfdddfdddfdZGd$d%d%e?Z@Gd&d'd'e?ZAGd(d)d)e?ZBdWd,d-ZCe5fd.d/ZDGd0d1d1eZEGd2d3d3eZFGd4d5d5eZGGd6dde)ejHd7ZId8d9ZJd:eeIeKffd;d<ZLd=d>ZM ?dXd@eKd:eNfdAdBZOdCdDZPdYdFdGZQGdHdIdIeRZSdJdKZTdLeIdMeKd:eNfdNdOZUdMeKd:eeIfdPdQZVd:eKfdRdSZWdTeKd:eKfdUdVZXdS)ZN)Enumunique)AnyDictList NamedTupleOptionalTupleUnioncast) atomic_helperdmiimporter lifecyclenet performance type_utils user_datautil) write_json)Distro) EventScope EventType) launch_index)Paths)CloudInitPickleMixin)eventsdisabledlocalrpass FILESYSTEMNETWORK DataSourcez|EXPERIMENTAL: The structure and format of content scoped under the 'ds' key may change in subsequent releases of cloud-init.zredacted for non-root user cloud-name_unsetunknownz aws-chinacC|dkSNawscr)r)Fr-zaws-govcCr&r'r)r*r)r)r,r-Gr.z azure-chinacCr&)Nazurer)r*r)r)r,r-Hr.)zcn-zus-gov-chinac@s2eZdZdZdZdZdZdZdZde fdd Z d S) NetworkConfigSourcezb Represents the canonical list of network config sources that cloud-init knows about. cmdlineds system_cfgfallback initramfsreturncC|jSNvalueselfr)r)r,__str__YzNetworkConfigSource.__str__N) __name__ __module__ __qualname____doc__CMD_LINEDS SYSTEM_CFGFALLBACK INITRAMFSstrr>r)r)r)r,r1Lsr1c@s&eZdZdZdZdZdefddZdS)NicOrderzRepresents ways to sort NICsmacnic_namer7cCr8r9r:r<r)r)r,r>cr?zNicOrder.__str__N)r@rArBrCMACNIC_NAMErIr>r)r)r)r,rJ]s rJc@eZdZdZdS)DatasourceUnpickleUserDataErrorzERaised when userdata is unable to be unpickled due to python upgradesNr@rArBrCr)r)r)r,rPgrPc@ eZdZdS)DataSourceNotFoundExceptionNr@rArBr)r)r)r,rTkrTc@rO)InvalidMetaDataExceptionz8Raised when metadata is broken, unavailable or disabled.NrQr)r)r)r,rWorRrWr)c Cst|}g}g}|D]V\}}|r|d|}n|}||vs(||vr-||t|trD|drD|||dd||<t|t rct |||} | | d| | d| ||<q t ||d<t ||d<|S)zProcess all instance metadata cleaning it up for persisting as json. Strip ci-b64 prefix and catalog any 'base64_encoded_keys' as a list @return Dict copy of processed metadata. /zci-b64:rXbase64_encoded_keyssensitive_keys)copydeepcopyitemslowerappend isinstancerI startswithreplacedictprocess_instance_metadataextendpopsorted) metadatakey_pathr[md_copyrZ sens_keyskeyval sub_key_path return_valr)r)r,ress0        recCs|dgs|St|}|dD]+}|d}|}|D]}||vr4t||tr4||dkr4||}q||vr=|||<q|S)zRedact any sensitive keys from to provided metadata dictionary. Replace any keys values listed in 'sensitive_keys' with redact_value. r[rY)getr\r]splitrard)ri redact_valuerkrj path_partsobjpathr)r)r,redact_sensitive_keyss"     rxc@s.eZdZUeed<eed<eed<eed<dS) URLParamsmax_wait_secondstimeout_seconds num_retriessec_between_retriesN)r@rArBint__annotations__r)r)r)r,rys  ryc@s"eZdZUeeed<eed<dS)DataSourceHostnamehostname is_defaultN)r@rArBrrIrboolr)r)r)r,rs  rc@s*eZdZUdZeed<eed<eed<dS)HotplugRetrySettingsz in seconds force_retry sleep_period sleep_totalN)r@rArBrCrrr~r)r)r)r,rs  rc @seZdZUeZdZdZdZee e d<dZ dZ dZ eeee fe d<ejejejejfZeedfe d<dZd Zd Zd Zejejejej ej!hiZ"ejejhiZ#d e$fd e$fdifddddddf Z%eee e&fdfe d<dZ'dZ(ee dfe d<dZ)e*dddZ+dZ,ee e d<d Z-dde.de/fddZ0de1d dfd!d"Z2d#d$Z3d e4fd%d&Z5d e4fd'd(Z6d)d*Z7d+d,Z8dd.d/Z9e:j;d0d1d2d e4fd3d4Zd:d;Z?ddd?ZAd@dAZBeCdBdCZDeCdDdEZEdFdGZFeCdHdIZGdJdKZHeCdLdMZIdNdOZJdPdQZKdRdSZLdTdUZMdVdWZNdXdYZOdZd[ZPd\d]ZQd^d_ZRd`daZSeCdbdcZTeCdddeZUdfdgZVddhdiZWdjdkZXdleYefdmdnZZdleYed e4fdodpZ[dqdrZ\dsdtZ]e^ddudvZ_eCdwdxZ`dydzZad{d|Zbd}d~ZcdS)r"z en_US.UTF-8_undefN _cloud_name_crawled_metadata.network_config_sourcesrq  ec2_metadata network_jsonri)userdataN) userdata_rawN) vendordataN)vendordata_rawN) vendordata2N)vendordata2_rawNcached_attr_defaultsF) combined_cloud_config merged_cfgmerged_system_cfgzsecurity-credentialsr user-datarr vendor-datazds/vendor_datasensitive_metadata_keysrextra_hotplug_udev_rulesdistropathscCs||_||_||_d|_i|_d|_d|_d|_d|_d|_ d|_ t |_ t |_ t|jd|jfi|_|js9i|_|sDt|j|_dS||_dS)N datasource)sys_cfgrrrrirrrrrmetadata_addressUNSETrrrget_cfg_by_pathdsnameds_cfgrUserDataProcessorud_proc)r=rrrrr)r)r,__init__Os* zDataSource.__init__ci_pkl_versionr7cCsdddtddtdddtdddd }|D]\}}t||s%t|||qt|ds3t|dddt|dr\|jdur^zt|jWdSty[}z t d |t |d}~wwdSdS) z(Perform deserialization fixes for Paths.NFr) r_platform_type _subplatformrrrrskip_hotplug_detectrrhotplug_retry_settingscheck_if_fallback_is_allowedcSdSNFr)r)r)r)r,r-}z&DataSource._unpickle..rz:Unable to unpickle datasource: %s. Ignoring current cache.) rrr^hasattrsetattrrrIAttributeErrorLOGdebugrP)r=rexpected_attrsrmr;er)r)r, _unpickleis<    zDataSource._unpicklecCs t|Sr9robj_namer<r)r)r,r>s zDataSource.__str__cCr)z#Check if running on this datasourceTr)r<r)r)r, ds_detectzDataSource.ds_detectcCsN|jtkrtd|dS|jdg|jgkr%td|dSdS)aJOverride if either: - only a single datasource defined (nothing to fall back to) - command line argument is used (ci.ds=OpenStack) Note: get_cmdline() is required for the general case - when ds-identify does not run, _something_ needs to detect the kernel command line definition. z6Kernel command line set to use a single datasource %s.Tdatasource_listz2Datasource list set to use a single datasource %s.F)rr_ parse_cmdlinerrrrrr<r)r)r,override_ds_detects zDataSource.override_ds_detectcCs<|r|S|rtd||Std|dS)z&Overrides runtime datasource detectionz Detected %szDid not detect %sF)r _get_datarrrr<r)r)r,_check_and_get_datas zDataSource._check_and_get_datacCs|j}|}|j}|d}diddgd|d|dt|j|j|jd|jd |jd |d d d |d dd|d dd|jd|d|dd|d|d|ddd|d||dd|j|j |d|ddiS)z2Return a dictionary of standardized metadata keys.sys_infov1 _beta_keys subplatformavailability-zoneavailability_zonecloud_idr# cloud_namerdistrdistro_versionrdistro_releaseplatformpublic_ssh_keyspython_versionpython instance-id instance_idkernel_releaseunamelocal-hostnamelocal_hostnamevariant)machineregionrsystem_platformr) get_hostnamerget_instance_idrcanonical_cloud_idrr platform_typeget_public_ssh_keysr)r= instance_datarrrsysinfor)r)r,_get_standardized_metadatas`         z%DataSource._get_standardized_metadatar)cCsP|jsdS|r |}n|j}|D]\}}t||rt|||q|s&d|_dSdS)zReset any cached metadata attributes to datasource defaults. @param attr_defaults: Optional tuple of (attr, value) pairs to set instead of cached_attr_defaults. NF) _dirty_cacherrr)r= attr_defaults attr_values attributer;r)r)r,clear_cached_attrss    zDataSource.clear_cached_attrszGetting metadataalways)log_modecCs"d|_|}|s |S||S)zDatasources implement _get_data to setup metadata and userdata_raw. Minimally, the datasource should return a boolean True on success. T)rrpersist_instance_data)r= return_valuer)r)r,get_datas zDataSource.get_dataTc Cs^|rtj|jjrt||jd|jdur2tt t |j}| dd| ddd|i}ndd|j ii}|jtkrE|j|dd<|jtkrQ|j|dd<t|dd <t |j|d <d |d d <t |d |d <d |d d <t|d<|||zt|}tt||jd}Wn5ty}ztdt |WYd}~dSd}~wt!y}ztdt |WYd}~dSd}~ww|j"d}|d#dd}tj$|jj%d} t&| d||dd} | d|} tj'| rtj(| } tj)| | dd| r| | krt*| t+||dd|j"d} t+| t,|dS)aPProcess and write INSTANCE_JSON_FILE with all instance metadata. Replace any hyphens with underscores in key names for use in template processing. :param write_cache: boolean set True to persist obj.pkl when instance_link exists. @return True on successful write, False otherwise. obj_pklNrrr3 meta_datarr_docrz, (str, bool). is_default is a bool and it's true only if hostname is localhost and was returned by util.get_hostname() as a default. This is used to differentiate with a user-defined localhost hostname. Optionally return (None, False) when metadata_only is True and local-hostname data is not available. localdomain localhostFrNT.rzip-%srrz%s.%s)rirrrrrget_fqdn_from_hostsfindrIrsris_ipv4_address gethostbyaddrrcrGr ) r=fqdn resolve_ip metadata_only defdomaindefhostdomainrtoksr hosts_fqdnlhostr)r)r,rKsF            zDataSource.get_hostnamecCs|jj|dS)N) data_source)rget_package_mirror_infor<r)r)r,rjsz"DataSource.get_package_mirror_infosource_event_typescCsPi}|D]!}|jD]\}}||vr$||st||<|||q q|Sr9)supported_update_eventsr^rrsetadd)r=rksupported_eventsevent update_scope update_eventsr)r)r,get_supported_eventss  zDataSource.get_supported_eventsc Cs||}|D]\}}td|jddd|D|d|tffq |r6||}|r6dStd|ddd|Dd S) aRefresh cached metadata if the datasource supports this event. The datasource has a list of supported_update_events which trigger refreshing all cached metadata as well as refreshing the network configuration. @param source_event_types: List of EventTypes which may trigger a metadata update. @return True if the datasource did successfully update cached metadata due to source_event_type. z:Update datasource metadata and %s config due to events: %s, cSg|]}|jqSr)r:.0rpr)r)r, z;DataSource.update_metadata_if_supported..z _%s_configTz(Datasource %s not updated for events: %scSrur)r:rvr)r)r,rxryF) rsr^rrr;r rrr)r=rkroscopematched_eventsresultr)r)r,update_metadata_if_supporteds& z'DataSource.update_metadata_if_supportedcCrrr))r=rr)r)r,check_instance_idrzDataSource.check_instance_idcCr)acheck_if_fallback_is_allowed() Checks if a cached ds is allowed to be restored when no valid ds is found in local mode by checking instance-id and searching valid data through ds list. @return True if a ds allows fallback, False otherwise. Fr)r<r)r)r,rsz'DataSource.check_if_fallback_is_allowedcCsT|durt}|dur t}|D]}|durq||vr|Std|||S|S)Nz%invalid dsmode '%s', using default=%s)DSMODE_NETWORK VALID_DSMODESrr) candidatesdefaultvalid candidater)r)r,_determine_dsmodeszDataSource._determine_dsmodecCrOr9r)r<r)r)r,network_configrzDataSource.network_configcCr)a(setup(is_new_instance) This is called before user-data and vendor-data have been processed. Unless the datasource has set mode to 'local', then networking per 'fallback' or per 'network_config' will have been written and brought up the OS at this point. Nr))r=is_new_instancer)r)r,setups zDataSource.setupcCr)aactivate(cfg, is_new_instance) This is called before the init_modules will be called but after the user-data and vendor-data have been fully processed. The cfg is fully up to date config, it contains a merged view of system config, datasource config, user config, vendor config. It should be used rather than the sys_cfg passed to __init__. is_new_instance is a boolean indicating if this is a new instance. Nr))r=cfgrr)r)r,activates zDataSource.activatecCr)z Called when the user issues 'cloud-init clean -c datasource' command. Individual data sources should implement their own cleanup handler. Nr)r<r)r)r,clean rRzDataSource.cleanr9)r)T)F)FFF)NN)dr@rArBrdsmoderSrrrrIrrrrr rr1rDrHrFrErr rr!r#r%rr!rBOOT_NEW_INSTANCEBOOT BOOT_LEGACYHOTPLUGrldefault_update_eventsrrrrrrrrr_ci_pkl_versionrrrr~rr>rrrrrrrtimedrrrr&r+r.r1propertyrrr2rr6rr)r(r-r0r?rrBrNrQrTrrrrrjrrsr}r~r staticmethodrrrrrr)r)r)r,r"s      % &  V <         H  &   ) metaclasscCs|sgSt|tr |St|ttfrt|St|trEg}|D]\}}t|tr/|g}t|ttfrB|D] }|rA||q8q#|SgSr9)rarI splitlineslistrmrdr^r`) pubkey_datakeys_keynameklistpkeyr)r)r,r@s$    r@r7c Cs.t|||}dd|D}t|vrdnd} td| |t||D]j\} } tjd| ddd | | fd | | f|d } z?| 3td | | |||} | t j grkd | | f| _ | t | fWdWSWdn1suwYWq!tyttd| Yq!wdd|}t|)NcSsg|]}t|qSr)r)rwr>r)r)r,rx4szfind_source..networkrz#Searching for %s data source in: %sz search-%sr"rXzsearching for %s data from %szno %s data found from %s)name descriptionmessageparentz%Seeing if we can get any data from %szfound %s data from %szGetting data from %s failedz4Did not find any data source, searched classes: (%s)rt) list_sources DEP_NETWORKrrziprReportEventStackrcr}rrrrrr$rr r rT)rrrds_depscfg_listpkg_listreporterds_listds_namesrrclsmyrepsmsgr)r)r, find_source0s>       rc Csg}td||||D]5}t|}t||dg\}}|s%td||D]}t|} t| d} | |} | r@|| nq'q |S)zReturn a list of classes that have the same depends as 'depends' iterate through cfg_list, loading "DataSource*" modules and calling their "get_datasource_list". Return an ordered list of classes that match (if any) zLLooking for data source in: %s, via packages %s that matches dependencies %sget_datasource_listzDCould not import %s. Does the DataSource exist and is it importable?) rrr"match_case_insensitive_module_name find_moduleerror import_modulegetattrrf) rdependsrsrc_listr3ds_namem_locs _looked_locsm_locmodlistermatchesr)r)r,rQs6    r system-uuidfieldcCs*|sdSt|}|s dS||kSr)r read_dmi_datar_)rr dmi_valuer)r)r,instance_id_matches_system_uuidus  rcCsl|st}|st}|tkr|tkr|S|StD]\}}|\}}||r-||r-|Sq|tkr4|S|S)z@Lookup the canonical cloud-id for a given cloud_name and region.)r4CLOUD_ID_REGION_PREFIX_MAPr^rb)rrrprefix cloud_id_testr valid_cloudr)r)r,rs rTcCsf|sdSt|tr |St|trt|St|tr+|dur't|dddStdtdt |)aLdata: a loaded object (strings, arrays, dicts). return something suitable for cloudinit vendordata_raw. if data is: None: return None string: return string list: return data the list is then processed in UserDataProcessor dict: return convert_vendordata(data.get('cloud-init')) NTz cloud-initF)recursez'vendordata['cloud-init'] cannot be dictz$Unknown data type for vendordata: %s) rarIrr\r]rdconvert_vendordatarrrr7)datarr)r)r,rs     rc@rS)BrokenMetadataNrUr)r)r)r,rrVrcCs4g}t|}|D]\}}|t|kr||q|Sr9)rmr`)rrret_listdepsetrdepsr)r)r,list_from_dependss   rrvfnamecCsnzt|}Wntyttd|YdSwz tj||dddWdSty6ttd|YdSw)z[Use pickle to serialize Datasource to a file as a cache. :return: True on success zFailed pickling datasource %sFwb)omoderz Failed pickling datasource to %sT)pickledumpsr$rr rr )rvr pk_contentsr)r)r,rs  rc Csd}zt|}Wnty(}ztj|rtd||WYd}~nd}~ww|s-dSzt |WSt y=YdStyMt td|YdSw)zBUse pickle to deserialize a instance Datasource from a cache file.Nzfailed loading pickle in %s: %sz#Failed loading pickled blob from %s) rload_binary_filer$rrwisfilerrrrrPr )rpickle_contentsrr)r)r,pkl_loads$    rcCs ttS)zCheck if command line argument for this datasource was passed Passing by command line overrides runtime datasource detection )parse_cmdline_or_dmir get_cmdliner)r)r)r,rs rinputcCstd|}td|}td|}|p|p|}|p|}|r7|d}tjd|d|dd|d d |rC|drC|dSd S) Nz(?:^|\s)ds=([^\s;]+)z(?:^|\s)ci\.ds=([^\s;]+)z (?:^|\s)ci\.datasource=([^\s;]+)rz8Defining the datasource on the command line using ci.ds=z or ci.datasource=z23.2zUse ds=z instead) deprecateddeprecated_version extra_messagerX)researchgroupstripr deprecate)r ds_parse_0 ds_parse_1 ds_parse_2r3rrr)r)r,rs&      r)rXr))rr)Yabcr\rloggingrrrenumrrtypingrrrrrr r r cloudinitr r rrrrrrrcloudinit.atomic_helperrcloudinit.distrosrcloudinit.eventrrcloudinit.filtersrcloudinit.helpersrcloudinit.persistencercloudinit.reportingrDSMODE_DISABLED DSMODE_LOCALr DSMODE_PASSrDEP_FILESYSTEMr DS_PREFIXrREDACT_SENSITIVE_VALUEr5rr4 getLoggerr@rrr1rJr$rPrTrWrerxryrrABCMetar"r@rIrrrrrrIOErrorrrrrrrr)r)r)r,s (,             #R  !%