#!/usr/bin/python
# Copyright (c) 2010-2024 Broadcom. All Rights Reserved.
# Broadcom Confidential. The term "Broadcom" refers to Broadcom Inc.
# and/or its subsidiaries.

"""High-level operations of inspecting image metadata and modifying
   the image running on the system.
"""

import logging
import os
import sys
import tempfile

if sys.version_info[0] >= 3:
   from urllib.parse import urlparse
else:
   from urlparse import urlparse

from . import (ALLOW_DPU_OPERATION, BaseImage, Bulletin, Database,
               DepotCollection, Downloader, Errors, HostImage, IS_ESXIO,
               Metadata, PATCHER_COMP_NAME,
               PERSONALITY_MANAGER_COMPONENT_REMOVAL_ENABLED, ReleaseCollection,
               Vib, VibCollection)
from .ImageManager.Constants import (SOURCE_BASEIMAGE, SOURCE_ADDON, SOURCE_HSP,
                                     SOURCE_SOLUTION, SOURCE_USER)
from .ImageManager.HostSeeding import NoVibCacheError, ReservedVibCache
from .ImageManager.Utils import getCommaSepArg
from .Utils import ArFile, HostInfo, PathUtils, Ramdisk
from .Utils.Misc import LogLargeBuffer
from .Version import Version

from . import MIB, PYTHON_VER_STR

log = logging.getLogger("Transaction")

# The string that gets appended to an image profile name when its modified
UPDATEDPROFILESTR = "(Updated) "

TMP_DIR = '/tmp'
PATCHER_VIB_NAMES = (PATCHER_COMP_NAME, 'loadesx')
ESX_BASE_VIB = 'VMware_bootbank_esx-base_'
ESXIO_BASE_VIB = 'VMware_bootbank_esxio-base_'
NSX_SCX_BUNDLE = 'nsx-scx-bundle'
NSX_SOLUTION_ID = 'com.vmware.nsxt'

# VAPI tasks.
APPLY_TASK_ID = 'com.vmware.esx.settingsdaemon.software.apply'
STAGE_TASK_ID = 'com.vmware.esx.settingsdaemon.software.stage'
COMP_APPLY_TASK_ID = 'com.vmware.esx.settingsdaemon.components.apply'
SOLUTION_APPLY_TASK_ID = 'com.vmware.esx.settingsdaemon.solutions.apply'
SOLUTION_REMOVAL_TASK_ID = 'com.vmware.esx.settingsdaemon.solutions.remove'

# VOB IDs
VOB_SOFTWARE_APPLY = 'software.apply.succeeded'

# SEVERITY (from RFC 5424)
SEVERITY_INFO = 6
SEVERITY_NOTICE = 5
SEVERITY_CRITICAL = 2

getCommaSepArg = lambda x: ', '.join(sorted(x))

class InstallResult(object):
   """Holds the results of installation methods.
      Attributes:
         * installed  - A list of IDs for components/VIBs installed.
         * removed    - A list of IDs for components/VIBs removed.
         * skipped    - A list of IDs for components/VIBs skipped and not
                        installed. One reason is if keeponlyupdates=True was
                        passed to InstallVibsFromSources().
         * exitstate  - The exit state of the installation results.
         * stagedpath - The staged path with regards to ESXi Stage() operation.
   """
   def __init__(self, **kwargs):
      self.installed  = kwargs.pop('installed', [])
      self.removed    = kwargs.pop('removed', [])
      self.skipped    = kwargs.pop('skipped', [])
      self.exitstate  = kwargs.pop('exitstate', Errors.NormalExit())
      self.stagedpath = kwargs.pop('stagedpath', '')

class InstallResultExt(InstallResult):
   """Holds the overall installation results, including results on DPUs.
      Attributes:
         * dpuresult  - A list of Dpu Results.
   """
   def __init__(self, **kwargs):
      super().__init__(**kwargs)
      self.dpuresult = kwargs.pop('dpuresult', [])

class Transaction(object):
   """Class with methods to install, remove, and inventory VIBs, and to scan a
      host or image against one or more depots.
      Attributes:
         * host - A HostImage instance.
         * _platform - Platform of the system, e.g. embeddedEsx/esxio.
         * _enableQuickPatch - A boolean that indicates whether
                               QuickPatchInstaller is enabled.
         * _enforceQuickPatch - A boolean that indicates whether Quick Patch is
                                enforced in QuickPatchInstaller.
                                _enableQuickPatch must be True for it to be
                                effective.
   """
   def __init__(self, initInstallers=True, enableQuickPatch=False,
                enforceQuickPatch=True):
      self.host = HostImage.HostImage(
         initInstallers=initInstallers,
         enableQuickPatch=enableQuickPatch,
         enforceQuickPatch=enforceQuickPatch)
      self._platform = Vib.GetHostSoftwarePlatform()
      self._enableQuickPatch = enableQuickPatch
      self._enforceQuickPatch = enforceQuickPatch

   @staticmethod
   def DownloadMetadatas(metadataUrls, toggleFirewall=True):
      '''Downloads metadatas to temporary files, parses them and builds up a
         consolidated metadata object. If toggleFirewall is set, turn on and
         off esxupdate firewall rule before and after download.
         Returns:
            A Metadata object.
         Raises:
            MetadataDownloadError
      '''
      if toggleFirewall:
         Downloader.Downloader.setEsxupdateFirewallRule('true')
      try:
         allMeta = Metadata.Metadata()
         for metaUrl in metadataUrls:
            with tempfile.NamedTemporaryFile() as f:
               try:
                  d = Downloader.Downloader(metaUrl, f.name)
                  mfile = d.Get()
               except Downloader.DownloaderError as e:
                  raise Errors.MetadataDownloadError(metaUrl, None, str(e))

               m = Metadata.Metadata()
               m.ReadMetadataZip(mfile)
               m.SetVibRemoteLocations(d.url)
               allMeta.Merge(m)
         return allMeta
      finally:
         if toggleFirewall:
            Downloader.Downloader.setEsxupdateFirewallRule('false')

   @staticmethod
   def ParseDepots(depotUrls):
      '''Parse depot or offline bundle to build a depot collection.
         Returns:
            A DepotCollection object.
         Paramaters:
            * depotUrls - A list of URLs to depots or offline bundle zip file.
                          For offline bundle URL, only server local file path is
                          supported. Offline bundle must end with '.zip'.
         Raises:
            ValueError  - If offline bundle is not of server local file path.
            DepotConnectError - An error occurred when connecting to one of the
                                depot URLs.
      '''
      remotebundles = []
      for url in depotUrls:
         if url.lower().endswith('.zip') and not PathUtils.IsDatastorePath(url):
            scheme = urlparse(url)[0]
            if scheme not in ('file', ''):
               remotebundles.append(url)

      if remotebundles:
         msg = ('Only server local file path is supported for offline '
                'bundles. (%s) seem to be remote URIs.' % (', '.join(
                     remotebundles)))
         raise ValueError(msg)

      dc = DepotCollection.DepotCollection()
      dc.ConnectDepots(depotUrls, ignoreerror=False)
      return dc

   @staticmethod
   def GetMetadataFromUrls(metaUrls, depotUrls):
      """Given metadata zip and/or depot URLs, return a Metadata object with all
         metadata from the URLs.
      """
      metaUrls = metaUrls or []
      depotUrls = depotUrls or []
      meta = Metadata.Metadata()
      if metaUrls:
         meta.Merge(Transaction.DownloadMetadatas(metaUrls))
      if depotUrls:
         # DepotCollection has all metadata except notifications and config
         # schemas.
         meta.Merge(Transaction.ParseDepots(depotUrls),
                    excludes={Metadata.Metadata.INVENTORY_NOTIFICATIONS,
                              Metadata.Metadata.INVENTORY_CONFIGSCHEMAS})
      return meta

   def GetProfile(self, useVisorFs=False):
      '''Gets an image profile from the host with the vibs VibCollection filled
         out.
         Parameters:
            * useVisorFs - Boolean to specify if Visor FS DB should be used to
                           get the profile
         Raises:
            InstallationError if a profile is not present.
      '''
      if useVisorFs:
         # For patcher case, we are interested in the version which is still
         # running on the host. Use visorfs db for this.
         return self.host.GetProfile(database=HostImage.HostImage.DB_VISORFS)

      curprofile = self.host.GetProfile()
      if curprofile is None or len(curprofile.vibIDs) == 0:
         msg = ('No image profile is found on the host or '
                'image profile is empty. An image profile is required to '
                'install or remove VIBs.  To install an image profile, '
                'use the esxcli image profile install command.')
         raise Errors.InstallationError(None, None, msg)

      curprofile.vibs = self.host.GetInventory()
      curprofile._reservedVibs = self.host.GetReservedVibs()
      curprofile._reservedVibIDs = set(curprofile._reservedVibs.keys())
      if len(curprofile.vibs) == 0:
         msg = 'No VIB database was found on the host'
         raise Errors.InstallationError(None, None, msg)

      return curprofile

   def GetCompsAndMetaFromSources(self, nameIds, depotUrls):
      '''Get specified components and other supporting metadata from depots.
         Parameters:
            * nameIds     - A list of Component specification strings;
                            when not provided, all depot components will be
                            returned as specified.
            * depotUrls   - A list of remote URLs of the depot index.xml
                            or local offline bundle.
         Returns:
            A tuple of collections of specified components, all depot
            components, VIBs, base images, and solutions.
         Raises:
            * ValueError - If the same Component is specified as both a URL and
                           a metadata and their descriptors do not match.
            * NoMatchError - If a name ID provided does not match to any
                             component.
      '''
      dc = Transaction.ParseDepots(depotUrls)
      compCollection = Bulletin.ComponentCollection(dc.bulletins, True)

      if nameIds:
         filteredComps = Bulletin.ComponentCollection() # Components to return.
         for nameId in nameIds:
            try:
               if ':' in nameId:
                  # name:version, find exactly one component.
                  comp = compCollection.GetComponent(nameId)
                  filteredComps.AddComponent(comp)
               else:
                  # name only, find all components with the name.
                  comps = compCollection.GetComponents(nameId)
                  if not comps:
                     # Check the result is not an empty list.
                     raise KeyError
                  for comp in comps:
                     filteredComps.AddComponent(comp)
            except KeyError:
               msg = 'No component matches search criteria %s' % nameId
               raise Errors.NoMatchError(nameId, msg)
      else:
         filteredComps = compCollection

      return filteredComps, compCollection, dc.vibs, dc.baseimages, dc.solutions


   @staticmethod
   def GetVibsAndMetaFromSources(vibUrls, metaUrls, vibspecs,
                                 fillfrommeta=True, depotUrls=None,
                                 newestonly=True):
      '''Get VIBs to install and the combined metadata from VIBs, metadata zips
         and depots.
         Parameters: See InstallVibsFromSources bellow.
            * vibUrls      - A list of URLs to VIB packages
            * metadataUrls - A list of URLs to metadata.zip files
            * vibspecs     - A list of VIB specification strings;
            * fillfrommeta - If metaUrls are passed in with no vibspecs, and
                             fillfrommeta is True, then all the VIBs from
                             the meta/depot will automatically fill the list of
                             VIBs. The option does not affect components and
                             base images collection that assist VIB installation
                             by URLs.
            * depotUrls    - The full remote URLs of the depot index.xml
                             or local offline bundle.
            * newestonly   - If newestonly is True, will filter out older VIBs
                             either in depots or in vibspecs matches.
         Returns:
            A VibCollection and a ComponentCollection, which include downloaded
            VIB instances and Bulletin Instances respectively.
         Raises:
            * ValueError - If the same VIB is specified as both a URL and a
                           metadata and their descriptors do not match.
      '''
      metaUrls = metaUrls or []
      depotUrls = depotUrls or []
      vibsToInstall = Transaction._getVibsFromUrls(vibUrls)
      allMeta = Metadata.Metadata()
      allMeta.vibs += vibsToInstall
      if metaUrls or depotUrls:
         if vibspecs:
            vibsFromSpec, meta = \
               Transaction._getVibsAndMetaByNameidsFromDepot(metaUrls, vibspecs,
                                                      depoturls=depotUrls,
                                                      onevendor=True,
                                                      newestonly=newestonly)
            vibsToInstall += vibsFromSpec
         else:
            # If newestonly is set, newestVibs will be a subset of vibs,
            # otherwise they will be the same.
            newestVibs, meta = \
               Transaction._getVibsAndMetaFromDepot(metaUrls, depotUrls,
                                                    newestonly=newestonly)
            if fillfrommeta:
               vibsToInstall += newestVibs
         allMeta.Merge(meta)

      return vibsToInstall, allMeta

   def _checkApplyMaintenanceMode(self, newProfile, ignoreInstallerError=False):
      """For an image apply command, maintenance mode needs to be enabled
         if the transaction requires a reboot or contains a live VIB change
         that requires maintenance mode.
         If ignoreInstallerError is set, catch the exception of live installer
         not running and simply assume a reboot is required. This is used to
         rescue a broken image db with apply.
         This method is only for non-QuickPatch apply.
      """
      from .ImageManager.Constants import IMPACT_NONE, IMPACT_REBOOT
      from .ImageManager.Scanner import getImageProfileImpact

      try:
         impact, _ = getImageProfileImpact(self.host, newProfile)
      except Errors.InstallerNotAppropriate:
         if ignoreInstallerError:
            log.warning('Unable to determine if maintenance mode is required: '
                        'live installer is not running. Assuming a reboot is '
                        'required.')
            impact = IMPACT_REBOOT
         else:
            raise

      if impact != IMPACT_NONE and not HostInfo.GetMaintenanceMode():
         msg = ('The transaction requires the host to be in '
                'MaintenanceMode')
         raise Errors.MaintenanceModeError(msg)

   def InstallComponentsFromSources(self, componentSpecs, force=False,
                                    forcebootbank=False, stageonly=False,
                                    dryrun=False, depotUrls=None,
                                    noSigCheck=False, installSolution=False,
                                    allowDowngrades=False, taskId=None):
      '''Installs Components from depots.

         Parameters:
            * componentSpecs  - A list of Component specification strings;
                                the format is <name> or <name>:<version>.
            * depotUrls       - The full remote URLs of the depot index.xml
                                or local offline bundle. If URL ends with
                                '.zip', the URL will be treated as pointing
                                to an offline bundle and local file path is
                                required.
                                If the URL does not end in .xml, it will be
                                assumed to be a directory and 'index.xml'
                                will be added.
            * dryrun          - Do not perform stage/install; just report
                                what components will be installed/removed/
                                skipped.
            * checkmaintmode  - Check if maintenance mode is required in a
                                live install.
            * nosigcheck      - If True, Component signature
                                will not be validated. A failed validation
                                raises an exception.
            * installSolution - When set, if a solution has at least one
                                component in its constraints installed, the
                                solution will be added to the image DB.
                                In case more than one solutions are suitable,
                                the highest versioned one will be added.
            * allowDowngrades - When set, allow downgrades of components when
                                supported; existing components that obsolete
                                the downgraded ones will be removed.
            * taskId          - UUID of the task, generated in vAPI, or None
                                when a localcli call was made.
         Returns:
            An instance of InstallResult, with the installed/removed/skipped
            attributes filled out.
         Raises:
            InstallationError - Error installing the components.
            NoMatchError      - Cannot find components using componentSpecs in
                                depotUrls.
            DependencyError   - Installing the components results in invalid
                                image.
      '''
      # Avoid importing new dependencies outside esximage.zip in a legacy
      # VUM upgrade, or dependencies outside early tardisks in secureBoot.
      from lifecycle.task import Task

      if taskId is not None:
         task = Task(taskId, COMP_APPLY_TASK_ID)
      else:
         # Allows reporting and does not write to disk.
         task = Task('local-component-apply', COMP_APPLY_TASK_ID, taskFile=None)
      task.startTask()

      try:
         self.host._getLock()
      except Exception as e:
         task.failTask(_getExceptionNotification(e))
         raise

      try:
         depotUrls = depotUrls or []
         addComps, allComps, allVibs, baseImages, solutions = \
            self.GetCompsAndMetaFromSources(componentSpecs, depotUrls)
         task.setProgress(10)

         dpuRes = dict()
         if ALLOW_DPU_OPERATION:
            from .ESXioImage.ImageOperations import componentsApplyOnDPUs

            # installSolution is only set for esxcli
            isEsxcli = not installSolution
            exitState, dpuRes = componentsApplyOnDPUs(componentSpecs, depotUrls,
                  esxcli=isEsxcli, dryRun=dryrun, noSigCheck=noSigCheck)
            if not isinstance(exitState, Errors.NormalExit):
               # TODO: error notification for DpuOperationError.
               raise exitState

         curProfile = self.GetProfile()
         newProfile = curProfile.Copy()

         task.setProgress(20)

         # XXX: include vibs for other platform currently installed at boot
         allVibs += curProfile.vibs

         # Generate the new image profile:
         # 1) Adjust components in the image profile with the specified ones.
         #    Reserved components are re-calculated in this process.
         newProfile.AddComponents(addComps, allVibs,
                                  replace=allowDowngrades,
                                  partialReservedVibs=True)

         # 2) Check if we have unsupported component downgrades when downgrades
         #    are allowed.
         hasConfigDowngrade = False
         if allowDowngrades:
            try:
               _checkComponentDowngrades(curProfile, newProfile)
            except Errors.ComponentConfigDowngrade as e:
               hasConfigDowngrade = True
               log.info('Allowing potential config downgrade in component(s): '
                        '%s', ', '.join(e.components))

         # 3) Pick the correct base image.
         _setProfileBaseImage(newProfile, curProfile, None, baseImages)

         # 4) Pick suitable solution(s) to add to the new image profile.
         if installSolution:
            # PR2980429, fail ApplyHA if host is in "rebootRequired" state
            if self.host.imgstate == self.host.IMGSTATE_BOOTBANK_UPDATED:
               raise Errors.ApplyHARebootRequiredError('Failed to apply HA due '
                  'to the host software configuration requires a reboot. '
                  'Reboot the host and try again.')
            # The component being applied may already exists, but corresponding
            # solution(s) need to be enabled.
            _addSolutionsToProfile(newProfile, addComps, solutions)

         # 5) If base image has been changed, reserved metadata should be
         #    re-calculated again in case of a partial upgrade.
         if newProfile.baseimageID != curProfile.baseimageID:
            newProfile.PopulateReservedMetadata(
               allComps + newProfile.GetKnownComponents(),
               allVibs + newProfile.GetKnownVibs(),
               ignoreMissingVibs=True)

         # Calculate diffs and update profile info.
         addSet = set(addComps.GetComponentIds())
         added, removed = newProfile.DiffComponents(curProfile)
         skipped = list(addSet - set(added) - set(removed))

         sortedAdded = sorted(added)
         msg = ('The following Components have been applied:\n%s'
                % '\n'.join(addComps.GetComponentNameIds(sortedAdded)))
         self._updateProfileInfo(newProfile, msg, force)
         task.setProgress(30)

         log.info("Final list of components being installed: %s" %
                  ', '.join(sortedAdded))
         stagedPath, exitstate = self._validateAndInstallProfile(
            newProfile,
            curProfile,
            force,
            forcebootbank,
            stageonly,
            dryrun,
            nosigcheck=noSigCheck,
            checkCompDeps=True,
            hasConfigDowngrade=hasConfigDowngrade)
         task.setProgress(95)

         res = InstallResultExt(installed=added, removed=removed, skipped=skipped,
                                dpuresult=dpuRes, exitstate=exitstate)
         self.host.SendVob(VOB_SOFTWARE_APPLY, len(added), len(removed))

         task.completeTask()
         return res
      except Exception as e:
         task.failTask(_getExceptionNotification(e))
         raise
      finally:
         self.host._freeLock()

   def _isNoMatchErrors(self, res):
      """Verify NoMatchError in output received from managed DPUs
         for failure cases.
            Parameters:
               * res: Received managed DPUs output.
            Returns:
               A tuple of (isNoMatchError, dpuOutput)
               isNoMatchError - True if one or more DPUs have NoMatchError.
                                False if none of the DPUs have NoMatchError.
               dpuOutput - Result from the successful DPUs.
                           None if operation failed on all DPUs.
      """
      if not res:
         return False, None

      isNoMatchError = False
      dpuOutput = dict()
      for dpuId, result in res.items():
         if '[NoMatchError]' in result['output']:
            isNoMatchError = True
         elif result['status'] == 0:
            dpuOutput = {dpuId: result}
      return isNoMatchError, dpuOutput

   def RemoveComponents(self, nameIds, force=False,
                        forcebootbank=False, dryrun=False):
      """Removes one or more Components from the host image. Validation
         will be performed before the actual stage and install.
            Parameters:
               * nameIds      - list of component name IDs for removal, in name
                                or name:version format.
               * dryrun       - Do not perform stage/install;
                                just report on what Components
                                will be installed/removed/skipped
            Returns:
               An instance of InstallResult.
            Raises:
               InstallationError - error removing the Component
               DependencyError   - removing the Component results
                                   in an invalid image
               NoMatchError      - The Component ID does not exist on the host
      """
      curProfile = self.GetProfile()
      newProfile = curProfile.Copy()

      allComponents = curProfile.components
      rmComponents = Bulletin.ComponentCollection()

      allVibs = curProfile.GetKnownVibs()

      for nameId in nameIds:
         try:
            comp = allComponents.GetComponent(nameId)
            comp = Bulletin.Component.FromBulletin(comp)
            rmComponents.AddComponent(comp)
         except KeyError:
            msg = ('Component %s is not found in the current image profile'
                   % nameId)
            raise Errors.NoMatchError(nameId, msg)

      # Handle component removal and VIB adjustments.
      newProfile.RemoveComponents(rmComponents, partialReservedVibs=True)

      self.host._getLock()
      try:
         dpuRes = dict()
         if ALLOW_DPU_OPERATION:
            # DPU operation
            from .ESXioImage.ImageOperations import componentRemoveOnDPUs
            exitState, dpuRes = componentRemoveOnDPUs(nameIds, dryRun=dryrun)
            if not isinstance(exitState, Errors.NormalExit):
               # There can be one DPU where the operation is successful. So,
               # show the output of that DPU.
               isNoMatchError, dpuRes = self._isNoMatchErrors(dpuRes)
               if not isNoMatchError:
                  raise exitState

         # Update profile info
         msg = ('The following Components have been removed:\n%s'
                % '\n'.join(rmComponents.GetComponentNameIds()))
         self._updateProfileInfo(newProfile, msg, force)

         stagedPath, exitstate = \
                     self._validateAndInstallProfile(newProfile,
                                                     curProfile,
                                                     force,
                                                     forcebootbank,
                                                     dryrun=dryrun,
                                                     checkCompDeps=True)

         added, removed = newProfile.DiffComponents(curProfile)
         self.host.SendVob(VOB_SOFTWARE_APPLY, len(added), len(removed))

         return InstallResultExt(installed=added, removed=removed,
                                 dpuresult=dpuRes, exitstate=exitstate)
      finally:
         self.host._freeLock()

   def RemoveSolutions(self, solutionNames, taskId):
      """Removes solutions and their components from the host.
         Parameters:
            solutionNames - names of Solutions to remove.
            taskId        - UUID of the VAPI solution remove task.
         Returns:
            An instance of InstallResult, with the installed and skipped
            attributes filled out.
         Raises:
            LockingError          - contention when locking esximg lock.
            SolutionNotFound      - one or more Solutions are not found by the
                                    given names.
            InstallationError     - an error occurred during the remediation.
            LiveInstallationError - an error occurred when disabling a live VIB.
      """
      # Avoid importing new dependencies outside esximage.zip in a legacy
      # VUM upgrade, or dependencies outside early tardisks in secureBoot.
      from lifecycle.task import Task

      task = Task(taskId, SOLUTION_REMOVAL_TASK_ID)
      task.startTask()
      try:
         self.host._getLock()
      except Exception as e:
         task.failTask(_getExceptionNotification(e))
         raise

      try:
         curProfile = self.GetProfile()
         newProfile = curProfile.Copy()
         task.setProgress(10)

         if ALLOW_DPU_OPERATION:
            from .ESXioImage import ImageOperations
            exitState = ImageOperations.removeSolutionOnDPUs(
                           solutionNames, task)
            if not isinstance(exitState, Errors.NormalExit):
               task.completeTask()
               return InstallResult(skipped=list(solutionNames),
                                    exitstate=exitState)

         # Solution remove VAPI does not fail on solution not found.
         rmSols = newProfile.RemoveSolutions(solutionNames, ignoreNotFound=True,
                                             partialReservedVibs=True)
         added, removed = newProfile.DiffComponents(curProfile)
         exitstate = Errors.HostNotChanged()
         if rmSols:
            # Perform actual change when at least one solution is present.
            rmSolIds = (set(curProfile.solutions.keys()) -
                        set(newProfile.solutions.keys()))
            msg = ('The following Solution(s) have been removed:\n%s'
                   % '\n'.join(sorted(rmSolIds)))
            if removed:
               msg += ('\n\nThe following Solution Component(s) have been '
                       'removed:\n%s' % '\n'.join(sorted(removed)))
            self._updateProfileInfo(newProfile, msg, False)
            task.setProgress(20)

            stagedPath, exitstate = \
                        self._validateAndInstallProfile(newProfile,
                                                        curProfile,
                                                        False,
                                                        False,
                                                        checkCompDeps=True)
            task.setProgress(90)

            self.host.SendVob(VOB_SOFTWARE_APPLY, len(added), len(removed))

         res = InstallResult(installed=added, removed=removed,
                             exitstate=exitstate)
         task.completeTask()
         return res
      except Exception as e:
         task.failTask(_getExceptionNotification(e))
         raise
      finally:
         self.host._freeLock()

   def ApplySolutions(self, imageSpec, depotUrls, taskId=None,
                      dryrun=False, nosigcheck=False):
      """Applies solutions and their components on the host.
         Parameters:
            imageSpec     - Desired image spec, a JSON dictionary.
                            Note: Only the 'solutions' field is processed.
            depotUrls     - URLs of depot to connect and fetch metadata from.
            taskId        - UUID of the VAPI solution apply task, or None
                            when a localcli call was made.
            dryrun        - Do not perform stage/install; just report
                            what components will be installed/removed/
                            skipped.
            nosigcheck    - If True, signatures of component VIBs
                            will not be validated. Otherwise, a failed
                            validation raises an exception.
         Raises:
            LockingError          - contention when locking esximg lock.
            SolutionNotFound      - one or more Solutions are not found by the
                                    given names.
            InstallationError     - an error occurred during the remediation.
            LiveInstallationError - an error occurred when enabling a live VIB.
      """
      # Avoid importing new dependencies outside esximage.zip in a legacy
      # VUM upgrade, or dependencies outside early tardisks in secureBoot.
      from lifecycle.task import Task

      if taskId is not None:
         task = Task(taskId, COMP_APPLY_TASK_ID)
      else:
         # Allows reporting and does not write to disk.
         task = Task('local-component-apply', COMP_APPLY_TASK_ID, taskFile=None)
      task.startTask()

      try:
         self.host._getLock()
      except Exception as e:
         task.failTask(_getExceptionNotification(e))
         raise

      try:
         depotUrls = depotUrls or []
         _, _, allVibs, _, solutions = \
            self.GetCompsAndMetaFromSources(None, depotUrls)
         task.setProgress(10)

         curProfile = self.GetProfile()
         newProfile = curProfile.Copy()

         task.setProgress(20)

         finalComponents = getSolutionComponents(imageSpec, depotUrls,
                                                 curProfile.components)
         newProfile.AddComponents(finalComponents, allVibs,
                                  replace=True, partialReservedVibs=True)

         # PR 3393476
         # NSX SCX component is installed by default on a host which is part of
         # VUM cluster configured for NSX. SCX drift is being ignored on VC side
         # to allow NSX apply to proceed. While performing NSX apply, the SCX
         # component will be uninstalled as scx and lcp bundles can't co-exist
         for key in imageSpec['solutions']:
            if key == NSX_SOLUTION_ID:
               removeComp = Bulletin.ComponentCollection()
               if curProfile.components.HasComponent(NSX_SCX_BUNDLE):
                  log.info('Removing component [%s] as part of solution '
                           'apply.', NSX_SCX_BUNDLE)
                  removeComp.AddComponent(
                     curProfile.components.GetComponent(NSX_SCX_BUNDLE)
                  )
                  newProfile.RemoveComponents(removeComp)
                  break

         if not dryrun and ALLOW_DPU_OPERATION:
            from .ESXioImage import ImageOperations
            exitState = ImageOperations.applySolutionOnDPUs(
                           imageSpec['solutions'], depotUrls, task)
            if not isinstance(exitState, Errors.NormalExit):
               task.failTask(_getExceptionNotification(exitState))
               return InstallResult(installed=[], removed=[],
                                    skipped=list(newProfile.componentIDs),
                                    exitstate=exitState)

         # Calculate diffs and update profile info.
         addSet = set(finalComponents.GetComponentIds())
         added, removed = newProfile.DiffComponents(curProfile)
         skipped = list(addSet - set(added) - set(removed))

         # Pick suitable solution(s) to add to the new image profile.
         # The component being applied may already exists, but corresponding
         # solution(s) need to be enabled.
         _addSolutionsToProfile(newProfile, finalComponents, solutions)

         newSolIds = (set(newProfile.solutions.keys()) -
                      set(curProfile.solutions.keys()))
         msg = ('The following Solution(s) have been applied:\n%s'
                % '\n'.join(sorted(newSolIds)))
         if added:
            msg += ('\n\nThe following Solution Component(s) have been applied:'
                    '\n%s' % '\n'.join(sorted(added)))
         self._updateProfileInfo(newProfile, msg)
         task.setProgress(30)

         log.info("Final list of solution components being installed: %s",
                  ', '.join(added))
         stagedPath, exitstate = \
                     self._validateAndInstallProfile(newProfile,
                                                     curProfile,
                                                     dryrun=dryrun,
                                                     nosigcheck=nosigcheck,
                                                     checkCompDeps=True)
         task.setProgress(95)

         res = InstallResult(installed=added, removed=removed, skipped=skipped,
                             exitstate=exitstate)
         self.host.SendVob(VOB_SOFTWARE_APPLY, len(added), len(removed))

         task.completeTask()
         return res
      except Exception as e:
         task.failTask(_getExceptionNotification(e))
         raise
      finally:
         self.host._freeLock()

   def InstallVibsFromSources(self, vibUrls, metaUrls, vibspecs,
                              fillfrommeta=True, keeponlyupdates=False,
                              force=False, forcebootbank=False,
                              stageonly=False, dryrun=False, depotUrls=[],
                              checkmaintmode=True, nosigcheck=False,
                              downloadedMeta=None, esxcli=False):
      '''Installs VIBs from either direct URLs or from metadata.
         VIBs from both URLs and metadata are merged.  If the same VIB
         is specified as both a URL and a metadata and their descriptors
         do not match, an error results.
         **This method is a patch-the-patcher interface that is called by an
         older ESXi, parameter/return change must be backward-compatible.
            Parameters:
               * vibUrls      - A list of URLs to VIB packages
               * metaUrls     - A list of URLs to metadata.zip files
               * vibspecs     - A list of VIB specification strings; can be
                                <name>, <vendor>:<name>, <name>:<version>,
                                <vendor>:<name>:<version>.  Note that
                                if the spec matches multiple packages,
                                and they are from different vendors,
                                an error results. If they are from the same
                                vendor but different versions, the highest
                                versioned package is taken (since multiple
                                versions cannot be installed at once)
               * fillfrommeta - If metaUrls are passed in with no vibspecs, and
                                fillfrommeta is True, then the newest VIBs from
                                the meta will automatically fill the list of
                                VIBs to install. If False, no VIBs will be
                                populated from meta when there are no vibspecs.
                                The VIBs from vibUrls will contribute either
                                way.
               * keeponlyupdates - If True, only VIBs which update VIBs on the
                                   host will be kept as part of the transaction.
                                   The rest will be skipped.
               * force - Skips most image profile validation checks
               * forcebootbank  - Force a bootbank install even if VIBs are
                                  live installable
               * depotUrls      - The full remote URLs of the depot index.xml
                                  or local offline bundle. If URL ends with
                                  '.zip', the URL will be treated as pointing to
                                  an offline bundle and local file path is
                                  required. If the URL does not end in .xml, it
                                  will be assumed to be a directory and
                                  'index.xml' will be added.
               * dryrun       - Do not perform stage/install; just report on
                                what VIBs will be installed/removed/skipped.
               * checkmaintmode - Check maintenance mode if required by live
                                  install.
               * nosigcheck - If True, VIB signature will not be validated.
                              A failed validation raises an exception.
               * downloadedMeta - metadata already downloaded, expect a
                                  VibCollection and a Metadata instance.
                                  **This parameter is only used in
                                  non-patch-the-patcher scenario and can change.
               * esxcli       - True if the apply was called from esxcli,
                                False when called from VUM or the caller
                                has not provided the option.
            Returns:
               An instance of InstallResultExt, with the installed and skipped
               attributes filled out.
            Raises:
               InstallationError - Error installing the VIBs
               NoMatchError      - Cannot find VIBs using vibspecs in metaUrls
               DependencyError   - Installing the VIBs results in invalid image
      '''
      self.host._getLock()
      try:
         curprofile = self.GetProfile()
         if downloadedMeta:
            log.debug('Metadata is provided, skip download')
            vibs, meta = downloadedMeta
         else:
            vibs, meta = \
               self.GetVibsAndMetaFromSources(vibUrls, metaUrls, vibspecs,
                                              fillfrommeta, depotUrls)
         knownVibs, newComponents, newBaseImages = \
            (meta.vibs, meta.components, meta.baseimages)

         # Scan and keep only the updates
         if keeponlyupdates:
            allvibs = VibCollection.VibCollection()
            allvibs += vibs
            allvibs += curprofile.vibs
            updates = allvibs.Scan().GetUpdatesSet(curprofile.vibs)
            skiplist = list(set(vibs.keys()) - updates)
            log.info("Skipping non-update VIBs %s" % (', '.join(skiplist)))
         else:
            # skip installed VIBs
            skiplist = list(set(vibs.keys()) & set(curprofile.vibs.keys()))
            log.info("Skipping installed VIBs %s" % (', '.join(skiplist)))

         for vibid in skiplist:
            vibs.RemoveVib(vibid)

         # Remove the VIBs that are not applicable to this platform from the
         # skiplist that the user will see. This is done after RemoveVib so
         # the metadata of these VIBs will still be kept in image DB.
         nonApplicableVibs = vibs.GetNonApplicableVibs(self._platform)
         if nonApplicableVibs:
            # skip VIBs not belonging to the current platform
            skiplist.extend(nonApplicableVibs.keys())
            log.info('Skipping non-applicable VIBs %s', nonApplicableVibs.keys())

         log.info("Final list of VIBs being installed: %s" %
                  (', '.join(vibs.keys())))

         if esxcli:
            for vib in vibs:
               if ESX_BASE_VIB in vib or ESXIO_BASE_VIB in vib:
                  raise Errors.EsxVersionChangeError(vib,
                                    "ESXi version change is not allowed using"
                                    " esxcli software vib commands. Please use"
                                    " a supported method to upgrade ESXi.")

         dpuRes = dict()
         if ALLOW_DPU_OPERATION:
            # DPU operation.
            # Only esxcli route will hit since PatchManager blocks InstallV2
            # when there is DPU.
            from .ESXioImage.ImageOperations import vibInstallUpdateOnDPUs
            exitState, dpuRes = vibInstallUpdateOnDPUs(vibUrls, vibspecs, depotUrls,
                  isInstallCmd=not keeponlyupdates, dryRun=dryrun, force=force,
                  noLiveInstall=forcebootbank, checkMaintMode=checkmaintmode,
                  noSigCheck=nosigcheck)
            if not isinstance(exitState, Errors.NormalExit):
               raise exitState

         inst, removed, exitstate = self._installVibs(curprofile,
                                                vibs,
                                                force,
                                                forcebootbank,
                                                stageonly=stageonly,
                                                dryrun=dryrun,
                                                checkmaintmode=checkmaintmode,
                                                nosigcheck=nosigcheck,
                                                refMetadata=meta)

         # Remove nonApplicableVibs from current platform's installedList
         inst = list(set(inst) - set(nonApplicableVibs))
         # See #bora/apps/addvob/addvob.c for the vob format string.
         self.host.SendVob("vib.install.successful", len(inst), len(removed))

         return InstallResultExt(installed=inst, removed=removed, skipped=skiplist,
                                 dpuresult=dpuRes, exitstate=exitstate)
      finally:
         self.host._freeLock()

   @staticmethod
   def _getVibsFromUrls(urls):
      '''Create VIBs instances from urls for metadata usage.
      '''
      Downloader.Downloader.setEsxupdateFirewallRule('true')
      try:
         vibs = VibCollection.VibCollection()
         for url in urls:
            if not urlparse(url).scheme and not os.path.isabs(url):
               msg = "Not a valid absolute path: %s" % url
               raise Errors.VibDownloadError(url, None, msg)
            try:
               d = Downloader.Downloader(url)
               with d.Open() as rObj:
                  # ArFile will not auto-populate with a non-seekable fobj,
                  # parse the first header of the VIB manually and then read
                  # the XML.
                  arFile = ArFile.ArFile(fileobj=rObj, mode='rb')
                  if arFile.seekable():
                     # Local file will have members populated with offsets
                     if not arFile.filelist:
                        raise Errors.VibFormatError(url,
                                                    'The ar file is empty')
                     arInfo = arFile.filelist[0]
                     rObj.seek(arInfo.offset, 0)
                  else:
                     arInfo = arFile._parseHeader()
                  if arInfo.filename != 'descriptor.xml':
                     raise Errors.VibFormatError(url,
                                     'The first member of the ar file is %s, '
                                     'descriptor.xml expected'
                                     % arInfo.filename)
                  xmlStr = rObj.read(arInfo.size)
               vib = Vib.ArFileVib.FromXml(xmlStr)
               vib.remotelocations.append(url)
               vibs.AddVib(vib)
            except Downloader.DownloaderError as e:
               raise Errors.VibDownloadError(url, None, str(e))
            except ArFile.ArError as e:
               raise Errors.VibFormatError(url, str(e))
      finally:
         Downloader.Downloader.setEsxupdateFirewallRule('false')
      return vibs

   @staticmethod
   def _getVibsAndMetaByNameidsFromDepot(metaurls, nameids,
                                         depoturls=None, onevendor=False,
                                         newestonly=False):
      '''Get VIBs that match the nameids and the combined metadata from
         metadata zips and depots.
         newestonly - If True, return the latest VIBs matching nameids otherwise
                      all the VIBs for each match are included.
         metaurls - a list of paths to metadata.zip's.
         nameids - a list of <name> / <name>:<version> / <vendor>:<name> VIB
                   spec strings
         onevendor - only allow matches from one vendor for each nameid
      '''
      meta = _getMetadataFromMetaDepotUrls(metaurls, depoturls)
      allVibs = meta.vibs

      vibs = VibCollection.VibCollection() # VIBs to return
      for nameid in nameids:
         try:
            matches = allVibs.FindVibsByColonSpec(nameid, onevendor=onevendor)
         except ValueError as e:
            raise Errors.NoMatchError(nameid, str(e))

         if len(matches) == 0:
            raise Errors.NoMatchError(nameid, "Unable to find a VIB that "
                                      "matches '%s'." % (nameid))
         if newestonly:
            # Since we have only one vendor and name, if there are multiple
            # matches they must be multiple versions of the same package.
            # Find the highest versioned one for installation.
            vib = max((allVibs[v].version, allVibs[v]) for v in matches)[1]
            vibs.AddVib(vib)
         else:
            for v in matches:
               vibs.AddVib(allVibs[v])

      return vibs, meta

   @staticmethod
   def _getVibsAndMetaFromDepot(metaurls, depoturls, newestonly=False):
      """Get VIBs and the combined metadata from metadata zips and depots.
         If newestonly is True, return the latest VIBs in the depots/metas and
         separately all VIBs, otherwise return two copies of all VIBs.
      """
      if newestonly:
         vibselection = 'newest'
      else:
         vibselection = 'all'

      LogLargeBuffer("Populating VIB list from %s VIBs in metadata "
                     "'%s'; depots '%s'" % (vibselection,
                     ', '.join(metaurls), ', '.join(depoturls)), log.debug)

      meta = _getMetadataFromMetaDepotUrls(metaurls, depoturls)
      allVibs = meta.vibs

      if newestonly:
         result = allVibs.Scan()
         vibids = result.GetNewestSet()
         newestVibs = VibCollection.VibCollection(
            (vid, allVibs[vid]) for vid in vibids)
      else:
         newestVibs = allVibs

      return newestVibs, meta

   def _installVibs(self, curProfile, vibs, force,
                    forcebootbank, stageonly=False, dryrun=False,
                    checkmaintmode=True, nosigcheck=False, deploydir=None,
                    refProfile=None, refMetadata=None,
                    isoLowerVersionVibs=None):
      """Add vibs to the current profile, validate it, then install it
         returns a list of VIB IDs installed, VIB IDs removed and install
         exitstate.
         refProfile - used in VUM upgrade and profile install/update to supply
         the ISO/target image profile which contains components, base image,
         addon and reserved VIBs/components.
         refMetadata - used in VIB install/update and VUM rollup to help
         setting base image, addon, components and reserved metadata.
         isoLowerVersionVibs - List of VIBs in the ISO that have a lower version
         than the VIB version installed on the host. These VIBs are skipped
         during the VUM ISO upgrade process.
      """
      def checkFdmConfigDowngrade(curProfile, newProfile):
         """Checks FDM VIB downgrade, returns True if FDM is the only
            downgrading VIB with config schema, False if there is no downgrade
            or there are other downgrades.
            If FDM VIB downgrade is detected with other downgrades involving
            config schema, raise an exception as this is not a supported
            workflow; HA is supposed to initiate a FDM-only VIB change.
         """
         FDM_COMP = 'vsphere-fdm'
         FDM_VIB = 'vmware-fdm'

         compDowngrades = curProfile.GetCompsDowngradeInfo(newProfile)
         orphanVibDowngrades = curProfile.GetOrphanVibsDowngradeInfo(newProfile)

         if not compDowngrades and not orphanVibDowngrades:
            return False

         # Determine if FDM is downgrading in terms of config schema, either
         # as a component or as a standalone VIB.
         # This heavily rely on FDM not changing its VIB/component name.
         compNames = set([name for name, v in compDowngrades.items() if v[4]])
         hasFdm = FDM_COMP in compNames

         vibNames = set()
         for vibName, (_, _, compName, configSchema) in \
               orphanVibDowngrades.items():
            if configSchema:
               if vibName == FDM_VIB:
                  hasFdm = True
               if compName:
                  compNames.add(compName)
               else:
                  vibNames.add(vibName)

         if hasFdm and (len(compNames) + len(vibNames) > 1):
            # FDM is supposed to be installed by itself by HA via installv2 API,
            # if there are multiple config downgrades including FDM, we are not
            # equipped to handle it.
            otherComps, otherVibs = compNames.copy(), vibNames.copy()
            otherComps.discard(FDM_COMP)
            otherVibs.discard(FDM_VIB)
            msg = 'Downgrade of FDM VIB/Component with '
            if compNames and vibNames:
               msg += ('Components(s) %s and VIB(s) %s '
                  % (getCommaSepArg(otherComps), getCommaSepArg(otherVibs)))
            elif compNames:
               msg += 'Components(s) %s ' % getCommaSepArg(otherComps)
            else:
               msg += 'VIB(s) %s ' %  getCommaSepArg(otherVibs)
            if vibNames:
               compNames.add('(Standalone VIB)')
            msg += 'is not supported due to possible config incompatibility.'
            raise Errors.ComponentConfigDowngrade(sorted(compNames), msg)

         return hasFdm


      newProfile = curProfile.Copy()
      refVibs = curProfile.GetKnownVibs()

      try:
         newProfile.AddVibs(vibs, replace=True)
      except KeyError:
         msg = 'One or more of the VIBs %s is already on the host.' % \
               (', '.join(v.name for v in vibs.values()))
         raise Errors.HostNotChanged(msg)

      # Prevent downgrade with vib install.
      #  Let full validation take care of missing esx-version.
      Transaction._checkEsxVersionDowngrade(newProfile,
                                            allowMissingEsxVersion=True)

      refComps = None
      if refProfile:
         refComps = refProfile.GetKnownComponents()
         refVibs += refProfile.GetKnownVibs()

      if refMetadata:
         if refMetadata.vibs:
            refVibs += refMetadata.vibs
         newBaseImages = refMetadata.baseimages
         newAddons = refMetadata.addons
         newComponents = refMetadata.components
      else:
         newBaseImages, newAddons, newComponents = None, None, None

      if newComponents:
         refComps = refComps + newComponents if refComps else newComponents

      # Set base image in newProfile
      _setProfileBaseImage(newProfile, curProfile, refProfile, newBaseImages)

      # Re-calculate components and reserve components according to VIBs
      # in the image profile.
      # Existing reserved components may still be reserved without access to all
      # of their VIBs.
      newProfile.SyncComponents(refComps=refComps, refVibs=refVibs,
                                partialReservedVibs=True)

      # Filter out solely the skipped VIBs that are associated with the base
      # image and addon. The reserved VIBs comprise the final list of VIBs
      # that should be reserved, and the intersection of the skipped VIBs
      # and the reserved VIBs should result in the final list of VIBs to cache
      # from the ISO.
      vibsToCache = None
      if isoLowerVersionVibs:
         commonVibIDs = (isoLowerVersionVibs.keys() &
                         newProfile.reservedVibs.keys())
         vibsToCache = {vibId:isoLowerVersionVibs[vibId]
                        for vibId in commonVibIDs}
      # Set addon in newProfile, components must be set before this step.
      addonChanged = _setProfileAddon(newProfile, refProfile, newAddons)
      if addonChanged:
         # Populate reserved metadata again if the addon has changed.
         # Existing reserved components may still be reserved without access to
         # all of their VIBs.
         newProfile.PopulateReservedMetadata(refComps, refVibs,
                                             ignoreMissingVibs=True)

      if PERSONALITY_MANAGER_COMPONENT_REMOVAL_ENABLED:
         # set removed components in newProfile
         _setRemovedComponents(newProfile, curProfile, refProfile)

      # Whether need to handle FDM config downgrade.
      hasConfigDowngrade = checkFdmConfigDowngrade(curProfile, newProfile)

      # update profile info
      self._updateProfileInfo(newProfile,
                              'The following VIBs are installed:\n%s' % (
                                 '\n'.join('  %s\t%s' % (v.name, v.versionstr)
                                 for v in vibs.values())),
                              force)
      _, exitstate = self._validateAndInstallProfile(
            newProfile, curProfile, force, forcebootbank,
            stageonly=stageonly,
            dryrun=dryrun,
            checkmaintmode=checkmaintmode,
            nosigcheck=nosigcheck,
            deploydir=deploydir,
            hasConfigDowngrade=hasConfigDowngrade,
            vibsToCache=vibsToCache)

      adds, removes = newProfile.Diff(curProfile)
      return adds, removes, exitstate

   def _applyImageProfile(self, imageProfile, dryRun, stageOnly, forceBootbank,
                          noSigCheck):
      """Apply an desired image profile and returns the components added,
         removed and the exit state.
         This function should be called with component-based image profile
         install. Otherwise, _validateAndInstallProfile() should normally be
         used.

         Returns:
            compsAdd   - Components added.
            compsRmd   - Components removed.
            stagedPath - The path to the staged contents. None if it is
                         a dry run.
            exitstate  - The exit state of the operation.
      """
      compNameVer = [(c.compNameStr, c.compVersionStr)
                     for c in imageProfile.bulletins.values()]
      applyMsg = ('The following components are applied on the system:\n%s'
                  % '\n'.join(['  %s\t%s' % t for t in compNameVer]))
      self._updateProfileInfo(imageProfile, applyMsg)

      try:
         curProfile = self.GetProfile()
      except (Errors.AcceptanceConfigError, Errors.InstallationError):
         # Broken image db, apply helps getting out of it.
         curProfile = None
         # Make sure ESX version is not downgrading.
         Transaction._checkEsxVersionDowngrade(imageProfile)
      else:
         # Check for component downgrade.
         _checkComponentDowngrades(curProfile, imageProfile)

      stagedPath, exitState = self._validateAndInstallProfile(imageProfile,
                                                  curprofile=curProfile,
                                                  force=False,
                                                  forcebootbank=forceBootbank,
                                                  stageonly=stageOnly,
                                                  dryrun=dryRun,
                                                  checkmaintmode=True,
                                                  nosigcheck=noSigCheck,
                                                  checkCompDeps=True)

      log.debug('Finished _validateAndInstallProfile()')
      if curProfile:
         added, removed = imageProfile.DiffComponents(curProfile)
      else:
         added, removed = imageProfile.componentIDs, set()
      self.host.SendVob(VOB_SOFTWARE_APPLY, len(added), len(removed))

      return added, removed, stagedPath, exitState

   def _validateAndInstallProfile(self, newprofile, curprofile=None,
                                  force=False,
                                  forcebootbank=False,
                                  stageonly=False, dryrun=False,
                                  checkmaintmode=True,
                                  nosigcheck=False,
                                  deploydir=None,
                                  checkCompDeps=False,
                                  hasConfigDowngrade=False,
                                  vibsToCache=None):
      """Remediates a previously staged image.  The exact behavior depends on
         the *Installer subclasses carrying out the work. Maintenance Mode is
         checked if any VIBs require it. The databases will be locked for
         writing; therefore only one process may invoke Remediate() at a time.

         Parameters:
            newprofile         - Desired image profile.
            curprofile         - Current image profile.
            force              - When set, this skips image validations and
                                 signature checks.
            forcebootbank      - When set, force a bootbank-only transaction
                                 that requires a reboot; no effect with a
                                 reboot-required transaction.
            stageonly          - When set, the desired image is staged on the
                                 host to a temporary path ready to be copied
                                 (installed) at a later time to the bootbank.
            dryrun             - When set, report components to be added/
                                 removed/skipped and do not perform the action.
            checkmaintmode     - Boolean, check that the ESX host is in
                                 maintenance mode if the VIBs require it.
            nosigcheck         - When set, VIB signatures will not be validated.
            deploydir          - Directory which contain image in deploy format,
                                 such as ISO and PXE. When supplied, database
                                 and payloads will be fetched from the
                                 directory.
            checkCompDeps      - Boolean, when set, raise dependency issues with
                                 component info instead of VIB info.
            hasConfigDowngrade - Boolean, set when the transaction contains
                                 downgrade of config, which will affect behavior
                                 of the installers.
            vibsToCache        - List of vibs to cache under reservedVibs
                                 folder for seeding.
         Returns:
            stagedPath         - The path to the staged contents. None if it is
                                 a dry run.
            exitstate          - The exit state of the operation
      """
      skipvalidation = force
      systemUpdate = not (stageonly or dryrun)
      skipAuditEvent = False
      auditSeverity = SEVERITY_INFO # Default audit severity is Informational.
      auditStartMsg = None
      auditVibAdds = []
      auditVibRemoves = []

      if systemUpdate:
         # Prepare general audit information
         if curprofile:
            adds, removes = newprofile.Diff(curprofile)
            auditVibAdds = sorted(adds)
            auditVibRemoves = sorted(removes)
         else:
            # With curprofile missing, removes is empty
            auditVibAdds = sorted(list(newprofile.vibIDs))
            auditVibRemoves = []

         if force or nosigcheck:
            # Start success note, record any attempt to bypass signature check.
            auditStartMsg = self.host.AUDIT_NOTE_NOSIG_IGNORED if \
                            HostInfo.IsHostSecureBooted() else \
                            self.host.AUDIT_NOTE_NOSIG
            auditSeverity = SEVERITY_NOTICE # Elevate the default audit severity to Notice.
         else:
            auditStartMsg = None

      # Audit events.
      def sendAuditEvent(eventID, result, auditSeverity, vibID=None, msg=None):
         nonlocal skipAuditEvent
         if systemUpdate and not skipAuditEvent:
            try:
               self.host.SendAuditEvent(eventID, result, auditSeverity, msg, vibID)
            except HostImage.NoAuditLogError as e:
               skipAuditEvent = True

      def sendSuccessAuditEvents(adds, removes, auditSeverity, msg=None):
         """Function to send system.update audit events when transaction is successful.
         Parameters:
            adds - Vibs that are added to image profile.
            removes - Vibs that are removed from image profile.
            auditSeverity - Severity of the audit event.
            msg - Message of this object is used to form the reason field.
         """
         sendAuditEvent(self.host.AUDIT_START_EVENTID, True,
                        auditSeverity, msg=auditStartMsg)
         for vib in adds:
            sendAuditEvent(self.host.AUDIT_ADD_EVENTID, True, auditSeverity, vibID=vib)
         for vib in removes:
            sendAuditEvent(self.host.AUDIT_REMOVE_EVENTID, True, auditSeverity, vibID=vib)
         sendAuditEvent(self.host.AUDIT_END_EVENTID, True, auditSeverity, msg=msg)

      # Warn if trying to do an install with no validation
      if force:
         if HostInfo.IsHostSecureBooted():
            msg = ("Secure Boot enabled: Signatures will be checked.")
            log.warning(msg)
            self.host.SendConsoleMsg(msg)
            skipvalidation = False
            nosigcheck = False

         msg = ("Attempting to install an image profile with "
                "validation disabled. This may result in an image "
                "with unsatisfied dependencies, file or package "
                "conflicts, and potential security violations.")
         # See #bora/apps/addvob/addvob.c for the vob format string.
         self.host.SendVob("install.novalidation")
         log.warning(msg)

      if nosigcheck and HostInfo.IsHostSecureBooted():
         msg = ("UEFI Secure Boot enabled: Cannot skip signature checks. "
                "Installing unsigned VIBs will prevent the system from "
                "booting. So the vib signature check will be enforced.")

         self.host.SendVob("install.nobypasssigcheck")
         log.warning(msg)
         skipvalidation = False
         nosigcheck = False

      # Warn acceptance check is disabled
      checkacceptance = not (skipvalidation or nosigcheck)
      if not checkacceptance:
         msg = ("Attempting to install an image profile bypassing signing and "
                "acceptance level verification. This may pose a large "
                "security risk.")

         self.host.SendVob("install.nosigcheck")
         log.warning(msg)

      noextrules = force or nosigcheck

      # validate and generate vfatname
      problems = newprofile.Validate(nodeps=force,
                                     noconflicts=force,
                                     allowobsoletes=force,
                                     noacceptance=skipvalidation,
                                     allowfileconflicts=force,
                                     noextrules=noextrules,
                                     depotvibs=newprofile.GetKnownVibs(),
                                     checkCompDeps=checkCompDeps,
                                     coreCompCheck=not force)
      if problems:
         # extract first group of problems with highest priority and raise
         PRIORITY_MAPPING = {0: Errors.ProfileValidationError,
                             1: Errors.VibValidationError,
                             2: Errors.AcceptanceConfigError,
                             3: Errors.DependencyError}

         priority = problems[0].priority
         instances = list(filter(lambda x: x.priority == priority, problems))
         e = PRIORITY_MAPPING[priority](problemset=instances)
         raise e

      newprofile.GenerateVFATNames(curprofile, platform=self._platform)

      # Store reserved VIBs for host seeding on ESXi hosts.
      resVibCache = None
      # The stagebootbank path to be returned back as a result of stage-task.
      stagedPath = None
      if systemUpdate and not IS_ESXIO:
         # Store VIBs for host seeding on ESXi only.
         try:
            resVibCache = ReservedVibCache()
            resVibCache.cacheVibs(self.host, newprofile, deploydir, vibsToCache)
         except NoVibCacheError:
            # No reserved VIB cache location is not fatal.
            log.info('There is no location to store reserved VIBs, skip adding '
                     'new reserved VIBs')

      # install and remediate
      try:
         stagedPath = self.host.Stage(newprofile, forcebootbank=forcebootbank,
                         dryrun=dryrun, stageonly=stageonly,
                         checkacceptance=checkacceptance,
                         deploydir=deploydir)
         if systemUpdate:
            self.host.Remediate(newprofile, checkmaintmode,
                                hasConfigDowngrade=hasConfigDowngrade)
            stagedPath = None
         exitstate = Errors.NormalExit()
      except (Errors.QuickPatchExit, Errors.NormalExit) as e:
         exitstate = e
      except Exception as e:
         if resVibCache:
            # Restore reserved VIB cache.
            resVibCache.revert()
         # Not an exit state, raise the error
         raise

      if resVibCache:
         # Finalize reserved VIB cache.
         resVibCache.finalize()

      # Update completed
      sendSuccessAuditEvents(auditVibAdds, auditVibRemoves, auditSeverity)
      return stagedPath, exitstate

   def RemoveVibs(self, vibids, force=False, forcebootbank=False, dryrun=False,
                  checkmaintmode=True):
      """Removes one or more VIBs from the existing host image.  Validation
         will be performed before the actual stage and install.
            Parameters:
               * vibids         -  list of VIB IDs to remove.
               * force -  Skips most image profile validation checks
               * forcebootbank  -  Force a bootbank install even if VIBs are
                                   live installable
               * dryrun         - Do not perform stage/install; just report on
                                  what VIBs will be installed/removed/skipped
               * checkmaintmode - Check if maintenance mode is required by live
                                  removal
            Returns:
               An instance of InstallResultExt.
            Raises:
               InstallationError - error removing the VIB
               DependencyError   - removing the VIB results in an invalid image
               NoMatchError      - The VIB ID does not exist on the host
      """
      curprofile = self.GetProfile()
      newprofile = curprofile.Copy()

      rmVibs = VibCollection.VibCollection()
      for vibid in vibids:
         try:
            rmVibs.AddVib(curprofile.vibs[vibid])
            newprofile.RemoveVib(vibid)
         except KeyError:
            raise Errors.NoMatchError('',
                                      "VIB ID %s not found in current image "
                                      "profile" % vibid)

      self.host._getLock()
      try:
         # update profile info
         self._updateProfileInfo(newprofile,
                              'The following VIBs have been removed:\n%s' % (
                              '\n'.join('  %s\t%s' % (curprofile.vibs[v].name,
                              curprofile.vibs[v].versionstr) for v in vibids)),
                                 force)

         # Re-calculate components and reserve components/VIBs.
         # Existing reserved components may still be reserved without all of its
         # VIBs available.
         newprofile.SyncComponents(refVibs=rmVibs, partialReservedVibs=True)

         dpuRes = dict()
         if ALLOW_DPU_OPERATION:
            # DPU operation.
            from .ESXioImage.ImageOperations import vibRemoveOnDPUs

            # We already searched which VIBs to remove, just need to form the
            # name:version specs for the DPU.
            nameSpecs = ['%s:%s' % (v.name, v.version.versionstring)
                         for v in rmVibs.values()]

            exitState, dpuRes = vibRemoveOnDPUs(nameSpecs, dryRun=dryrun,
                                    force=force, noLiveInstall=forcebootbank,
                                    checkMaintMode=checkmaintmode)
            if not isinstance(exitState, Errors.NormalExit):
               # There can be one DPU where the operation is successful. So,
               # show the output of that DPU.
               isNoMatchError, dpuRes = self._isNoMatchErrors(dpuRes)
               if not isNoMatchError:
                  raise exitState

         stagedPath, exitstate = \
                     self._validateAndInstallProfile(newprofile, curprofile,
                                                     force, forcebootbank,
                                                     dryrun=dryrun,
                                                  checkmaintmode=checkmaintmode)
         # See #bora/apps/addvob/addvob.c for the vob format string.
         self.host.SendVob("vib.remove.successful", len(vibids))
         return InstallResultExt(removed=vibids, dpuresult=dpuRes, exitstate=exitstate)
      finally:
         self.host._freeLock()

   @staticmethod
   def GetProfileAndMetaFromSources(profilename, creator=None,
                                    metadataUrls=None, depotUrls=None):
      """Returns an image profile and the combined metadata from metadata zips
         and depots. The image profile will be populated with all infomation.
         Parameters:
            * profilename  - name of the image profile to look for.
            * creator      - image profile creator.  Used to help refine the
                             search for image profiles if multiple names match.
            * metadataUrls - a list of URLs to metadata.zips
            * depotUrls    - The full remote URLs of the depot index.xml
                             or local offline bundle.
         Returns:
            An ImageProfile and a Metadata instance.
         Raises:
            NoMatchError - if profilename is not found, matches more than
                           once, or cannot populate metadata of the image
                           profile.
      """
      meta = Transaction.GetMetadataFromUrls(metadataUrls, depotUrls)

      matches = meta.profiles.FindProfiles(name=profilename, creator=creator)
      if len(matches) == 0:
         raise Errors.NoMatchError(profilename,
            "No image profile found with name '%s'" % (profilename))
      elif len(matches) > 1:
         raise Errors.NoMatchError(profilename,
            "More than one image profile found with name '%s'" % (profilename))
      newprofile = list(matches.values())[0]

      # Populate VIB instances of image profile
      missingVibs = newprofile.vibIDs - set(meta.vibs.keys())
      if missingVibs:
         msg = ("VIBs %s from image profile '%s' cannot be found "
                "in metadata."
                % (', '.join(list(missingVibs)), newprofile.name))
         raise Errors.NoMatchError(profilename, msg)
      newprofile.PopulateVibs(meta.vibs)

      # Populate components belong to this profile
      missingBulls = newprofile.componentIDs - set(meta.bulletins.keys())
      if missingBulls:
         msg = ("Bulletins %s from image profile '%s' cannot be found "
                "in metadata."
                % (', '.join(list(missingBulls)), newprofile.name))
         raise Errors.NoMatchError(profilename, msg)
      newprofile.PopulateComponents(bulletins=meta.bulletins)

      if newprofile.baseimageID is not None:
         try:
            newprofile.PopulateBaseImage(meta.baseimages)
         except KeyError:
            msg = ("Base Image '%s' from image profile '%s' cannot be found "
                   "in metadata." % (newprofile.baseimageID, newprofile.name))
            raise Errors.NoMatchError(newprofile.name, msg)

      if newprofile.addonID is not None:
         try:
            newprofile.PopulateAddon(meta.addons)
         except KeyError:
            msg = ("Addon '%s' from image profile '%s' cannot be found "
                   "in metadata." % (newprofile.addonID, newprofile.name))
            raise Errors.NoMatchError(newprofile.name, msg)

      if newprofile.manifestIDs:
         try:
            newprofile.PopulateManifests(meta.manifests)
         except KeyError as e:
            msg = ("%s from image profile '%s'" % (str(e).replace("'", ""),
                                                   newprofile.name))
            raise Errors.NoMatchError(newprofile.name, msg)

      allComps = Bulletin.ComponentCollection(meta.bulletins,
                                              ignoreNonComponents=True)
      newprofile.PopulateReservedMetadata(allComps, meta.vibs)
      newprofile.PopulatePlatformInfo()

      return newprofile, meta

   def InstallProfile(self, metadataUrls, profileName, depotUrls=None,
                      force=False, forcebootbank=False,
                      dryrun=False, checkmaintmode=True,
                      allowRemovals=True, nosigcheck=False, nohwwarning=None,
                      **kwargs):
      """Installs an image profile from online or offline zip depot sources.
         During transaction, current image will be completely replaced by the
         new image profile. The VIBs in the image profile will be evaluated
         against the host acceptance level.
         **This method is a patch-the-patcher interface that is called by an
         older ESXi, parameter/return change must be backward-compatible.
            Parameters:
               * metadataUrls   - metadata.zip urls, no op for this function
               * profileName    - Name of image profile to install
               * depotUrls      - The full remote URLs of the depot index.xml
                                  or local offline bundle.
               * force          - Skips most image profile validation checks
               * forcebootbank  - Force a bootbank install even if VIBs are
                                  live installable, no effect for profile
                                  install as it is always reboot required
               * dryrun         - Do not perform stage/install; just report on
                                  what VIBs will be installed/removed/skipped
               * checkmaintmode - Check if maintenance mode is required by live
                                  install, no effect for profile install as
                                  it is bootbank only.
               * allowRemovals  - If True, allows profile installs to remove
                                  installed VIBs.  If False, will raise an error
                                  if profile install leads to removals
               * nosigcheck     - If True, VIB signature will not be validated
               * nohwwarning    - If True, do not show hardware precheck
                                  warnings.
               * kwargs         - Discarded options for patch-the-patcher
                                  backward compatibility.
                                  WARNING: only an option that has never made
                                  into a product branch can be discarded.
         Returns:
            An instance of InstallResultExt.
         Raises:
            DowngradeError    - If ESXi version downgrade is not supported
            InstallationError - If there is an error in transaction
            NoMatchError      - If profilename matches more than one image
                                profile or if there is no match
            ProfileVibRemoval - If installing the profile causes VIBs to be
                                removed and allowRemovals is False
      """
      self.host._getLock() # Could be an re-entry.
      try:
         newProfile, _ = Transaction.GetProfileAndMetaFromSources(
                                                      profileName,
                                                      metadataUrls=metadataUrls,
                                                      depotUrls=depotUrls)

         # Set the image profile acceptance level to the host acceptance value
         # and evaluate the VIBs inside against that
         newProfile.acceptancelevel = self.host.GetHostAcceptance()

         # Execute upgrade prechecks.
         # For an 6.7 host, only catch-up VmkLinux checks. We know this by
         # not receiving a True/False value for nohwwarning.
         # VmkLinux checks return only errors, no warnings.
         vmkLinuxOnly = nohwwarning is None
         noHwWarning = False if nohwwarning is None else nohwwarning
         Transaction._runUpgradePrecheck(newProfile, vmkLinuxOnly,
                                         noHwWarning)

         dpuRes = dict()
         if ALLOW_DPU_OPERATION:
            # Run on DPUs
            from .ESXioImage.ImageOperations import profileInstallOnDPUs

            res, dpuRes = profileInstallOnDPUs(profileName, depotUrls,
                           dryRun=dryrun, force=force,
                           noLiveInstall=forcebootbank,
                           checkMaintMode=checkmaintmode,
                           okToRemove=allowRemovals, noSigCheck=nosigcheck,
                           noHardwareWarn=nohwwarning)

            if not isinstance(res, Errors.NormalExit):
               raise res

         try:
            curProfile = self.GetProfile()
            adds, removes = newProfile.Diff(curProfile)
         except (Errors.AcceptanceGetError, Errors.InstallationError):
            curProfile = None
            adds = list(newProfile.vibIDs)
            removes = []

         skiplist = list(set(newProfile.vibs.keys()) - set(adds))

         # Detect if any VIBs will be removed due to installing the profile
         if curProfile:
            # Preserve original installdate and signature for skipped VIBs.
            for vibid in skiplist:
               newVib = newProfile.vibs[vibid]
               curVib = curProfile.vibs[vibid]
               newVib.installdate = curVib.installdate
               newVib.SetSignature(curVib.GetSignature())
               newVib.SetOrigDescriptor(curVib.GetOrigDescriptor())

            _, _, gone, _ = newProfile.ScanVibs(curProfile.vibs)
            if len(gone) > 0 and not allowRemovals:
               msg = "You attempted to install an image profile which would " \
                     "have resulted in the removal of VIBs %s. If this is " \
                     "not what you intended, you may use the esxcli software " \
                     "profile update command to preserve the VIBs above. " \
                     "If this is what you intended, please use the " \
                     "--ok-to-remove option to explicitly allow the removal." \
                     % gone
               raise Errors.ProfileVibRemoval(newProfile.name, gone, msg)
         else:
            log.warning("No existing image profile, will not be able to detect "
                        "if any VIBs are being removed.")

         stagedPath, exitstate = \
                     self._validateAndInstallProfile(newProfile, force=force,
                                   forcebootbank=forcebootbank, dryrun=dryrun,
                                   checkmaintmode=checkmaintmode,
                                   nosigcheck=nosigcheck)

         # Create an ESXi boot option.
         self._createUefiBootOption()

         # See #bora/apps/addvob/addvob.c for the vob format string.
         self.host.SendVob("profile.install.successful", newProfile.name,
               len(adds), len(removes))
         return InstallResultExt(installed=adds, removed=removes, skipped=skiplist,
                                 dpuresult=dpuRes, exitstate=exitstate)
      finally:
         self.host._freeLock()

   def InstallVibsFromDeployDir(self, deploydir, dryrun=False):
      """Installs all the VIBs from an image in deployment format,
         such as ISO and PXE.
         VIBs that are not present on the host or upgrade an existing VIB
         will be installed.
         This is currently used during VUM upgrade where all contents
         of an ISO are in a ramdisk.

         Parameters:
            * deploydir - directory with image in deploy format
            * dryrun    - only report on what VIBs will be
                          installed/removed/skipped
         Returns:
            An instance of InstallResult.
         Raises:
            InstallationError
      """
      # Locate image database
      DB_FILE = 'IMGDB.TGZ'
      dbpaths = [os.path.join(deploydir, DB_FILE),
                 os.path.join(deploydir, DB_FILE.lower())]
      db = None
      try:
         for dbpath in dbpaths:
            if os.path.exists(dbpath):
               db = Database.TarDatabase(dbpath)
               db.Load()
               break
      except (Errors.DatabaseIOError, Errors.DatabaseFormatError) as e:
         raise Errors.InstallationError(e, None, str(e))

      # No image db found
      if not db:
         raise Errors.InstallationError(None, None,
                  "Failed to locate image database in folder %s." % deploydir)

      self.host._getLock()
      try:
         newprofile = db.profile
         newprofile.PopulateWithDatabase(db)
         curprofile = self.GetProfile()

         # Store source payload localname for staging use
         for vibid, vib in newprofile.vibs.items():
            vibstate = newprofile.vibstates[vibid]
            for payload in vib.payloads:
               if payload.name in vibstate.payloads:
                  payload.localname = vibstate.payloads[payload.name]

         # Use new profile name as target profile name
         curprofile.name = newprofile.name

         # Figure out which VIBs to update or add
         _, downgrades, _, existing = curprofile.ScanVibs(
                                                               newprofile.vibs)
         skipped = set(existing) | set(downgrades)
         vibstoinstall = VibCollection.VibCollection((vid, newprofile.vibs[vid])
                         for vid in newprofile.vibIDs if vid not in skipped)
         isoLowerVersionVibs = VibCollection.VibCollection(
                        (vid, newprofile.vibs[vid])
                        for vid in newprofile.vibIDs if vid in set(downgrades))

         installed, removed, exitstate = self._installVibs(
                                          curprofile,
                                          vibstoinstall,
                                          False, False,
                                          dryrun=dryrun,
                                          deploydir=deploydir,
                                          refProfile=newprofile,
                                          isoLowerVersionVibs=isoLowerVersionVibs)

         # Create an ESXi boot option.
         self._createUefiBootOption()

         return InstallResult(installed=installed, removed=removed,
                              skipped=skipped, exitstate=exitstate)
      finally:
         self.host._freeLock()

   def InstallVibsFromProfile(self, metadataUrls, profileName, depotUrls=None,
                              force=False, forcebootbank=False, dryrun=False,
                              checkmaintmode=True, allowDowngrades=False,
                              nosigcheck=False, nohwwarning=None, **kwargs):
      """Installs all the VIBs in an image profile from online or offline zip
         depot sources.
         As opposed to InstallProfileFromDepot(), VIBs which are
         unrelated to VIBs in the target image profile will be preserved;
         VIBs of higher version in the target image profile will be installed;
         VIBs of lower version in the target image will be installed only with
         allowDowngrades.
         **This method is a patch-the-patcher interface that is called by an
         older ESXi, parameter/return change must be backward-compatible.
         Parameters:
            Same as InstallProfile(), except:
            * allowRemovals    - this parameter is not supported here.
            * allowDowngrades  - if True, VIBs in the new profile which
                                 downgrade existing VIBs will be part of the
                                 combined profile.
                                 if False (default), VIBs which downgrade will
                                 be skipped.
            * kwargs           - Discarded options for patch-the-patcher
                                 backward compatibility.
                                 WARNING: only an option that has never made
                                 into a product branch can be discarded.
         Returns:
            An instance of InstallResultExt.
         Raises:
            DowngradeError
            InstallationError
      """
      # Could be an re-entry in Patch the Patcher from before 7.0 U3.
      self.host._getLock(enableQuickPatch=self._enableQuickPatch)
      try:
         curProfile = self.GetProfile()
         newProfile, meta = \
            Transaction.GetProfileAndMetaFromSources(profileName,
                                                     metadataUrls=metadataUrls,
                                                     depotUrls=depotUrls)
         # Use new profile name as target profile name
         curProfile.name = newProfile.name

         # Figure out which VIBs to update or add
         _, downgrades, _, existing = curProfile.ScanVibs(newProfile.vibs)
         skiplist = set(existing)
         if not allowDowngrades:
            skiplist |= downgrades
         vibstoinstall = VibCollection.VibCollection((vid, newProfile.vibs[vid])
            for vid in newProfile.vibIDs if vid not in skiplist)

         # Execute upgrade prechecks.
         # Unlike profile install, also check the system is actually about
         # to change with VIBs to be installed.
         if vibstoinstall:
            # While the final image profile will be assembled later in
            # _installVibs, we don't want to complicate it more with a new flag.
            finalProfile = curProfile.Copy()
            finalProfile.AddVibs(vibstoinstall, replace=True)
            # For an 6.7 host, only catch-up VmkLinux checks. We know this by
            # not receiving a True/False value for nohwwarning.
            # VmkLinux checks return only errors, no warnings.
            vmkLinuxOnly = nohwwarning is None
            noHwWarning = False if nohwwarning is None else nohwwarning
            Transaction._runUpgradePrecheck(finalProfile, vmkLinuxOnly,
                                            noHwWarning)

         dpuRes = dict()
         if ALLOW_DPU_OPERATION:
            # Run on DPUs
            from .ESXioImage.ImageOperations import profileUpdateOnDPUs

            res, dpuRes = profileUpdateOnDPUs(profileName, depotUrls,
                                              dryRun=dryrun, force=force,
                                              noLiveInstall=forcebootbank,
                                              checkMaintMode=checkmaintmode,
                                              allowDowngrades=allowDowngrades,
                                              noSigCheck=nosigcheck,
                                              noHardwareWarn=nohwwarning)

            if not isinstance(res, Errors.NormalExit):
               raise res

         inst, removed, exitstate = self._installVibs(
                                                curProfile,
                                                vibstoinstall, force,
                                                forcebootbank, dryrun=dryrun,
                                                checkmaintmode=checkmaintmode,
                                                nosigcheck=nosigcheck,
                                                refProfile=newProfile,
                                                refMetadata=meta)

         # Create an ESXi boot option.
         self._createUefiBootOption()

         # See #bora/apps/addvob/addvob.c for the vob format string.
         log.debug("Finished self._installVibs")
         self.host.SendVob("profile.update.successful", newProfile.name,
                           len(inst), len(removed))
         log.debug("Finished SendVob")

         return InstallResultExt(installed=inst, removed=removed,
                                 skipped=skiplist, dpuresult=dpuRes,
                                 exitstate=exitstate)
      finally:
         self.host._freeLock()

   def ScanImage(self, imageSpec, depotUrls, taskId):
      """Preparation of scanning a desired image using a spec.
         This method takes care of patch the patcher before invoking Scanner.
         Parameters:
            imageSpec       - Desired image spec, a JSON dictionary.
            depotUrls       - URLs of depot to connect and fetch metadata from.
            taskId          - UUID of the task, generated in vAPI, or by
                              localcli when a local call is made.
      """
      # Avoid importing new dependencies outside esximage.zip in a legacy
      # VUM upgrade, or dependencies outside early tardisks in secureBoo
      from .ImageManager.Scanner import SCAN_TASK_ID
      from lifecycle.task import Task

      # TODO:
      # 1. lifecycle task sync file locking.
      # 2. Sort out the return of the scan, currently the return is written
      #    in the task file.

      if taskId is not None:
         task = Task(taskId, SCAN_TASK_ID)
      else:
         # Allows reporting and does not write to disk.
         task = Task('local-scan', SCAN_TASK_ID, taskFile=None)
      task.startTask()

      # Lock so we do not run into any apply.
      # The side effect is that scan will not be concurrent.
      try:
         self.host._getLock(enableQuickPatch=self._enableQuickPatch)
      except Exception as e:
         task.failTask(_getExceptionNotification(e))
         raise

      upgradeDir = None
      upgradeTardisks = []
      try:
         newPatcherComp, newPatcherVibs = \
            self._getPatcherWrapper(
               *_getPatcherFromImageSpec(imageSpec, depotUrls))
         liveProfile = self.GetProfile(useVisorFs=True)
         oldPatcherComp, oldPatcherVibs = self._getPatcherWrapper(
            *_findPatcher(liveProfile))

         task.setProgress(15)

         # Proceed with scan with this esximage when the patcher version
         # has not updated in the target image.
         if not _requiresNewPatcher(oldPatcherComp, oldPatcherVibs,
                                    newPatcherComp, newPatcherVibs):
            from .ImageManager import DepotMgr, Scanner
            depotSpec = DepotMgr.getDepotSpecFromUrls(depotUrls)

            task.setProgress(20)
            Scanner.HostScanner(imageSpec, depotSpec, task,
                                enableQuickPatch=self._enableQuickPatch).scan()
            return

         # Use no-sig-check to allow scanning across GA and test builds.
         upgradeDir, upgradeTardisks = self._setupPatcher(newPatcherVibs,
                                                          False,
                                                          True)
         from esximage.ImageManager import DepotMgr as NewDepotMgr
         from esximage.ImageManager import Scanner as NewScanner

         log.info('Invoking Scanner of the new esximage library')
         depotSpec = NewDepotMgr.getDepotSpecFromUrls(depotUrls)

         task.setProgress(20)
         NewScanner.HostScanner(imageSpec, depotSpec, task,
                                enableQuickPatch=self._enableQuickPatch).scan()
      except Exception as e:
         # Scan generally should not throw and exception, if it does,
         # mark task as failed.
         task.failTask(_getExceptionNotification(e))
         raise
      finally:
         self._cleanupPatcher(upgradeDir, upgradeTardisks)
         self.host._freeLock()

   def ApplyImage(self, imageSpec, depotUrls, taskId, dryRun=False,
                  stageOnly=False, forceBootbank=False, noSigCheck=False,
                  runHwCheck=False, noHwWarning=False, esxcli=False):
      """Preparation of applying a desired image using a spec.
         This method takes care of spec translation, patch the patcher,
         and calling the actual apply method (which also performs hardware
         prechecks).
         Parameters:
            imageSpec       - Desired image spec, a JSON dictionary.
            depotUrls       - URLs of depot to connect and fetch metadata from.
            taskId          - UUID of the task, generated in vAPI, or by
                              localcli when a local call is made.
            dryRun          - When set, report components to be added/removed/
                              skipped and do not perform the action.
            stageOnly       - When set, the desired image is staged on the host
                              to a temporary path ready to be copied (installed)
                              at a later time to the bootbank.
            forceBootbank   - When set, force a bootbank-only transaction that
                              requires a reboot; no effect with a
                              reboot-required transaction.
            noSigCheck      - When set, VIB signatures will not be validated.
            runHwCheck      - When set, run hardware precheck; this needs to be
                              set explicitly in case of a standalone localcli
                              call.
            noHwWarning     - When set, ignore hardware precheck warnings,
                              errors are still shown; no effect when runHwCheck
                              is not set.
            esxcli          - Set when the call comes in from esxcli.
      """
      # Avoid importing new dependencies outside esximage.zip in a legacy
      # VUM upgrade, or dependencies outside early tardisks in secureBoot.
      from lifecycle.task import Task

      # TODO: lifecycle task sync file locking.

      if taskId is not None:
         if stageOnly:
            task = Task(taskId, STAGE_TASK_ID)
         else:
            task = Task(taskId, APPLY_TASK_ID)
      else:
         # Allows reporting and does not write to disk.
         if stageOnly:
            task = Task('local-stage', STAGE_TASK_ID, taskFile=None)
         else:
            task = Task('local-apply', APPLY_TASK_ID, taskFile=None)
      task.startTask()

      try:
         self.host._getLock(task=task, enableQuickPatch=self._enableQuickPatch,
            enforceQuickPatch=self._enforceQuickPatch)
      except Errors.LockingError as e:
         task.failTask(_getExceptionNotification(e))
         raise

      upgradeDir = None
      upgradeTardisks = []

      def _completeApplyImageTask(res):
         if stageOnly:
            # Ensure backward compatibility for patch-the-patcher scenarios.
            try:
               from com.vmware.esx.settings_daemon_client import Software
               from .ImageManager.Scanner import vapiStructToJson
               stageInfoObj = Software.StageInfo(res.stagedpath, None)
               task.completeTask(result=vapiStructToJson(stageInfoObj))
            except ImportError:
               task.completeTask()
            log.info("PerformImageApply (stage only) completed "
                     "successfully.")
         else:
            ex = res.exitstate
            goodStates = ('NormalExit', 'QuickPatchExit', 'NeedsRebootResult',
                          'HostNotChanged')
            if ex.__class__.__name__ in goodStates:
               task.completeTask()
               log.info("PerformImageApply completed successfully.")
            else:
               task.failTask(_getExceptionNotification(ex))
               log.info("PerformImageApply failed with %s.", str(ex))

      try:
         try:
            liveProfile = self.GetProfile(useVisorFs=True)
         except (Errors.AcceptanceConfigError, Errors.InstallationError) as e:
            # Broken image db, apply helps getting out of it.
            log.warning('No valid image profile on host: %s, image db might '
                        'be corrupted. Skip component downgrade check and force'
                        ' patch the patcher.', str(e))
            liveProfile = None
            # Always patch the patcher.
            useNewPatcher = True
         else:
            newPatcherComp, newPatcherVibs = \
               self._getPatcherWrapper(
                  *_getPatcherFromImageSpec(imageSpec, depotUrls))
            oldPatcherComp, oldPatcherVibs = self._getPatcherWrapper(
               *_findPatcher(liveProfile))
            useNewPatcher = _requiresNewPatcher(oldPatcherComp, oldPatcherVibs,
                                                newPatcherComp, newPatcherVibs)
         task.setProgress(15)

         if not useNewPatcher:
            # Proceed with apply with this esximage when the patcher version
            # has not updated in the target image.

            newProfile = getImageProfileFromSpec(imageSpec, depotUrls)
            task.setProgress(30)
            res = self.PerformImageApply(newProfile, dryRun,
                                         stageOnly, True, forceBootbank,
                                         noSigCheck, runHwCheck, noHwWarning,
                                         # New 8.0 parameters
                                         esxcli=esxcli, task=task,
                                         imageSpec=imageSpec,
                                         depotUrls=depotUrls)
            _completeApplyImageTask(res)
            return res

         upgradeDir, upgradeTardisks = self._setupPatcher(newPatcherVibs, False,
                                                          noSigCheck)
         task.setProgress(20)

         from esximage import Transaction as NewTransaction

         # The image spec needs to be translated again with new code.
         newProfile = NewTransaction.getImageProfileFromSpec(imageSpec,
                                                             depotUrls)
         task.setProgress(30)
         log.info('Invoking PerformImageApply() of the new esximage library')
         newTransaction = \
            NewTransaction.Transaction(enableQuickPatch=self._enableQuickPatch,
                  enforceQuickPatch=self._enforceQuickPatch)
         res = newTransaction.PerformImageApply(newProfile, dryRun,
                                                stageOnly, True,
                                                forceBootbank, noSigCheck,
                                                runHwCheck, noHwWarning,
                                                # New 8.0 parameters
                                                esxcli=esxcli, task=task,
                                                imageSpec=imageSpec,
                                                depotUrls=depotUrls)
         _completeApplyImageTask(res)
         return res
      except Exception as e:
         task.failTask(_getExceptionNotification(e))
         raise
      finally:
         self._cleanupPatcher(upgradeDir, upgradeTardisks)
         self.host._freeLock()

   def PerformImageApply(self, imageProfile, dryRun, stageOnly, allowDowngrades,
                         forceBootbank, noSigCheck, runHwCheck, noHwWarning,
                         esxcli=False, task=None, imageSpec=None,
                         depotUrls=None):
      """Apply a desired image, called after patch the patcher when applicable.
         **This method is a patch-the-patcher interface that is called by an
         older ESXi, parameter/return change must be backward-compatible.
         Parameters:
            imageProfile - The translated image profile of the desired image
                           spec. When patch the patcher is required,
                           translation must be done using the new library code.
            allowDowngrades - Option exists solely to keep compatiblity with 7.0
                              GA. Behavior wise, _applyImageProfile() checks
                              downgrades and allows supported ones.
            Other parameters until noHwWarning are the same as
            ImageApplyPreAction(), the setup function should have all of them.

            Below are parameters added after the initial release of VLCM, they
            must be optional to support upgrade from an older ESXi.

            Added to support DPU operation when applying on ESXi >= 8.0:
            esxcli - True if the apply was called from esxcli, False when from
                     VAPI or the caller has not provided the option.
            task - The task object to update notification.
            imageSpec - Desired image spec, a JSON dictionary.
            depotUrls - URLs of depot to connect and fetch metadata from.
         Returns:
            installResultExt - Object holding the results of installation
      """
      if runHwCheck:
         # Run hardware precheck for standalone CLI call.
         self._runUpgradePrecheck(imageProfile, False, noHwWarning)

      if not dryRun and not stageOnly and not self._enableQuickPatch:
         # For non-QuickPatch apply, check if the host is in maintenance mode
         # if the VIBs to install/remove require either mmode or reboot.
         # Quick Patch maintenance mode check for conflicting partial mmode
         # will be executed later in QuickPatchInstaller to avoid executing
         # Quick Patch scan scripts twice.
         self._checkApplyMaintenanceMode(
            imageProfile,
            # Allows an apply to go through with a broken image db.
            ignoreInstallerError=True)

      # For DPU apply operation, esxcli supports dry-run, and neither esxcli
      # nor VAPI supports staging.
      toRunOnDpus = not stageOnly and (esxcli or not dryRun)

      dpuRes = dict()
      if ALLOW_DPU_OPERATION and toRunOnDpus:
         # Run apply on DPUs.
         # ALLOW_DPU_OPERATION is always false on ESXi 7.0.
         from .ESXioImage.ImageOperations import applyOnDPUs
         profComps = imageProfile.GetKnownComponents()
         componentsToDownload = \
            [(name, ver) for name in profComps for ver in profComps[name]]
         exitState, dpuRes = applyOnDPUs(
               imageSpec, depotUrls, componentsToDownload,
               esxcli=esxcli, parentTask=task, dryRun=dryRun,
               noLiveInstall=forceBootbank, noSigCheck=noSigCheck,
               noHardwareWarn=noHwWarning)

         if not isinstance(exitState, Errors.NormalExit):
            if esxcli:
               raise exitState
            return InstallResultExt(installed=[], removed=[],
                                    skipped=list(imageProfile.componentIDs),
                                    dpuresult=[], exitstate=exitState)

      compsAdd, compsRmd, stagedPath, exitState = self._applyImageProfile(
                                                              imageProfile,
                                                              dryRun,
                                                              stageOnly,
                                                              forceBootbank,
                                                              noSigCheck)

      skipComps = list(set(imageProfile.componentIDs) - set(compsAdd))
      return InstallResultExt(installed=compsAdd, removed=compsRmd,
                              skipped=skipComps, exitstate=exitState,
                              dpuresult=dpuRes, stagedpath=stagedPath)

   def _getPatcherWrapper(self, patcherComp, patcherVibs):
      """Wrapper for get/find patcher method.
         TODO: refactor all patcher helpers into LivePatcherMount
      """
      if not patcherComp.HasPlatform(self._platform):
         raise ValueError('Patcher component %s does not support platform %s'
                          % (patcherComp.id, self._platform))
      return patcherComp, patcherVibs.GetVibsForSoftwarePlatform(self._platform)

   @staticmethod
   def _mountVibInRamdisk(metaVib, ramdiskName, ramdiskPath, uniquePostfix,
                          checkAcceptance):
      """Mount one VIB in an existing ramdisk, return the names of tardisks
         mounted.
         With checkAcceptance, the signature of the VIB will be checked.
      """
      tardiskNames = []

      localPath = os.path.join(ramdiskPath, metaVib.id)
      Downloader.Downloader.setEsxupdateFirewallRule('true')
      try:
         d = Downloader.Downloader(metaVib.remotelocations[0],
                                   local=localPath)
         localPath = d.Get()
      except Exception as e:
         raise Errors.VibDownloadError(d.url, None, str(e))
      finally:
         Downloader.Downloader.setEsxupdateFirewallRule('false')

      vibObj = Vib.ArFileVib.FromFile(localPath)
      try:
         try:
            vibObj.VerifyAcceptanceLevel()
         except Errors.VibSignatureError as e:
            if checkAcceptance:
               raise
            else:
               log.warning('Ignoring signature error for %s: %s',
                           vibObj.id, str(e))

         # Gunzip payloads to the ramdisk
         for payload, sfp in vibObj.IterPayloads():
            # Avoid name clash when mounting
            tarName = '-'.join([payload.name, uniquePostfix])
            tarPath = os.path.join(ramdiskPath, tarName)

            with open(tarPath, 'wb') as fobj:
               Vib.copyPayloadFileObj(payload, sfp, fobj, decompress=True,
                                      checkdigest=True)

            # Mount tardisk and attach to the temp ramdisk
            Ramdisk.MountTardiskInRamdisk(localPath, payload.name, tarPath,
                                          ramdiskName, ramdiskPath,
                                          checkAcceptance=checkAcceptance)
            tardiskNames.append(tarName)
      finally:
         vibObj.Close()

      return tardiskNames

   @staticmethod
   def _setupPatcher(patcherVibs, force, nosigcheck):
      """"Download/validate esx-update VIB and mount its tardisks to
         kick off image apply commands with the new library.
         Returns path to the temp ramdisk and a list of mounted tardisks,
         or None with empty list when the VIB cannot be found.

         Side effect: esximage and weasel libraries from the VIB are located
                      and added to sys.path for import.
      """
      # Signature check of patcher VIBs is skipped with force or nosigcheck;
      # however, with UEFI secureboot, the check is always executed.
      secureBooted = HostInfo.IsHostSecureBooted()
      userSkipSigCheck = force or nosigcheck
      checkAcceptance = secureBooted or not userSkipSigCheck
      if secureBooted and userSkipSigCheck:
         log.warning('SecureBoot is enabled, signature of patcher VIB '
                     'will be checked.')

      # Use lower pid digits to create a unique ramdisk
      uniquePostfix = str(os.getpid())[-7:]
      #  Any change to the ramdisk name must sync with __init__.py.
      ramdiskName = '%s-%s' % (PATCHER_COMP_NAME, uniquePostfix)
      ramdiskPath = os.path.join(TMP_DIR, ramdiskName)
      payloadSize = sum([p.size for vib in patcherVibs.values()
                         for p in vib.payloads])
      # The ramdisk is used for VIB download, payload extraction and possibly
      # tardisk extraction.
      ramdiskSize = (payloadSize // MIB + 1) * 7

      Ramdisk.CreateRamdisk(ramdiskSize, ramdiskName, ramdiskPath)
      tardiskNames = []
      try:
         # Mount all VIBs in the ramdisk.
         for vib in patcherVibs.values():
            tardiskNames += Transaction._mountVibInRamdisk(vib,
                                                           ramdiskName,
                                                           ramdiskPath,
                                                           uniquePostfix,
                                                           checkAcceptance)

         # Alter sys.path to be able to import precheck, esximage, uefi and
         # systemStorage.
         sitepkgPath = os.path.join(ramdiskPath, 'lib64', PYTHON_VER_STR,
                                    'site-packages')
         vmwarePath = os.path.join(sitepkgPath, 'vmware')
         sysStorageZipPath = os.path.join(ramdiskPath, 'usr', 'lib', 'vmware',
                                          'esxupdate', 'systemStorage.zip')
         weaselPath = os.path.join(ramdiskPath, 'usr', 'lib', 'vmware')

         importPaths = (sitepkgPath, vmwarePath, sysStorageZipPath,
                        weaselPath)
         missingPaths = []
         for importPath in importPaths:
            if os.path.exists(importPath):
               sys.path.insert(0, importPath)
            else:
               missingPaths.append(importPath)

         if missingPaths:
            msg = ('Failed to locate libraries to import when setting up the '
                   'patcher.')
            log.error('%s Missing paths: %s.', msg, str(missingPaths))
            raise RuntimeError(msg)
         log.debug('Added %s to sys.path', ', '.join(importPaths))
      except Exception as e:
         Transaction._cleanupPatcher(ramdiskPath, tardiskNames)
         msg = 'Failed to setup patcher for upgrade: %s' % str(e)
         raise Errors.InstallationError(e, patcherVibs.keys(), msg)

      # Return ramdisk path and tardisk paths for cleanup before exit
      return ramdiskPath, tardiskNames

   @staticmethod
   def _cleanupPatcher(ramdiskPath, tardiskNames):
      '''Remove esx-update ramdisk and mounted tardisks after an image apply or
         profile command.
      '''
      if ramdiskPath:
         Ramdisk.RemoveRamdisk(os.path.basename(ramdiskPath), ramdiskPath)
      if tardiskNames:
         for tardiskName in tardiskNames:
            Ramdisk.UnmountManualTardisk(tardiskName, raiseException=False)

   @staticmethod
   def _runUpgradePrecheck(newProfile, vmkLinuxOnly=False, noHwWarning=False):
      '''Execute upgrade precheck before executing profile install/upgrade.
         Parameters:
            * newProfile   - the image profile to be installed.
            * vmkLinuxOnly - only execute VmkLinux prechecks that require the
                             image profile; this happens in 6.7 upgrades
                             where regular hardware checks have already been
                             executed.
            * noHwWarning  - ignore warnings from the precheck; an error will
                             still be raised.
         Raises:
            HardwareError - when an error is found during the precheck, or
                            an warning when noHwWarning unset.
      '''
      from weasel.util import upgrade_precheck

      # In non patch-the-patcher scenario, need to make sure vmware folder is
      # in sys.path.
      vmwarePath = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
      if not vmwarePath in sys.path:
         sys.path.insert(0, vmwarePath)

      errorMsg, warningMsg = upgrade_precheck.cliUpgradeAction(newProfile,
                                                               vmkLinuxOnly)

      # Raise exception when an error is found, --no-hardware-warning and
      # --force will not stop an error.
      if errorMsg:
         msg = 'Hardware precheck of profile %s failed with errors: ' \
               '%s' % (newProfile.name, errorMsg)
         raise Errors.HardwareError(msg)
      # Warnings can be bypassed with --no-hardware-warning
      if warningMsg:
         if not noHwWarning:
            msg = 'Hardware precheck of profile %s failed with warnings: ' \
                  '%s\n\nApply --no-hardware-warning option to ignore the ' \
                  'warnings and proceed with the transaction.' \
                  % (newProfile.name, warningMsg)
            raise Errors.HardwareError(msg)
         else:
            log.warning('Hardware precheck warnings are ignored with '
                        '--no-hardware-warning')

   @staticmethod
   def _checkEsxVersionDowngrade(newProfile, allowMissingEsxVersion=False):
      """Check if the VIB transaction will lead to a downgrade of ESXi
         release, raise DowngradeError if it is the case.
         Starting 7.0, API changes and major features are allowed in an
         update release, thus only downgrades within an update release
         are allowed, e.g. 7.0.1 for 7.0 U1.
         If allowMissingEsxVersion is set, allow esx-version to be not found
         in newProfile and skip the check, caller must validate newProfile
         separately.
      """
      # It is not necessary to get current profile with all the metadata;
      # and with corrupted image database it is impossible to do so.
      # Therefore let's use a stable method to get current version.
      curVer = HostInfo.GetEsxVersion()

      try:
         newVer = str(newProfile.GetEsxVersion(True).version)
      except ValueError:
         if allowMissingEsxVersion:
            log.info('Target image profile %s does not have esx-version, skip '
                     'downgrade check.', newProfile.name)
            return
         else:
            # Should only happen with an invalid profile in profile
            # install/update, this is basically an advance validation error.
            raise Errors.ProfileFormatError('Base ESXi package is not found in '
               'Image profile %s.' % newProfile.name)

      if Version.fromstring(curVer) > Version.fromstring(newVer):
         msg = ("Downgrade ESXi from version %s to %s is not supported."
                % (curVer, newVer))
         raise Errors.DowngradeError(msg)

   @staticmethod
   def _updateProfileInfo(profile, changelog, force=False):
      """Update Image Profile Information. Restore original vendor name
         from description if it was changed to hostname before.
      """
      # If original vendor is in description, restore it.
      splitdstr = profile.description.split('\n', 1)[0].split(':', 1)
      if splitdstr[0] == '(Original Vendor)':
         profile.creator = splitdstr[1]

      if not profile.name.startswith(UPDATEDPROFILESTR):
         profile.name = UPDATEDPROFILESTR + profile.name
      flagstr = ""
      if force:
         flagstr = "WARNING: A --force install has been performed, the image " \
             "may not be valid."
      profile.description = '%s: %s\n%s\n----------\n%s' % \
                            (profile.modifiedtime.isoformat(), changelog,
                             flagstr, profile.description)

   @staticmethod
   def _createUefiBootOption():
      """Create an UEFI boot option for this ESXI installation.
         This function is called during VUM or esxcli upgrade.
      """
      if not HostInfo.IsFirmwareUefi():
         return

      from systemStorage.esxboot import addUefiBootDisk
      from uefi import bootorder

      try:
         addUefiBootDisk()
         bootOpts = [bootorder.formatBootOption(optNum) for optNum in
                     bootorder.getBootOrder()]
         log.info('Final system boot options:\n%s' % '\n'.join(bootOpts))
      except Exception as e:
         # Do not panic, log the error
         log.error('Failed to create UEFI boot option: %s' % str(e))

class LivePatcherMount(object):
   """Class that manages live patcher mounting and calling into the patcher.
      Attributes:
         * _patcherVibs - Patcher VIBs to mount, None if Patch the Patcher
                          is not needed.
         * _force - Boolean that represents the --force option.
         * _noSigCheck - Boolean that represents the --no-sig-check option.
         * _enableQuickPatch - Boolean that indicates whether Quick Patch is
                               enabled.
         * downloadedMeta - A tuple of a VibCollection and a Metadata instance
                            that contain downloaded metadata from before Patch
                            the Patcher.
   """
   # TODO: add support of taking profile or image spec,
   # move _findPatcher(), _setupPatcher() and _cleanupPatcher() here.
   def __init__(self, patcherVibs, force, noSigCheck, enableQuickPatch=False,
                downloadedMeta=None):
      """Common constructor.
      """
      self._patcherVibs = patcherVibs
      self._force = force
      self._noSigCheck = noSigCheck
      self._enableQuickPatch = enableQuickPatch

      self.downloadedMeta = downloadedMeta

      self._mountDir = None
      self._mountedTardisks = None

   def __enter__(self):
      """When a new patcher is required, live mount and return a new
         Transaction class object; otherwise return an object of the current
         class.
      """
      if self._patcherVibs:
         self._mountDir, self._mountedTardisks = \
            Transaction._setupPatcher(self._patcherVibs, self._force,
                                      self._noSigCheck)

         from esximage.Transaction import Transaction as NewTransaction
         # Live mounted patcher, metadata will be re-downloaded and parsed
         # by the new code.
         return NewTransaction(initInstallers=False,
                               enableQuickPatch=self._enableQuickPatch)
      else:
         # Live mounted patcher is not needed, downlaoded metadata can be
         # re-used.
         return Transaction(initInstallers=False,
                            enableQuickPatch=self._enableQuickPatch)

   def __exit__(self, *args):
      """Exit live mounted patcher scope.
      """
      if self._patcherVibs:
         Transaction._cleanupPatcher(self._mountDir, self._mountedTardisks)

   @classmethod
   def InitVibInstall(cls, vibUrls, metaUrls=None, vibSpecs=None,
                      fillFromMeta=True, depotUrls=None, force=False,
                      noSigCheck=False):
      """Constructor used in VIB install/update methods.
      """
      liveProfile = Transaction().GetProfile(useVisorFs=True)
      curPatcherComp, curPatcherVibs = _findPatcher(liveProfile)

      vibsToInstall, meta = \
         Transaction.GetVibsAndMetaFromSources(vibUrls, metaUrls, vibSpecs,
                                               fillFromMeta, depotUrls)
      newComps = meta.components
      newPatcherComp, newPatcherVibs = \
         _findPatcherFromCompsVibs(newComps, vibsToInstall)

      if _requiresNewPatcher(curPatcherComp, curPatcherVibs,
                             newPatcherComp, newPatcherVibs):
         return cls(newPatcherVibs, force, noSigCheck)
      else:
         # Live mounted patcher is not neeed, metadata can be re-used.
         return cls(None, None, None, downloadedMeta=(vibsToInstall, meta))

   @classmethod
   def InitProfileInstallUpdate(cls, profileName, depotUrls, force=False,
                                noSigCheck=False, enableQuickPatch=False):
      """Constructor used in profle install/update methods.
      """
      liveProfile = Transaction().GetProfile(useVisorFs=True)
      curPatcherComp, curPatcherVibs = _findPatcher(liveProfile)

      newPatcherComp, newPatcherVibs = _getPatcherForProfileCmd(profileName,
                                                                depotUrls)

      if _requiresNewPatcher(curPatcherComp, curPatcherVibs,
                             newPatcherComp, newPatcherVibs):
         return cls(newPatcherVibs, force, noSigCheck,
                    enableQuickPatch=enableQuickPatch)
      else:
         return cls(None, None, None, enableQuickPatch=enableQuickPatch)

def _checkComponentDowngrades(curProfile, newProfile):
   """Check unsupported component downgrade from the current to the new image
      profile, including:
      1) Downgrade of components between same-name release units while release
         unit version goes up.
      2) Any component downgrade with config schema in both versions of
         the components.
      3) Orphan VIB to component VIB downgrade with config schema in both
         versions of the VIBs.
   """
   from vmware.esximage import Addon, Manifest, Solution

   compDowngrades = curProfile.GetCompsDowngradeInfo(newProfile)
   orphanVibDowngrades = curProfile.GetOrphanVibsDowngradeInfo(newProfile)

   if not compDowngrades and not orphanVibDowngrades:
      return

   oldRelUnitCompPairs, newRelUnitCompPairs = (curProfile.GetReleaseUnitComps(),
                                               newProfile.GetReleaseUnitComps())
   compSrcToRelType = {
      SOURCE_BASEIMAGE: BaseImage.BaseImage.releaseType,
      SOURCE_ADDON: Addon.Addon.releaseType,
      SOURCE_HSP: Manifest.Manifest.releaseType,
      # Solution's default value for releaseType after initiated.
      SOURCE_SOLUTION: Solution.Solution.solutionAttrib,
   }
   def findCompRelUnit(name, relType, relUnitCompPairs):
      for relUnit, relUnitComps in relUnitCompPairs:
         if relType == relUnit.releaseType and relUnitComps.HasComponent(name):
            return relUnit
      raise ValueError('Component %s: not found in release units.' % name)
   def isSameNameDowngrade(name, compSrc):
      """Returns if the component is a downgrade between release units of
         the same name, i.e. of the same name or a same HSM/HSP combination
         in a manifest.
      """
      if compSrc == SOURCE_USER:
         return False
      if compSrc == SOURCE_BASEIMAGE:
         return True
      relType = compSrcToRelType[compSrc]
      oldUnit = findCompRelUnit(name, relType, oldRelUnitCompPairs)
      newUnit = findCompRelUnit(name, relType, newRelUnitCompPairs)
      if compSrc == SOURCE_HSP:
          oldHsi, newHsi = (oldUnit.hardwareSupportInfo,
                            newUnit.hardwareSupportInfo)
          if (oldHsi.manager == newHsi.manager and
              oldHsi.package.name == newHsi.package.name):
            return True
      elif oldUnit.nameSpec.name == newUnit.nameSpec.name:
         return True
      return False

   configDgComps = set()
   sameTypeDgComps = set()
   for name, (_, _, src, dest, configSchema) in compDowngrades.items():
      if configSchema:
         configDgComps.add(name)
      if src == dest and isSameNameDowngrade(name, src):
         sameTypeDgComps.add(name)

   msg = ''
   if configDgComps:
      msg += ('Downgrade of Component(s) %s is not supported due to possible '
              'config downgrade.' % getCommaSepArg(configDgComps))
   if sameTypeDgComps:
      msg += ('\nComponent(s) %s is unexpectedly downgraded when updating '
              'BaseImage, Addon, Hardware Support Package or Solution.'
              % getCommaSepArg(sameTypeDgComps))

   dgVibs = set()
   dgVibComps = set()
   for vibName, (_, _, compName, configSchema) in orphanVibDowngrades.items():
      if configSchema:
         dgVibs.add(vibName)
         if compName:
            dgVibComps.add(compName)
         else:
            # This util should only be used to check standalone VIB to component
            # downgrade, but anyway provide some info instead of panicking.
            dgVibComps.add('(Standalone VIB)')

   if dgVibs:
      msg += ('\nDowngrade of standalone VIB(s) %s within Component(s) %s is '
              'not supported due to possible config downgrade.'
              % (getCommaSepArg(dgVibs), getCommaSepArg(dgVibComps)))

   if msg:
      if sameTypeDgComps or dgVibs:
         raise Errors.ComponentDowngradeError(
                  sorted(configDgComps | sameTypeDgComps | dgVibComps),
                  msg.lstrip('\n'))
      else:
         # Config downgrade only, use the special error to identify it.
         raise Errors.ComponentConfigDowngrade(sorted(configDgComps), msg)

def _setProfileBaseImage(newProfile, curProfile, refProfile, newBaseImages):
   """Set proper base image in the new image profile where VIB changes were
      made. Called in vib install/update commands, profile update and VUM ISO
      upgrade.
      Reference input are current image profile, reference profile given to
      profile update command or in the upgrade ISO, and base images from the
      connected depots.
   """
   try:
      oldEsxVer = curProfile.GetEsxVersion(rawversion=True)
      newEsxVer = newProfile.GetEsxVersion(rawversion=True)
   except ValueError as e:
      # Happens when the new image profile does not have esx-base, we shall
      # let validiation catch it and raise a proper error.
      log.warning('Unable to set base image, failed to get esx-version: %s',
                  str(e))
      return
   if newEsxVer == oldEsxVer:
      return

   allBaseImages = ReleaseCollection.BaseImageCollection()
   if curProfile.baseimage:
      allBaseImages.AddBaseImage(curProfile.baseimage)
   if refProfile and refProfile.baseimage:
      allBaseImages.AddBaseImage(refProfile.baseimage)
   if newBaseImages:
      allBaseImages += newBaseImages

   # Look up base image quickly with release ID, this assumes esx-version
   # from the esx-base VIB is the same as base image version.
   newBiId = BaseImage.GenerateReleaseID(str(newEsxVer))
   if newBiId in allBaseImages:
      newProfile.baseimageID = newBiId
      newProfile.baseimage = allBaseImages[newBiId]
   else:
      # No suitable base image, it is only safe to unset base image since
      # we enforce base image -> ESX component mapping.
      log.info('Unable to find new base image with version %s, proceed with '
               'no base image, candidates are: %s',
               str(newEsxVer), ','.join(allBaseImages.keys()))
      newProfile.baseimageID = None
      newProfile.baseimage = None

def _setProfileAddon(newProfile, refProfile, refAddons):
   """Sets proper addon in the new image profile where VIB changes were made.
      The process:
      1) Insert reference profile's addon if it is compatible. This addon has
         higher priority because it is the image profile specified for
         install/upgrade.
      2) If there was a previous addon, rank all known compatible addons of the
         same name by the their versions and rate of component installation. The
         addon with the highest rate and higher version will be set. In case of
         same component installation rate the higher version one will be set.
      3) If step 2 does not yield any addon, i.e. the current one became
         incompatible after the upgrade and no compatible addon is found, unset
         the addon.
      Returns a boolean indicating whether the addon has been changed in the new
      image profile.
   """
   if (refProfile and refProfile.addon and newProfile.baseimage and
         refProfile.addon.IsBaseImageSupported(newProfile.baseimage)):
      # When using a full image to install VIBs, its addon will be used even
      # if some components may not be the same version.
      newProfile.SetAddon(refProfile.addon, replace=True)
      return True

   if not newProfile.addon:
      return False

   if not newProfile.baseimage:
      log.debug("Unseting addon '%s' from the image profile due to missing "
                "base image.", newProfile.addonID)
      newProfile.addonID = None
      newProfile.addon = None
      return True

   def getAddonCompInstallRate(addon):
      """Returns the complete rate of the addon, calculated by the rate of its
         components being installed in newProfile.
      """
      total = len(addon.components)
      installed = 0
      for name, ver in addon.components.items():
         if newProfile.components.HasComponent(name, ver):
            installed += 1
      return installed/total

   addonName = newProfile.addon.nameSpec.name
   curAddonVer = newProfile.addon.versionSpec.version
   if newProfile.addon.IsBaseImageSupported(newProfile.baseimage):
      canKeepAddon = True
      maxRate = getAddonCompInstallRate(newProfile.addon)
   else:
      canKeepAddon = False
      maxRate = 0
   addonToSet = None

   if maxRate == 1.0:
      # Current addon is fully installed, no need to look further.
      return False

   if refAddons:
      # Get higher version addons of the same name.
      addons = [addon for addon in refAddons.values()
                if addon.nameSpec.name == addonName and
                   addon.versionSpec.version > curAddonVer and
                   addon.IsBaseImageSupported(newProfile.baseimage)]
      addons = sorted(addons, key=lambda x: x.versionSpec.version, reverse=True)
      for addon in addons:
         # Try addon from higher version to lower.
         curRate = getAddonCompInstallRate(addon)
         if curRate > maxRate:
            maxRate = curRate
            addonToSet = addon

         if maxRate == 1.0:
            break

   if addonToSet:
      # Setting a different addon than previously set.
      log.debug("Setting addon '%s' in the image profile.",
                addonToSet.releaseID)
      newProfile.SetAddon(addonToSet, replace=True)
      return True

   if not canKeepAddon:
      # Unset if no compatible addon.
      log.debug("Unseting addon '%s' from the image profile due to no "
                "compatible addon with the same name.", newProfile.addonID)
      newProfile.addonID = None
      newProfile.addon = None
      return True

   return False

def _setRemovedComponents(newProfile, curProfile, refProfile):
   """Sets removedComponents in the new image profile.
      If a component was previously removed via a desired image and is not
      installed in the new profile, it is considered a removed component.
   """
   if not refProfile:
      # Patch baseline upgrade path
      return

   for comp in refProfile.removedComponents:
      if not curProfile.GetComponentInfo(comp):
         newProfile.removedComponents.append(comp)

def _addSolutionsToProfile(imgProfile, components, solutions):
   """Add suitable solutions for the components to the image profile.
      If multiple versions of solutions are suitable, only the highest
      version will be added.
   """
   solDict = dict()
   for solution in solutions.values():
      if solution.MatchComponents(components):
         solName = solution.nameSpec.name
         curSol = solDict.get(solName, None)
         if (not curSol or
             curSol.versionSpec.version < solution.versionSpec.version):
            solDict[solName] = solution

   for solution in solDict.values():
      log.debug('Adding solution %s (version %s) to the image profile.',
                solution.nameSpec.name,
                solution.versionSpec.version.versionstring)
      imgProfile.AddSolution(solution, replace=True)

def _getMetadataFromMetaDepotUrls(metaUrls, depotUrls):
   """Get a combined Metadata instance from metadata and depot URLs.
      Currently only vibs, bulletins, baseimages and addons are merged.
   """
   if metaUrls:
      meta = Transaction.DownloadMetadatas(metaUrls)
   else:
      meta = Metadata.Metadata()

   if depotUrls:
      dc = Transaction.ParseDepots(depotUrls)
      meta.vibs, meta.bulletins, meta.baseimages, meta.addons = \
         _mergeMetadataCollections(
            (meta.vibs, meta.bulletins, meta.baseimages, meta.addons),
            (dc.vibs, dc.bulletins, dc.baseimages, dc.addons))

   return meta

def _mergeMetadataCollections(allMeta, newMeta):
   """Merge metadata collections by adding the new ones to the ones that hold
      all metadata.
      Input are two tuples/lists of collections that are of matching types at
      each position. Return is the merged collections in a list.
   """
   return [allObj + newObj for allObj, newObj in zip(allMeta, newMeta)]

def _requiresNewPatcher(oldComp, oldVibs, newComp, newVibs):
   """Using old and new patcher component/VIBs, return if the new patcher
      needs to be installed.
   """
   if not oldVibs:
      raise ValueError('Current patcher VIBs must be provided')

   if newVibs is None:
      return False

   if oldComp and newComp:
      # Speedy path with component version.
      return (newComp.componentversionspec['version'] >
              oldComp.componentversionspec['version'])
   else:
      # Compare by each VIB.
      oldVibsVers = dict((vib.name, vib.version) for vib in oldVibs.values())
      newVibsVers = dict((vib.name, vib.version) for vib in newVibs.values())

      for vibName, vibVer in newVibsVers.items():
         if vibName not in oldVibsVers:
            # New patcher VIB -> True.
            # Assume downgrade has been checked so this should not be a
            # problem.
            return True
         if vibVer > oldVibsVers[vibName]:
            return True
      return False

def _getPatcherFromImageSpec(imageSpec, depotUrls):
   """Get patcher component and VIBs with an image spec.
   """
   specMgr = _getSoftwareSpecMgr(imageSpec, depotUrls)
   baseImage = specMgr._getBaseImage()
   if baseImage:
      # If base image is present, get the component.
      compVer = baseImage.GetComponentVersion(PATCHER_COMP_NAME)
      comp = specMgr.getComponent(PATCHER_COMP_NAME, compVer)
      vibs = specMgr.getComponentVibs(comp)
      return comp, vibs
   else:
      # Base image is not available, take the long route to use
      # image profile to figure out patcher component/VIB.
      return _findPatcher(getImageProfileFromSpec(imageSpec, depotUrls))

def _findPatcher(imageProfile):
   """Get the patcher component and VIBs from an image profile.
   """
   return _findPatcherFromCompsVibs(imageProfile.components, imageProfile.vibs,
                                    inImageProfile=True)

def _findPatcherFromCompsVibs(comps, vibs, inImageProfile=False):
   """Get patcher component and VIBs from a component collection and a VIB
      collection.
   """
   candidates = set()
   patcherComp, patcherVibs = None, None

   for comp in comps.IterComponents():
      # Find highest esx-update component with VIBs to back it.
      if comp.compNameStr == PATCHER_COMP_NAME:
         # Component metadata may supply candidates without VIBs to back
         # them in VUM patching.
         vibDict = dict((vibId, vibs[vibId]) for vibId in comp.vibids
                        if vibId in vibs)
         if (len(vibDict) == len(comp.vibids) and
             (patcherComp is None or
              comp.compVersion > patcherComp.compVersion)):
            patcherComp, patcherVibs = \
               comp, VibCollection.VibCollection(vibDict)

            if inImageProfile:
               # Image profile is not supposed to have more than one.
               break
         elif len(vibDict) > 0:
            # Incomplete patcher found.
            candidates.add(comp.compUiStr)

   if patcherComp and patcherVibs:
      return patcherComp, patcherVibs
   elif candidates:
      # If there is supporting component info, it is unexpected to have an
      # incomplete patcher.
      msg = ('Upgrade component(s) "%s" is found, but it is missing VIB '
             'metadata. ' % getCommaSepArg(candidates))
      msg += ('Please use an image profile with a complete ESXi Base Image.'
              if inImageProfile else
              'Please use a depot with a complete ESXi Base Image.')
      raise Errors.InstallationError(None, None, msg)
   else:
      # If no patcher component is found, fallback to VIBs for legacy support.
      vibDict = dict()
      for vib in vibs.values():
         if (vib.name in PATCHER_VIB_NAMES and
             (vib.name not in vibDict or
              vib.version > vibDict[vib.name].version)):
            # Find highest-versioned patcher VIB.
            vibDict[vib.name] = vib

      if len(vibDict) == len(PATCHER_VIB_NAMES):
         vibs = VibCollection.VibCollection(
            dict((v.id, v) for v in vibDict.values()))
         return None, vibs
      elif len(vibDict) > 0:
         # Incomplete patcher.
         msg = ('Upgrade VIB(s) "%s" is required for the transaction. '
                'Please use a depot with a complete set of ESXi VIBs.'
                % getCommaSepArg(set(PATCHER_VIB_NAMES) - set(vibDict.keys())))
         raise Errors.InstallationError(None, None, msg)

      # No upgrade is involved.
      return None, None

def _getExceptionNotification(ex):
   """Get a notification from an exception.
      This is a wrapper of the function in ImageManager module as it is
      not included in esximage.zip for legacy VUM.
   """
   from .ImageManager.Utils import getExceptionNotification
   return getExceptionNotification(ex)

def _getPatcherForProfileCmd(profileName, depotUrls):
   """Get the patcher component and VIBs for an image profile install/update
      command.
   """
   newProfile, _ = Transaction.GetProfileAndMetaFromSources(
                                                   profileName,
                                                   depotUrls=depotUrls)
   # We want to avoid an old patcher reading current profile
   # (forward-compatibility), thus check for downgrade in advance.
   Transaction._checkEsxVersionDowngrade(newProfile)

   patcherComp, patcherVibs = _findPatcher(newProfile)

   # We have already checked for a downgrade, so at this point, a valid
   # image profile must contain an esx-update VIB.
   if not patcherComp and not patcherVibs:
      msg = ('Image profile %s does not contain component/VIB needed for '
             'setting up the upgrade. This image profile might be invalid.'
             % profileName)
      raise Errors.InstallationError(None, None, msg)
   return patcherComp, patcherVibs

def _getSoftwareSpecMgr(imageSpec, depotUrls):
   """Get the SoftwareSpecMgr instance from image spec and depot URLs.
   """
   # Avoid importing new dependencies outside esximage.zip in a legacy
   # VUM upgrade, or dependencies outside early tardisks in secureBoot.
   from .ImageManager import DepotMgr, SoftwareSpecMgr

   depotSpec = DepotMgr.getDepotSpecFromUrls(depotUrls)
   depotMgr = DepotMgr.DepotMgr(depotSpec, connect=True)
   return SoftwareSpecMgr.SoftwareSpecMgr(depotMgr, imageSpec)

def getImageProfileFromSpec(imageSpec, depotUrls):
   """Get the translated image profile of an image spec.
      The interface of this function needs to stay backward-compatible
      as it is used in patch the pacher.
   """
   softSpecMgr = _getSoftwareSpecMgr(imageSpec, depotUrls)
   return softSpecMgr.validateAndReturnImageProfile()

def getSolutionComponents(imageSpec, depotUrls, intents):
   """Resolve the solution constraints present inside the imageSpec on top
      of intents provided. Typically the currently installed components are
      passed in. Returns a ComponentCollection that contains intents from the
      solution constraints.

      Note: Only the 'solutions' field of the imageSpec is processed
            while resolving the solution constraints.
      Future: We can enhance this to use the rest of the imageSpec
              when the intent is set to None.
   """
   softSpecMgr = _getSoftwareSpecMgr(imageSpec, depotUrls)
   return softSpecMgr.resolveSolutionConstraints(intents)
