"""
Copyright (c) 2020-2024 Broadcom. All Rights Reserved.
Broadcom Confidential. The term "Broadcom" refers to Broadcom Inc.
and/or its subsidiaries.
"""
from datetime import datetime, timedelta
import glob
import json
import logging
import os
import shutil
import stat
import tarfile
import tempfile

from .Constants import *
from .Utils import getFormattedMessage, getCommaSepArg

from .. import (Depot, Downloader, Errors, HostImage,
                IS_ESXIO, MIB, OfflineBundle,
                PERSONALITY_MANAGER_COMPONENT_REMOVAL_ENABLED, Vib)
from ..Bulletin import ComponentCollection
from ..Utils import EsxGzip, HostInfo
from ..Utils.Misc import LogLargeBuffer
from ..Version import VibVersion
from ..VibCollection import VibCollection

try:
   # Calls to get image info or extract depot require bindings that aren't
   # found on earlier ESXi during upgrades.
   from .Scanner import getSolutionInfo, vapiStructToJson
   from com.vmware.esx.software_client \
      import (InstalledImage, SoftwareInfo, BaseImageInfo,
              AddOnInfo, ComponentInfo, HardwareSupportInfo, Notifications,
              Notification, SoftwareSpec, BaseImageSpec,
              AddOnSpec, HardwareSupportSpec)
   from com.vmware.vapi.std_client import LocalizableMessage
   VAPI_SUPPORT = True
except ImportError:
   VAPI_SUPPORT = False

isNotNone = lambda x: x is not None
EXTRACT_DEPOT_TASK_ID = 'com.vmware.esx.software.installedimage.extractdepot'
DEPOT_FILE_NAME = 'OfflineBundle'
LIFECYCLE_SCRACTCH = '/var/vmware/lifecycle'
_HOST_SEED_POSTFIX = 'hostSeed'
HOST_SEED_DIR_NAME = os.path.join(LIFECYCLE_SCRACTCH, _HOST_SEED_POSTFIX)

# TODOs:
#  Check if we have the required space on disk'

# Local exception classes for caller to be aware of the error context.
class ReserveVibCacheError(Exception):
   pass

class NoVibCacheError(ReserveVibCacheError):
   pass

class VibNotInCacheError(ReserveVibCacheError):
   pass

def getNotification(notificationId, msgId, msgArgs=None,
                     resArgs=None):
   """Helper function to compose the Notification Object
   """
   defMsg = getFormattedMessage(NOTIFICATION_MSG[msgId], msgArgs)
   msg = LocalizableMessage(id=msgId, default_message=defMsg,
                            args=msgArgs or [])
   resMsg = getFormattedMessage(
                  RESOLUTION_MSG.get(notificationId, ''), resArgs)
   if resMsg:
      # Populate the optional resolution when there is actually a message.
      resId = msgId + RESOLUTION_SUFFIX if resMsg else ''
      resolution = LocalizableMessage(id=resId,
                                      default_message=resMsg,
                                      args=resArgs or [])
   else:
      resolution = None

   return Notification(id=notificationId,
                       time=datetime.utcnow(),
                       message=msg,
                       resolution=resolution)

def _getImageNotification(msgId, info):
   """Helper function to form a notification for a components or VIBs.
   """
   if isinstance(info, dict):
      details = ['%s(%s)' % (name, version) for name, version in info.items()]
      return getNotification(msgId, msgId, msgArgs=[getCommaSepArg(details)])

   return getNotification(msgId, msgId, msgArgs=[getCommaSepArg(info)])

def _findVibPayload(vib, pName):
   """Returns payload object with the specified name in a VIB.
   """
   for payload in vib.payloads:
      if payload.name == pName:
         return payload
   raise ValueError('Payload %s is not found in VIB %s' % (pName, vib.id))

def _checkPayloadChecksum(fObj, pObj):
   """Checks if a payload file matches the expected checksum, an exception
      would be raised if the check does not pass.
   """
   for checksum in pObj.checksums:
      if checksum.verifyprocess == '':
         hashAlgo, expected = \
            checksum.checksumtype.replace('-', ''), checksum.checksum
         break
   else:
      raise ValueError('No checksum found for payload %s' % pObj.name)

   calculated, _ = Vib.calculatePayloadChecksumAndSize(
                   pObj, fObj, hashAlgo, False)
   logging.debug("Calculated %s checksum of payload %s '%s', expected '%s'",
                 hashAlgo, pObj.name, calculated, expected)

   if calculated != expected:
      raise RuntimeError('Calculated %s checksum of payload %s does not '
                         'match VIB metadata' % (hashAlgo, pObj.name))

def _getSeedImageProfile(profile):
   '''Orphan vibs, solutions and manifests are not needed in the extracted
      depot. Hence we remove them from profile and return the modifed profile.
   '''
   newProfile = profile.Copy()
   # Get list of orphan vibs and remove them from the cloned image profile.
   orphanVibs = newProfile.GetOrphanVibs()
   if orphanVibs:
      logging.info('Skipping orphan VIBs %s in depot extraction',
                   ', '.join(sorted(list(orphanVibs.keys()))))
   for vib in orphanVibs:
      newProfile.RemoveVib(vib)

   # Get list of solutions and remove them from the cloned image profile.
   sols = [s.nameSpec.name for s in newProfile.solutions.values()]
   if sols:
      logging.info('Skipping solutions %s in depot extraction',
                   ', '.join(sorted(sols)))
      newProfile.RemoveSolutions(sols)

   # Skip NSX component installed without a solution in a VUM cluster.
   # NSX needs to be specifically handled because HA and WCP are installed
   # only as orphan VIBs that are already being skipped.
   if newProfile.components.HasComponent(NSX_COMPONENT):
      nsxComp = newProfile.components.GetComponent(NSX_COMPONENT)
      solComps = ComponentCollection()
      solComps.AddComponent(nsxComp)
      logging.info('Skipping NSX %s in depot extraction', nsxComp.id)
      newProfile.RemoveComponents(solComps)

   # Get list of manifests and remove them from the cloned image profile.
   if newProfile.manifestIDs:
      logging.info('Skipping manifests %s in depot extraction',
                   ', '.join(sorted(newProfile.manifestIDs)))
      for manifestID in newProfile.manifestIDs:
         newProfile.manifests.RemoveManifest(manifestID)
      newProfile._syncVibs()

   return newProfile

def getIsoUpgradePayloadPath(vib, pName, hostImage, isoDir, isoProfile):
   """Get path to a VIB payload in an extracted ISO directory.
   """
   # localname of payload needs to be filled in.
   pObj = _findVibPayload(vib, pName)
   try:
      pObj.localname = isoProfile.vibstates[vib.id].payloads[pName]
   except KeyError:
      raise KeyError('VIB %s payload %s is not found in ISO directory image '
                     'database' % (vib.id, pName))
   logging.info('Finding %s in %s', pObj.name, isoDir)
   stagePath = hostImage.FindPayloadInDeployDir(vib, pObj, isoDir)
   with open(stagePath, 'rb') as fObj:
      _checkPayloadChecksum(fObj, pObj)

   return stagePath

def getBootBankPayloadPath(vib, pName, hostImage, tmpDir):
   """Get path to a bootbank VIB payload.
      Direct path into the current bootbank will be returned for gzipped
      payloads, a temp file created in tmpDir will be returned for misc
      esx-base payloads in basemisc.tgz and empty gzip payloads that are
      since modified after installation.
   """
   def createEmptyGzFile():
      """Creates an temp empty gzip file.
      """
      tempfObj = tempfile.NamedTemporaryFile(dir=tmpDir, delete=False)
      try:
         EsxGzip.GzipFile(tempfObj.name, 'wb').close()
         tempfObj.seek(0)
      except Exception:
         tempfObj.close()
         raise
      return tempfObj

   def createTempFile(fObj):
      with fObj:
         # Write the payload file to a temp file.
         tempfObj = tempfile.NamedTemporaryFile(dir=tmpDir, delete=False)
         shutil.copyfileobj(fObj, tempfObj, MIB)
         tempfObj.seek(0)
         return tempfObj

   if 'boot' not in hostImage.installers:
      raise RuntimeError('BootBankInstaller is not initiated')

   bbInstaller = hostImage.installers['boot']

   pObj = _findVibPayload(vib, pName)
   fObj = bbInstaller.OpenPayloadFile(vib.id, pObj)
   if fObj is None:
      logging.debug('Payload %s type %s of VIB %s cannot be opened by '
                   'BootBankInstaller', pObj.name, pObj.payloadtype, vib.id)
      return None

   if isinstance(fObj, tarfile.ExFileObject):
      fObj = createTempFile(fObj)
   stagePath = fObj.name

   # Check checksum of the payload., if it does not match and payload type is
   # boot, it might be an empty gzip payload (e.g. jumpstrt/features.gz).
   try:
      _checkPayloadChecksum(fObj, pObj)
   except RuntimeError:
      if pObj.payloadtype != pObj.TYPE_BOOT:
         raise
      # For boot payload, when checksum does not match, it might be an
      # empty/useropts gzip payload, re-create it.
      logging.debug('Payload %s of VIB %s might be an empty gzip payload or '
                    'modified during runtime', pObj.name, vib.id)
      if pObj.name == "useropts":
         bbfObj = bbInstaller.OpenPayloadFile(vib.id, pObj, isBaseMisc=True)
         with createTempFile(bbfObj) as tempfObj:
            _checkPayloadChecksum(tempfObj, pObj)
            stagePath = tempfObj.name
      else:
         with createEmptyGzFile() as tempfObj:
            _checkPayloadChecksum(tempfObj, pObj)
            stagePath = tempfObj.name
   finally:
      fObj.close()

   return stagePath

def getLockerPayloadPath(vib, pName, hostImage, tmpDir):
   """For locker vibs, payload is extracted in locker partition and kept there.
      To reconstruct a locker vib, we need to create the exact .tgz payload
      and we have to make sure that there are no timestamp or user information
      which can change checksum. Currently, there is only one locker vib i.e.
      tools-light which needs to be handled this way. This follows the similar
      implementation as done in scons build.
   """
   from ..Installer.LockerInstaller import LOCKER_ROOT, PAYLOAD_MAPPING_FILE
   if 'locker' not in hostImage.installers:
      raise Errors.VibRecreateError(vib.id,
                                    'LockerInstaller is not initiated')

   payloadPath = LOCKER_ROOT
   # Stage the .tar and .tgz payloads in hostSeed directory. These files have
   # to be deleted once the payload is added to the vib.
   payloadtar = os.path.join(tmpDir, pName + '.tar')
   gzipFile = os.path.join(tmpDir, pName)
   # Handle the case when multiple payloads are present for a given locker vib.
   # We have a file inside locker root which stores the payload to file mapping.
   if os.path.exists(PAYLOAD_MAPPING_FILE):
      try:
         with open(PAYLOAD_MAPPING_FILE, 'r') as f:
            payloadFileDict = json.load(f)
         fileList = payloadFileDict.get(vib.name, dict()).get(pName, []) or \
                    vib.filelist
      except Exception as e:
         logging.error('Failed to read from file %s: %s',
                       PAYLOAD_MAPPING_FILE, str(e))
         fileList = vib.filelist
   else:
      # If payload mapping file doesn't exist due to accidental delete or when
      # this host was upgraded without patch the patcher.
      # At present, we don't have any locker vib with more than one payload
      # and hence it is better to fall back on vib's filelist.
      fileList = vib.filelist

   try:
      # Order in which files are added to tar is important to ensure a
      # checksum match with descriptor.
      with tarfile.open(name=payloadtar, mode="w",
                        format=tarfile.GNU_FORMAT) as tar:
         for root, dirs, files in os.walk(payloadPath):
            for f in dirs + files:
               fspath = os.path.join(root, f)
               arcname = os.path.relpath(fspath, payloadPath)
               # Do not add a file that is not in VIB's filelist, or a dir
               # that is not in any of the files' path.
               if os.path.isdir(fspath) and not any(f in x for x in fileList):
                  continue
               if os.path.isfile(fspath) and arcname not in fileList:
                  continue

               # After VUM upgrade from 6.x, some directories may be left
               # behind and individual files may be found with 700 permission.
               # Skip the directories and adjust permissions back to default.
               # TODO: Remove after 6.x releases hit EOL.
               if os.path.isdir(fspath) and '6.5.0' in fspath:
                  continue

               mode = os.stat(fspath).st_mode
               if os.path.isfile(fspath) and oct(stat.S_IMODE(mode)) != '0o644':
                  os.chmod(fspath, 0o644)

               ti = tar.gettarinfo(fspath, arcname)
               ti.mtime = 0
               ti.uid = ti.gid = 0
               ti.uname = ti.gname = 'root'
               if ti.islnk():
                  ti.type = tarfile.REGTYPE
                  ti.size = os.path.getsize(fspath)
               tar.addfile(ti, None if ti.isdir() or ti.issym() else
                           open(fspath, 'rb'))

      # Now Gzip the tar file to convert into a tgz. EsxGzip ensures that gzip
      # header doesn't contain timestamp which can affect checksum.
      with EsxGzip.GzipFile(gzipFile, 'wb') as out:
          with open(payloadtar, 'rb') as inp:
             data = inp.read(Vib.PAYLOAD_READ_CHUNKSIZE)
             while data:
                out.write(data)
                data = inp.read(Vib.PAYLOAD_READ_CHUNKSIZE)
   except Exception as e:
      raise Errors.VibRecreateError(vib.id, "Failed to create payload %s: %s" %
                                    (pName, str(e)))
   finally:
      # Delete the .tar file as it is not needed anymore
      if os.path.isfile(payloadtar):
         os.remove(payloadtar)
   # Return the final stage path which will be used to add payload to vib.
   return gzipFile

class InstalledImageInfo(object):
   """This class provides methods to get the ESX host's current
      software information.
   """

   def __init__(self):
      """Constructor
      """
      if not VAPI_SUPPORT:
         raise RuntimeError('VAPI support classes could not be imported')

      self.currHostImage = HostImage.HostImage()
      db = self.currHostImage.DB_VISORFS
      self.currImageProfile = self.currHostImage.GetProfile(database=db)

      if self.currImageProfile is None or len(self.currImageProfile.vibIDs)==0:
         msg = "Could not extract profile from host"
         logging.error(msg)
         raise ValueError(msg)

      self.biAddonCompNames = set()
      if self.currImageProfile.baseimage:
         self.baseImageComponents = self.currImageProfile.baseimage.components
         self.biAddonCompNames |= set(self.baseImageComponents.keys())

      if self.currImageProfile.addon:
         self.addOnComponents = self.currImageProfile.addon.components
         self.biAddonCompNames |= set(self.addOnComponents.keys())
      else:
         self.addOnComponents = None

      # Manifests info
      self.hspDict, self.allHspCompDict, self.allHspRmCompNames = \
         self.currImageProfile.GetHardwareSupportInfo()

   def getLocalSolutionInfo(self):
      """Get a tuple of:
         1) A dict of SolutionComponentDetails for local solution components,
            indexed by solution name and then component name.
         2) Names of components that are installed as part of solutions.
      """
      solInfoDict = dict()
      ipSolDict, ipSolComps = self.currImageProfile.GetSolutionInfo()
      for solId, solComps in ipSolDict.items():
         solution = self.currImageProfile.solutions[solId]
         solInfoDict[solution.nameSpec.name] = getSolutionInfo(solution,
                                                               solComps)
      return solInfoDict, list(ipSolComps.keys())

   def _getCompVersions(self, name):
      """Returns versions of a component in base image, addon and HSP,
         None is used whenever the component is not found in a release unit.
      """
      getCompVerObj = lambda d, n: (VibVersion.fromstring(d[n])
                                    if d and n in d else None)
      return [getCompVerObj(comps, name)
              for comps in (self.baseImageComponents, self.addOnComponents,
                 self.allHspCompDict)]

   # TODO: Reuse duplicate code from Scanner - Bug 2678306
   def _getRemovedDgAndUgComponents(self, baseImageComponents, addOnComponents,
                                    addOnRemovedCompNames):
      """Scans current components and return a dictionary where removed/
         downgraded/upgraded components of Base Image/Addon are categorized.
      """
      installedComps = self.currImageProfile.ListComponentSummaries()

      compUserAdded = dict()
      addOnRemovedCompNames = addOnRemovedCompNames or []
      hspRemovedCompNames = self.allHspRmCompNames or set()

      # All component names appear in the addon, base image and manifests.
      biAddonHspCompNames = set(baseImageComponents.keys())
      if addOnRemovedCompNames:
         biAddonHspCompNames -= set(addOnRemovedCompNames)
      if addOnComponents:
         biAddonHspCompNames |= set(addOnComponents.keys())
      if hspRemovedCompNames:
         biAddonHspCompNames -= hspRemovedCompNames
      if self.allHspCompDict:
         biAddonHspCompNames |= set(self.allHspCompDict.keys())

      ADD, UPGRADE, DOWNGRADE = 'add', 'upgrade', 'downgrade'
      # Base image components.
      baseImageCompSpec = dict()
      # A breakdown of addon components regarding base image components.
      addonCompSpec = {
         ADD: dict(),
         UPGRADE: {
            BASE_IMG: dict(),
         },
         DOWNGRADE: {
            BASE_IMG: dict(),
         },
      }
      # A breakdown of hardware support components' relationship related to
      # base image and addon.
      hspCompSpec = {
         ADD: dict(),
         UPGRADE: {
            BASE_IMG: dict(),
            ADD_ON: dict()
         },
         DOWNGRADE: {
            BASE_IMG: dict(),
            ADD_ON: dict(),
         },
      }
      # A breakdown of user components regarding manifest components.
      userCompSpec = {
         ADD: dict(),
         UPGRADE: {
            BASE_IMG: dict(),
            ADD_ON: dict(),
            HARDWARE_SUPPORT: dict(),
         },
         DOWNGRADE: {
            BASE_IMG: dict(),
            ADD_ON: dict(),
            HARDWARE_SUPPORT: dict(),
         },
      }
      def _addCompInSpec(compName, comp, spec, subject, hostVer, subjectVer):
         """Triage a component to one of add, upgrade and downgrade categories
            according to host's and the spec piece's component versions.
            An addition in the spec dict means the spec piece (e.g. addon)
            adds the component, or upgrades/downgrades the component of another
            image piece (subject), e.g. base image.
         """
         if hostVer > subjectVer:
            spec[UPGRADE][subject][compName] = comp
         elif hostVer < subjectVer:
            spec[DOWNGRADE][subject][name] = comp
         else:
            # Even if multiple pieces claim the same version, the component
            # will be treated as "add" in all of them. This makes sure upgrade
            # and downgrade components are calculated correctly.
            spec[ADD][name] = comp

      solDict, solutionCompNames = self.getLocalSolutionInfo()
      installedCompNames = set()
      for comp in installedComps:
         # Loop to figure out upgrade/downgrade relations.
         name = comp['component']
         installedCompNames.add(name)
         if name in solutionCompNames:
            # Solution component.
            continue
         if name not in self.biAddonCompNames:
            # Component introduced by the user, could be a new component,
            # or one that is removed by addon but re-added by user.
            compUserAdded[name] = comp
         else:
            # Find the provider of the component and if there is any
            # upgrade/downgrade took place.
            hostVersion = VibVersion.fromstring(comp[VERSION])
            biVersion, addonVersion, hspVersion = self._getCompVersions(name)

            if isNotNone(biVersion):
               # Base image originally has the component.
               if biVersion == hostVersion:
                  # No override.
                  baseImageCompSpec[name] = comp
               elif isNotNone(addonVersion) and addonVersion == hostVersion:
                  # Addon override base image, it should be upgrade only, but
                  # we will check all cases nevertheless.
                  _addCompInSpec(name, comp, addonCompSpec, BASE_IMG,
                                 hostVersion, biVersion)
               elif isNotNone(hspVersion) and hspVersion == hostVersion:
                  # HSP overrides base image, it should be upgrade only, but
                  # we will check all cases just like addon.
                  _addCompInSpec(name, comp, hspCompSpec, BASE_IMG,
                                 hostVersion, biVersion)
               else:
                  # User component override base image, "add" was already
                  # handled, so here should only be upgrade/downgrade.
                  _addCompInSpec(name, comp, userCompSpec, BASE_IMG,
                                 hostVersion, biVersion)
            elif isNotNone(addonVersion):
               # Addon originally has the component.
               if addonVersion == hostVersion:
                  # Addon adds the component.
                  addonCompSpec[ADD][name] = comp
               elif isNotNone(hspVersion) and hspVersion == hostVersion:
                  # HSP overrides addon, it should be upgrade only, but
                  # we will check all cases just like addon.
                  _addCompInSpec(name, comp, hspCompSpec, ADD_ON,
                                 hostVersion, addonVersion)
               else:
                  # User component override addon, "add" was already
                  # handled, so here should only be upgrade/downgrade.
                  _addCompInSpec(name, comp, userCompSpec, ADD_ON,
                                 hostVersion, addonVersion)
            else:
               # HSP originally has the component.
               if hspVersion == hostVersion:
                  # Manifest adds the component.
                  hspCompSpec[ADD][name] = comp
               else:
                  # User component upgrades/downgrades HSP.
                  _addCompInSpec(name, comp, userCompSpec, HARDWARE_SUPPORT,
                                 hostVersion, hspVersion)


      # Component on host may have been reserved due to VIB obsolete rather than
      # same component name replacement or manual removal. A catch-up component
      # scanning is required so we do not report release unit component removal
      # when a component was obsoleted as a result of the last apply.
      reservedComps = set(self.currImageProfile.reservedComponentIDs)

      # Sets for the components of interest.
      removedBIComps = ((set(baseImageComponents.keys()) - installedCompNames) -
                         set(addOnRemovedCompNames) - set(hspRemovedCompNames))
      downgradedBIComps = (set(addonCompSpec[DOWNGRADE][BASE_IMG].keys() |
                           set(hspCompSpec[DOWNGRADE][BASE_IMG].keys()) |
                            set(userCompSpec[DOWNGRADE][BASE_IMG].keys())) -
                            set(addOnRemovedCompNames) -
                            set(hspRemovedCompNames))

      removedAddonComps, downgradedAddonComps = set(), set()
      if addOnComponents:
         removedAddonComps = (set(addOnComponents.keys()) -
                              installedCompNames -
                              set(hspRemovedCompNames))
         downgradedAddonComps = (set(hspCompSpec[DOWNGRADE][ADD_ON].keys()) |
                                 set(userCompSpec[DOWNGRADE][ADD_ON].keys()) -
                                 set(hspRemovedCompNames))
      # All addon components
      allAddonComps = dict()
      allAddonComps.update(addonCompSpec[ADD])
      allAddonComps.update(addonCompSpec[UPGRADE][BASE_IMG])
      allAddonComps.update(addonCompSpec[DOWNGRADE][BASE_IMG])

      # Update the User Components with the Added, and Upgraded Components.
      compUserAdded.update(userCompSpec[ADD])

      compUserUpgraded = dict()
      compUserUpgraded.update(userCompSpec[UPGRADE][HARDWARE_SUPPORT])
      compUserUpgraded.update(userCompSpec[UPGRADE][ADD_ON])
      compUserUpgraded.update(userCompSpec[UPGRADE][BASE_IMG])

      compUserDowngraded = dict()
      compUserDowngraded.update(userCompSpec[DOWNGRADE][HARDWARE_SUPPORT])
      compUserDowngraded.update(userCompSpec[DOWNGRADE][ADD_ON])
      compUserDowngraded.update(userCompSpec[DOWNGRADE][BASE_IMG])

      compUserAddUpgradeDowngrade = dict()
      compUserAddUpgradeDowngrade.update(compUserAdded)
      compUserAddUpgradeDowngrade.update(compUserUpgraded)
      compUserAddUpgradeDowngrade.update(compUserDowngraded)

      userRmBIComps, userRmAddonComps = set(), set()
      if PERSONALITY_MANAGER_COMPONENT_REMOVAL_ENABLED:
         # Handle removed components in image customization; they are not to be
         # treated as missing.
         if self.currImageProfile.removedComponents:
            rmComps = set(self.currImageProfile.removedComponents)
            # User removed Base Image and Addon component names.
            userRmBIComps = removedBIComps & rmComps
            userRmAddonComps = removedAddonComps & rmComps
            removedBIComps -= rmComps
            removedAddonComps -= rmComps
            # If a removed component is both in Base Image and Addon,
            # only treat it as being removed from the addon so only
            # one notification will be added.
            userRmBIComps -= userRmBIComps & userRmAddonComps

      compInfo = dict()
      # Name to info dictionaries.
      compInfo[BASEIMAGE_COMPS_KEY] = baseImageCompSpec
      compInfo[ADDON_COMPS_KEY] = allAddonComps
      # Lists of names.
      compInfo[DOWNGRADED_BI_COMP_KEY] = downgradedBIComps
      compInfo[REMOVED_BI_COMP_KEY] = removedBIComps
      compInfo[DOWNGRADED_ADDON_COMP_KEY] = downgradedAddonComps
      compInfo[REMOVED_ADDON_COMP_KEY] = removedAddonComps
      compInfo[USER_OVERRIDDEN_COMPS_KEY] = list(
         compUserAddUpgradeDowngrade.values())


      # USER_REMOVED_BI_COMP_KEY and USER_REMOVED_ADDON_COMP_KEY store names
      # of components that are user removed via image customization.
      if PERSONALITY_MANAGER_COMPONENT_REMOVAL_ENABLED:
         compInfo[USER_REMOVED_BI_COMP_KEY] = userRmBIComps
         compInfo[USER_REMOVED_ADDON_COMP_KEY] = userRmAddonComps

      return compInfo

   def addNotification(self, notificationList, notification):
      if notificationList is None:
         notificationList = []
      notificationList.append(notification)
      return notificationList

   def _getNotificationList(self, baseImageComponents, addOnComponents,
                            addOnRemovedCompNames):
      """Get the info/warning/error messages that can be reported by the task
      """
      # Notifications are mandatory. Hence we initialize it at the begining
      # even if eventually no Notifications are populated. This is similar to
      # the scan operation.
      notifications = Notifications(info=None,
                                    warnings=None,
                                    errors=None)

      # Populating Notification type:
      # 1. [ERROR] Stateless host is not supported for seeding. No need to
      # proceed further if we encounter this.
      if HostInfo.IsPxeBooting():
         notifications.errors = \
            self.addNotification(notifications.errors,
               getNotification(UNSUPPORTED_STATELESS_HOST_ID,
                               UNSUPPORTED_STATELESS_HOST_ID))
         return notifications, None

      compInfo = self._getRemovedDgAndUgComponents(baseImageComponents, \
         addOnComponents, addOnRemovedCompNames)

      # Populating notifications for BaseImage/Addon removed and downgraded
      # components:
      # 2. [ERROR] BaseImage and Addon components removed that are not removed
      # via image customization.
      # 3. [WARNING] BaseImage and Addon components downgraded or removed via
      # image customization.
      checks = [
         (REMOVED_BI_COMP_KEY, COMPONENTS_REMOVED,
            'BaseImg Comps are removed: %s', 'error'),
         (DOWNGRADED_BI_COMP_KEY, COMPONENTS_DOWNGRADED,
            'BaseImg Comps are downgraded: %s', 'warning'),
         (REMOVED_ADDON_COMP_KEY, COMPONENTS_REMOVED,
            'Addon Comps are removed: %s', 'error'),
         (DOWNGRADED_ADDON_COMP_KEY, COMPONENTS_DOWNGRADED,
            'Addon Comps are downgraded: %s', 'warning')
      ]

      if PERSONALITY_MANAGER_COMPONENT_REMOVAL_ENABLED:
         checks.extend([
            (USER_REMOVED_BI_COMP_KEY, COMPONENTS_REMOVED,
               'BaseImage components were removed by the user: %s', 'warning'),
            (USER_REMOVED_ADDON_COMP_KEY, COMPONENTS_REMOVED,
               'Addon components were removed by the user: %s', 'warning'),
         ])

      for compKey, msgId, msg, notifType in checks:
         comps = set()
         for c in compInfo[compKey]:
            comps.add(c)
         if comps:
            if notifType == 'warning':
               notifications.warnings = self.addNotification(
                  notifications.warnings, _getImageNotification(msgId,
                                                                  comps))
               logger = logging.warning
            else:
               notifications.errors = self.addNotification(
                  notifications.errors, _getImageNotification(msgId, comps))
               logger = logging.error
            logger(msg, str(compInfo[compKey]))

      # Populating Notification type:
      # 4. [WARNING] Orphan Vibs
      hostVibs = self.currImageProfile.vibs
      orphanVibList = list(self.currImageProfile.GetOrphanVibs())

      if orphanVibList:
         orphanVibs = [hostVibs[vibId] for vibId in orphanVibList]
         orphanVibsInfo = dict()

         try:
            allVibs = VibCollection()
         except Exception as e:
            logging.error('Error while getting list of all VIBs: %s', str(e))
            raise

         for vib in orphanVibs:
            allVibs.AddVib(vib)

         scanResult = allVibs.Scan()
         for vib in orphanVibs:
            if not scanResult.vibs[vib.id].replacedBy:
               # Missing and will be removed during next remediation.
               orphanVibsInfo[vib.name] = vib.versionstr

         if orphanVibsInfo:
            notifications.warnings = \
               self.addNotification(notifications.warnings,
               _getImageNotification(ORPHAN_VIB, orphanVibsInfo))
            logging.debug('List of Orphan Vibs: %s', str(orphanVibsInfo))

      # Populating Notification type:
      # 5. [Warning] Host is pending reboot.
      if self.currHostImage.imgstate == \
            self.currHostImage.IMGSTATE_BOOTBANK_UPDATED:
         notifications.warnings = \
            self.addNotification(notifications.warnings,
               getNotification(SEEDING_PENDING_REBOOT_ID,
                               SEEDING_PENDING_REBOOT_ID))

      return notifications, compInfo

   def _deleteFile(self, fileName):
      try:
         os.remove(fileName)
         logging.info('File %s deleted', fileName)
      except OSError as e:
         logging.warn('Failed to remove %s: %s', fileName, str(e))

   def _cleanOldOfflineDepots(self):
      """Delete Offline Bundles older than 30 minutes
      """
      oldZips = glob.glob(HOST_SEED_DIR_NAME + \
         '/OfflineBundle-[2-9][0-9][0-9][0-9]-[0-1][0-9]-[0-3][0-9]--' \
         '[0-2][0-9].[0-5][0-9].zip')
      now = datetime.today()
      waittime = timedelta(minutes=30)
      for zipFile in oldZips:
         timePast = now - datetime.strptime(zipFile[-21:-4], '%Y-%m-%d--%H.%M')
         if timePast > waittime:
            self._deleteFile(zipFile)

   def getCurrentInfo(self):
      """Get the attributes that describe the current software
         information on the host
      """
      def _getBiAddonRemovedComps(names):
         """Given removed component names, return the reserved components
            from current image profile in a dict.
         """
         rmComps = {}
         for comp in self.currImageProfile.reservedComponents.IterComponents():
            if comp.compNameStr in names:
               if (comp.compNameStr not in rmComps) or \
                     comp.compVersion > rmComps[comp.compNameStr].compVersion:
                  # In case of multiple version of the same component name,
                  # keep only the higher version one, which will be added to
                  # removed component info.
                  rmComps[comp.compNameStr] = comp
         return rmComps

      def _addComponentInfoObj(compInfoDict, comp, isRemovedComp=False):
         """Adds a ComponentInfo instance into the component dict for the given
            component info dict or component object.
            If isRemovedComp is True, log message will indicate the component
            is a user removed one.
         """
         if isinstance(comp, dict):
            componentName = comp['component']
            componentVersion = comp['version']
            componentDisplayName = comp['display_name']
            componentDisplayVersion = comp['display_version']
            componentVendor = comp['vendor']
         else:
            componentName = comp.compNameStr
            componentVersion = comp.compVersionStr
            componentDisplayName = comp.compNameUiStr
            componentDisplayVersion = comp.compVersionUiStr
            componentVendor = comp.vendor

         logPrefix = 'User removed component' if isRemovedComp else 'Component'
         logging.debug(logPrefix + ' details : %s, %s, %s, %s, %s',
                       componentName,
                       componentVersion,
                       componentDisplayName,
                       componentDisplayVersion,
                       componentVendor)
         compInfoDict[componentName] = ComponentInfo(
            componentVersion, componentDisplayName, componentDisplayVersion,
            componentVendor)


      baseImageInfoObj, addOnInfoObj, hardwareSupportInfoObj, \
         notificationsObj, softwareInfoObj = None, None, None, None, None
      componentInfoDict, solutionInfoDict = dict(), dict()

      # Populate the Base Image
      baseImageComponents = None
      if self.currImageProfile.baseimage:
         baseImageComponents = self.currImageProfile.baseimage.components
         baseImageVersion = \
            self.currImageProfile.baseimage.versionSpec.version.versionstring
         baseImageDisplayName = BASEIMAGE_UI_NAME
         baseImageDisplayVersion = \
            self.currImageProfile.baseimage.versionSpec.uiString
         baseImageReleaseDate = self.currImageProfile.baseimage.releaseDate
         logging.debug('BaseImage details : %s, %s, %s, %s',
                       baseImageVersion,
                       baseImageDisplayName,
                       baseImageDisplayVersion,
                       baseImageReleaseDate)
         baseImageInfoObj = BaseImageInfo(baseImageVersion,
                                          baseImageDisplayName,
                                          baseImageDisplayVersion,
                                          baseImageReleaseDate)
      else:
         logging.error('SoftwareInfo must have a baseImage')

      # Populate the Add-on
      addOnComponents, addOnRemovedCompNames = None, None
      if self.currImageProfile.addon:
         addOnRemovedCompNames = self.currImageProfile.addon.removedComponents
         addOnComponents = self.currImageProfile.addon.components
         addonName = self.currImageProfile.addon.nameSpec.name
         addonVersion = \
            self.currImageProfile.addon.versionSpec.version.versionstring
         addonDisplayName = self.currImageProfile.addon.nameSpec.uiString
         addonVendor = self.currImageProfile.addon.vendor
         addonDisplayVersion = self.currImageProfile.addon.versionSpec.uiString
         logging.debug('AddOn details : %s, %s, %s, %s, %s',
                      addonName,
                      addonVersion,
                      addonDisplayName,
                      addonVendor,
                      addonDisplayVersion)
         addOnInfoObj = AddOnInfo(addonName,
                                  addonVersion,
                                  addonDisplayName,
                                  addonVendor,
                                  addonDisplayVersion)

      # Populate Notifications
      notificationsObj, compInfo = self._getNotificationList(
         baseImageComponents, addOnComponents, addOnRemovedCompNames)

      # If we have any error notifications, we need to Return an empty
      # SoftwareInfo. Ensure to retain in the mandatory elements, even if
      # empty : SoftwareInfo[BaseImageInfo, ComponentInfo, SolutionInfo]
      if notificationsObj.errors is not None:
         softwareInfoObj = SoftwareInfo(BaseImageInfo(None, None, None, None),
                                        None,
                                        dict(),
                                        dict(),
                                        None)
         # TODO: Currently we raise an error when Notification contains any
         # error messages as the UI does not handle this. This needs to change
         # once UI is able to consume Notifications and take action.
         # Tracked via - https://jira.eng.vmware.com/browse/ESXLCM-7097
         msg = list()
         for error in notificationsObj.errors:
            msg.append(error.message.default_message)
         logging.error("Software info extract errors: %s", ', '.join(msg))
         raise Errors.SoftwareInfoExtractError(', '.join(msg))

      else:
         # Populate the Components
         for installedComp in compInfo[USER_OVERRIDDEN_COMPS_KEY]:
            _addComponentInfoObj(componentInfoDict, installedComp)


         # Populate removed component info.
         removedCompInfo = {}
         if PERSONALITY_MANAGER_COMPONENT_REMOVAL_ENABLED:
            biAddonRmCompNames = compInfo[USER_REMOVED_BI_COMP_KEY] | \
               compInfo[USER_REMOVED_ADDON_COMP_KEY]
            biAddonRmCompDict = _getBiAddonRemovedComps(biAddonRmCompNames)
            for comp in biAddonRmCompDict.values():
               _addComponentInfoObj(removedCompInfo, comp, isRemovedComp=True)

         softwareInfoObj = SoftwareInfo(baseImageInfoObj,
                                        addOnInfoObj,
                                        componentInfoDict,
                                        solutionInfoDict,
                                        hardwareSupportInfoObj,
                                        removedCompInfo or None)

      installedImageInfo = \
                       InstalledImage.Info(notificationsObj, softwareInfoObj)
      return installedImageInfo

   def getSwSpecAndNotifs(self):
      """Get the attributes that describe the current software
         specification for the ESX host
      """
      baseImageSpecObj, addonSpecObj, hardwareSupportSpecObj, softwareSpecObj, \
         notificationsObj = None, None, None, None, None
      componentSpecDict, solutionsSpecDict = dict(), dict()

      # Populate the Base Image
      baseImageComponents = None
      if self.currImageProfile.baseimage:
         baseImageComponents = self.currImageProfile.baseimage.components
         logging.debug('BaseImage details : %s',
            self.currImageProfile.baseimage.versionSpec.version.versionstring)
         baseImageSpecObj = BaseImageSpec(
            self.currImageProfile.baseimage.versionSpec.version.versionstring)
      else:
         logging.error('SoftwareSpec must have a baseImage')

      # Populate the Add-on
      addOnComponents, addOnRemovedCompNames = None, None
      if self.currImageProfile.addon:
         addOnRemovedCompNames = self.currImageProfile.addon.removedComponents
         addOnComponents = self.currImageProfile.addon.components
         addonSpecName = self.currImageProfile.addon.nameSpec.name
         addonSpecVersion = \
            self.currImageProfile.addon.versionSpec.version.versionstring
         logging.debug('AddOn details : %s, %s',
                      addonSpecName,
                      addonSpecVersion)
         addonSpecObj = AddOnSpec(addonSpecName, addonSpecVersion)

      # Populate Notifications
      notificationsObj, compInfo = self._getNotificationList( \
      baseImageComponents, addOnComponents, addOnRemovedCompNames)

      # If we have any error notifications, we need to Return an empty
      # SoftwareSpec. Ensure to retain in the mandatory elements, even if
      # empty : SoftwareSpec[BaseImageSpec]
      if notificationsObj.errors is not None:
         softwareSpecObj = SoftwareSpec(BaseImageSpec(None),
                                        None,
                                        dict(),
                                        dict(),
                                        None)
      else:
         # Populate the Components
         for installedComp in compInfo[USER_OVERRIDDEN_COMPS_KEY]:
            componentName = installedComp['component']
            if componentName != NSX_COMPONENT:
               componentVersion = installedComp['version']
               logging.debug('Component details : %s, %s',
                        componentName,
                        componentVersion)
               componentSpecDict[componentName] = componentVersion
            else:
               # Skip NSX component from software specification
               logging.info('Skipping NSX %s from software specification in '
                            'depot extraction', NSX_COMPONENT)

         removedComps = set(self.currImageProfile.removedComponents) or None \
            if PERSONALITY_MANAGER_COMPONENT_REMOVAL_ENABLED else None
         softwareSpecObj = SoftwareSpec(baseImageSpecObj,
                                        addonSpecObj,
                                        componentSpecDict,
                                        solutionsSpecDict,
                                        hardwareSupportSpecObj,
                                        removedComps)

      return softwareSpecObj, notificationsObj

   def extractDepot(self, task):
      """1. Acquire transaction lock.
         2. Create hostSeed directory if not already available.
         3. Delete older zips.
         4. Invoke extracting depotDir of live running image.
         5. Create a zip of the depot and store it in the created directory.
         6. Finally delete the depotDir.
         Any failure results in created zip being deleted and acquired lock
         getting freed.
      """
      try:
         # Lock so that multiple scan/apply will not be concurrent.
         # Lock here as during directory creation itself as we don't want two
         # simultaneous extract depot operations to create multiple directories.
         self.currHostImage._getLock()
      except Errors.LockingError as e:
         logging.error("Extract depot failed. "
                       "Failed to get transaction lock: %s", e.msg)
         raise

      try:
         swSpec, notifications = self.getSwSpecAndNotifs()
         task.setProgress(10)
      except Exception as e:
         logging.error("Extract depot failed: %s", str(e))
         self.currHostImage._freeLock()
         raise

      # TODO: Currently we raise an error when Notification contains any
      # error messages as the UI does not handle this. This needs to change
      # once UI is able to consume Notifications and take action.
      # Tracked via - https://jira.eng.vmware.com/browse/ESXLCM-7097
      if notifications.errors is not None:
         msg = list()
         for error in notifications.errors:
            msg.append(error.message.default_message)
         logging.error("Exiting depot extract due to software info errors: %s",
                       ', '.join(msg))
         self.currHostImage._freeLock()
         raise Errors.SoftwareInfoExtractError(', '.join(msg))

      if not os.path.realpath('/var/vmware').startswith('/vmfs/volumes'):
         logging.error("Depot extraction failed: No available persistent "
                       "storage.")
         raise Errors.DataStorageNotFound("Depot extraction failed: "
            "No OSData storage partition is available to extract depot. "
            "Configure persistent storage for the host and retry.")

      logging.debug("Creating directory %s", HOST_SEED_DIR_NAME)
      try:
         os.makedirs(HOST_SEED_DIR_NAME, exist_ok=True)
         task.setProgress(25)
      except Exception as e:
         logging.error("Extract depot failed. "
                       "Failed to create %s: %s", HOST_SEED_DIR_NAME, str(e))
         self.currHostImage._freeLock()
         raise

      self._cleanOldOfflineDepots()
      depotDir = os.path.join(HOST_SEED_DIR_NAME, 'recreateVibs')
      shutil.rmtree(depotDir, ignore_errors=True)
      timeStamp = datetime.today().strftime("%Y-%m-%d--%H.%M")
      bundleFile = '%s-%s.zip' % (DEPOT_FILE_NAME, timeStamp)
      bundlePath = os.path.join(HOST_SEED_DIR_NAME, bundleFile)

      try:
         resVibCache = ReservedVibCache()
      except NoVibCacheError:
         logging.info('No reserved VIB cache available on the host')
         resVibCache = None

      # Get a modified image profile which doesn't contain orphan vibs,
      # solutions and manifests
      newProfile = _getSeedImageProfile(self.currImageProfile)

      # collect all the vibIds from current image profile for which
      # software platform is only esxio.
      esxioVibIds = set()
      for vibid, vib in self.currImageProfile.vibs.items():
         if not vib.hasSystemSoftwarePlatform and not IS_ESXIO and \
                     vib.HasPlatform(Vib.SoftwarePlatform.PRODUCT_ESXIO_ARM):
            esxioVibIds.add(vibid)
      logging.debug('List of esxio VIB Ids:')
      LogLargeBuffer(str(esxioVibIds), logging.debug)

      # Objects/data to help VIB generation
      args = dict(hostImage=self.currHostImage,
                  resVibIds=self.currImageProfile.reservedVibIDs,
                  esxioVibIds=esxioVibIds,
                  resVibCache=resVibCache)
      try:
         Depot.DepotFromImageProfile(newProfile, depotDir,
                                     vibdownloadfn=Depot.GenerateVib,
                                     vendor='VMware, Inc.', vendorcode='vmw',
                                     generateRollupBulletin=False,
                                     vibDownloadArgs=args)
         task.setProgress(65)
         ob = OfflineBundle.OfflineBundle(depotDir)
         ob.Load()
         # TODO: passing checkacceptance as False here since an unsigned
         # vib such as vib-test-cert would cause failure at this point.
         ob.WriteBundleZip(bundlePath, checkacceptance=False)
         task.setProgress(90)
         logging.debug('Depot created successfully at %s', bundlePath)
      except Exception as e:
         logging.error("Extract depot failed: %s", str(e))
         if os.path.exists(bundlePath):
            self._deleteFile(bundlePath)
         raise
      finally:
         shutil.rmtree(depotDir, ignore_errors=True)
         self.currHostImage._freeLock()

      try:
         depotExtractObj = \
            InstalledImage.DepotExtractInfo(bundlePath, notifications, swSpec)
      except Exception as e:
         logging.error("Extract depot failed: %s", str(e))
         if os.path.exists(bundlePath):
            self._deleteFile(bundlePath)
         raise

      task.completeTask(result=vapiStructToJson(depotExtractObj))

class ReservedVibCache(object):
   """Class that manages cache of reserved VIBs.
   """
   _VMFS_VOLUMES = os.path.join(os.sep, 'vmfs', 'volumes')

   # ESXi 7.0 and later has /var/vmware -> /scratch/vmware.
   _DEFAULT_CACHE = HOST_SEED_DIR_NAME

   # ESXi 6.x has no /var/vmware, we will use /scratch.
   _SCRATCH = os.path.join(os.sep, 'scratch')
   _SCRATCH_HOSTSEED = os.path.join(_SCRATCH, 'lifecycle', _HOST_SEED_POSTFIX)

   def __init__(self):

      # All locations with reservedVibs, the first one is the default one.
      self._cacheLocations = []

      # Current cached VIB ID and location sorted by ID.
      self._currCachedVibs = dict()

      # New cached VIBs, used to determine which VIBs to keep or revert
      # when a transaction is finalized.
      self._newCachedVibs = dict()

      # Esxio cached VIB ID and location.
      self._esxioCurrCachedVibs = dict()

      # New esxio cached VIBs, used to determine which VIBs to keep or revert
      # when a transaction is finalized.
      self._esxioNewCachedVibs = dict()

      # Esxio cache vib locations
      self._esxioCachedVibLocations = []

      self._loadCachedVibs()

   def _loadCachedVibs(self):
      """Initiate information about cache locations and cached VIBs.
      """
      def dirExistsAndOnDisk(path):
         return (os.path.isdir(path) and
                 os.path.realpath(path).startswith(self._VMFS_VOLUMES))

      def makeAndAddResVibsDir(resVibDir, cacheLocations):
         if not os.path.exists(resVibDir):
            os.makedirs(resVibDir)
         cacheLocations.append(resVibDir)

      def verifyAndAddVib(vibPath, cacheDict):
         if os.path.isfile(vibPath):
            try:
               vib = Vib.ArFileVib.FromFile(vibPath)
               cacheDict[vib.id] = vibPath
               logging.debug('Added %s VIB in the cache at: %s',
                             vib.id, vibPath)
            except Exception as e:
               logging.debug('File %s is not a VIB: %s', vibPath, str(e))

      if dirExistsAndOnDisk(os.path.dirname(self._DEFAULT_CACHE)):
         # ESXi 7.0 and later with disk-backed OSdata.
         resVibDir = os.path.join(self._DEFAULT_CACHE, 'reservedVibs')
         makeAndAddResVibsDir(resVibDir, self._cacheLocations)
         esxioVibDir = os.path.join(self._DEFAULT_CACHE, 'esxioCachedVibs')
         makeAndAddResVibsDir(esxioVibDir, self._esxioCachedVibLocations)
      elif dirExistsAndOnDisk(self._SCRATCH):
         # ESXi 6.x with disk-backed scratch.
         resVibDir = os.path.join(self._SCRATCH_HOSTSEED, 'reservedVibs')
         makeAndAddResVibsDir(resVibDir, self._cacheLocations)
         esxioVibDir = os.path.join(self._SCRATCH_HOSTSEED, 'esxioCachedVibs')
         makeAndAddResVibsDir(esxioVibDir, self._esxioCachedVibLocations)
      else:
         # If there is no OSdata/scratch or the default location is not
         # not disk-backed, then there cannot be any additional locations.
         logging.warn('OSdata/scratch does not exist or is not backed by '
                      'disk storage, reserved VIBs will not be cached.')
         return

      # Find any additional lifecycle/reservedVibs dirs in non-active
      # OSData/VMFS-L partitions. We do not need to look for additional
      # VFAT scratch since one is enough for temporary transfer during
      # an upgrade.
      for fsPath in HostInfo.GetVmfsOSFileSystems():
         if os.path.realpath(self._cacheLocations[0]).startswith(fsPath):
            # Skip default OSdata.
            continue
         resVibsDir = os.path.join(fsPath, 'vmware', 'lifecycle',
                                   _HOST_SEED_POSTFIX, 'reservedVibs')
         if os.path.isdir(resVibsDir):
            self._cacheLocations.append(resVibsDir)

         esxioResVibsDir = os.path.join(fsPath, 'vmware', 'lifecycle',
                                        _HOST_SEED_POSTFIX, 'esxioCachedVibs')
         if os.path.isdir(esxioResVibsDir):
            self._esxioCachedVibLocations.append(esxioResVibsDir)

      # Scan and record reserved VIBs.
      for cachePath in self._cacheLocations:
         for fn in os.listdir(cachePath):
            filePath = os.path.join(cachePath, fn)
            verifyAndAddVib(filePath, self._currCachedVibs)

      # Scan and record Esxio VIBs.
      for esxioCachePath in self._esxioCachedVibLocations:
         for vibId in os.listdir(esxioCachePath):
            vibPath = os.path.join(esxioCachePath, vibId)
            verifyAndAddVib(vibPath, self._esxioCurrCachedVibs)

   def getVibLocation(self, vibId):
      """Returns cached location of a VIB.
      """
      if not self._cacheLocations:
         raise NoVibCacheError('No VIB cache location is available')
      if not vibId in self._currCachedVibs and \
         not vibId in self._newCachedVibs and \
         not vibId in self._esxioCurrCachedVibs and \
         not vibId in self._esxioNewCachedVibs:
         raise VibNotInCacheError('VIB %s is not available in cached locations'
                                  % vibId)
      return (self._currCachedVibs.get(vibId, None) or
              self._newCachedVibs.get(vibId) or
              self._esxioCurrCachedVibs.get(vibId) or
              self._esxioNewCachedVibs.get(vibId))

   def cacheVibs(self, hostImage, imageprofile, deployDir, vibsToCache):
      """Invokes the appropriate handler to cache the reserved VIB from the
         incoming image profile. In case of a VUM upgrade, all the reserved
         VIBs are extracted from the resvibs.tgz in the deployDir. Otherwise,
         we go through each of the reserved VIB in the profile and add them
         to the cache.
         vibsToCache - List of vibs in ISO workflow needed these vibs to cache
         under reservedVibs folder for seeding.
      """
      def invokeAddVib(vib, hostImage):
         try:
            self.addVib(vib, hostImage, deployDir,
                        not vib.hasSystemSoftwarePlatform)
         except Errors.VibRecreateError as e:
            # There can be legit reason why a reserved VIB cannot be
            # re-created, e.g. the VIB was created by older tools and
            # has timestamp in gzip header.
            logging.exception('Skip storing reserved VIB %s due to '
                              're-creation failure', vib.id)
         except Exception as e:
            # XXX: ignore errors to not break an otherwise successful
            # transaction; we have to fix all other caveats/bugs before
            # this can be removed.
            logging.exception('Unexpected error when storing reserved '
                              'VIB %s', vib.id)
      try:
         if deployDir:
            self.extractResVibs(deployDir, hostImage)
            if vibsToCache is not None:
               for vib in vibsToCache.values():
                  invokeAddVib(vib, hostImage)
         else:
            Downloader.Downloader.setEsxupdateFirewallRule('true')
            for vib in imageprofile.reservedVibs.values():
               # Update reserved vibs with locker DB PR#2968739
               if 'locker' in hostImage.installers:
                  db = hostImage.installers['locker'].database
                  if db is not None and vib.id in db.vibs:
                     vib = db.vibs[vib.id]
                     logging.debug('Updated reserved VIB %s with locker DB.',
                                   vib.id)
               invokeAddVib(vib, hostImage)

         if not IS_ESXIO and hostImage.hasBootBankInstaller:
            # On ESXi, cache ESXio VIB. Also, cache only for bootbank installer
            # and ignore live image because stateless images do not have ESXio
            # VIBs cached as part of deployment but they still have the
            # metadata.
            self._cacheEsxioVibs(hostImage, imageprofile, deployDir)

      finally:
         if not deployDir:
            Downloader.Downloader.setEsxupdateFirewallRule('false')

   def addVib(self, vib, hostImage=None, isoDir=None, isEsxioVib=False):
      """Adds a VIB to the cache, input are VIB metadata object (required),
         HostImage reference for local installed VIB, isoDir for VUM upgrade
         ISO folder, and isEsxioVib to indicate esxio VIB or not.
      """
      if (not isEsxioVib and not self._cacheLocations) \
         or (isEsxioVib and not self._esxioCachedVibLocations):
         raise NoVibCacheError('No VIB cache location is available')

      def getCacheLocation(vibId):
         if isEsxioVib:
            return (os.path.join(self._esxioCachedVibLocations[0], vibId) +
                    '.vib')
         else:
            return os.path.join(self._cacheLocations[0], vibId) + '.vib'

      def isVibCached(currCache, newCache, cacheLocation):
         if vib.id in currCache:
            # VIB is present in the cache, but we want to move to the default
            # cache location.
            vibLocation = currCache[vib.id]
            vibRealPath = os.path.realpath(vibLocation)
            if vibRealPath.startswith(cacheLocation):
               # Already in default cache.
               newCache[vib.id] = vibLocation
               return True

            newCachePath = getCacheLocation(vib.id)
            logging.info('Moving VIB %s from %s to default cache location %s',
                          vib.id, os.path.dirname(vibLocation),
                          os.path.dirname(newCachePath))
            shutil.move(vibLocation, newCachePath)
            currCache[vib.id] = newCachePath
            newCache[vib.id] = newCachePath
            return True

         if vib.id in newCache:
            # VIB just added.
            return True

         return False

      if not vib.payloads:
         # VIB has no payloads, can be an unit test.
         logging.info('Reserved VIB %s does not have any payloads, skip '
                      'caching.', vib.id)
         return

      if ((isEsxioVib and isVibCached(self._esxioCurrCachedVibs, \
                                      self._esxioNewCachedVibs, \
                                      self._esxioCachedVibLocations[0])) \
         or \
         (not isEsxioVib and isVibCached(self._currCachedVibs, \
                                         self._newCachedVibs, \
                                         self._cacheLocations[0]))):
         return

      cachePath = getCacheLocation(vib.id)

      vibDownloadFn = (Depot.VibDownloader if vib.remotelocations
                       else Depot.GenerateVib)
      profile = None
      if hostImage is not None:
         profile = hostImage.GetProfile()

      # VUM ISO upgrade should continue calling GenerateVib PR-3383641
      if (not isoDir and vibDownloadFn == Depot.GenerateVib and
          profile is not None and vib.id not in profile.vibIDs):
         logging.warning("Skip storing VIB %s: VIB is not installed and does "
                         "not have a remote location.", vib.id)
         return

      vibDownloadFn(cachePath, vib, checkdigests=True,
                    extraArgs={'hostImage': hostImage,
                               'isoDir': isoDir,
                               'isReservedVib': True})
      if isEsxioVib:
         self._esxioNewCachedVibs[vib.id] = cachePath
      else:
         self._newCachedVibs[vib.id] = cachePath

   def extractResVibs(self, isoDir, hostImage):
      """Extract and cache all the reserved VIBs present in the ISO/PXE image.
      """
      if not self._cacheLocations:
         raise NoVibCacheError('No VIB cache location is available')

      cachePath = self._cacheLocations[0]
      resvibsTar = hostImage.TryLowerUpperPath(isoDir, 'RESVIBS.TGZ')
      if resvibsTar:
         try:
            os.makedirs(cachePath, exist_ok=True)
            with tarfile.open(name=resvibsTar, mode="r:gz",
                              format=tarfile.GNU_FORMAT) as t:
               for member in t.getmembers():
                  # Extract only the VIB file and not the complete directory.
                  if member.isfile():
                     member.name = os.path.basename(member.name)
                     t.extract(member, cachePath)

                     # Add the newly extracted vib in the new cache
                     vibPath = os.path.join(cachePath, member.name)
                     vib = Vib.ArFileVib.FromFile(vibPath)
                     self._newCachedVibs[vib.id] = vibPath
                     logging.debug('Extracted %s VIB in the cache at: %s',
                             vib.id, vibPath)
         except Exception as e:
            logging.exception('Unexpected error when storing reserved VIBs: %s',
                              str(e))

   def _cacheEsxioVibs(self, hostImage, imageProfile, deployDir):
      """Extract/download all ESXio VIBs into the cache.
      """
      ESXIODPT_TGZ = 'esxiodpt.tgz'
      ESXIO_DEPOT_ZIP = 'esxio-depot.zip'
      if deployDir:
         # ISO upgrade path, extract VIBs in esxio-depot.zip inside esxiodpt.tgz
         esxioDepot = hostImage.TryLowerUpperPath(deployDir, ESXIODPT_TGZ)
         if esxioDepot:
            zipPath = os.path.join(self._esxioCachedVibLocations[0],
                                   ESXIO_DEPOT_ZIP)
            try:
               with tarfile.open(name=esxioDepot, mode="r:gz",
                                 format=tarfile.GNU_FORMAT) as esxioTar:
                  esxioTar.extract(ESXIO_DEPOT_ZIP,
                                   self._esxioCachedVibLocations[0])
               bundle = OfflineBundle.OfflineBundle(zipPath)
               bundle.Load()
               for vibId, vib in bundle.vibs.items():
                  if not vib.hasSystemSoftwarePlatform:
                     localPath = os.path.join(self._esxioCachedVibLocations[0],
                                              vibId + '.vib')
                     if os.path.isfile(localPath):
                        # VIB previously cached, add directly to the new cache
                        # dict.
                        self._esxioNewCachedVibs[vibId] = localPath
                        continue
                     try:
                        Depot.VibDownloader(localPath, vib)
                        self._esxioNewCachedVibs[vibId] = localPath
                        logging.debug('Added %s VIB in the cache at: %s',
                             vibId, localPath)
                     except Exception as e:
                        logging.exception('Skip storing VIB %s due to '
                           'download error' % vibId)
            except Exception:
               logging.exception('Failed to extract ESXio VIBs to reserved VIB '
                                 'cache')
            finally:
               if os.path.isfile(zipPath):
                  try:
                     os.remove(zipPath)
                  except Exception as e:
                     logging.warning('Failed to remove %s: %s', zipPath, str(e))
         else:
            logging.debug('Failed to find the %s at %s', ESXIODPT_TGZ,
                          deployDir)
      else:
         for vib in imageProfile.vibs.values():
            if not vib.hasSystemSoftwarePlatform:
               # Active but non-applicable VIBs belong to ESXio.
               try:
                  self.addVib(vib, hostImage, isEsxioVib=True)
               except Exception:
                  # Tolerate exception when downloading or storing an ESXio VIB.
                  # While this is not the expected behavior, it's better to skip
                  # the VIB, which would only impact host seeding, rather than
                  # blocking the whole remediation. This is in-line with
                  # reserved VIB behavior.
                  logging.exception('Skip caching ESXio VIB %s due to an '
                                    'unexpected error', vib.id)

   def _safeRemove(self, filePath):
      """Safely remove a file.
      """
      if os.path.isfile(filePath):
         try:
            os.remove(filePath)
         except OSError as e:
            logging.warn('Failed to remove %s: %s', filePath, str(e))

   def revert(self):
      """Remove any newly added cache VIBs to revert to previous cache contents.
      """
      def revertToOldCache(newCache, currCache):
         for vibId, vibPath in newCache.items():
            if vibId not in currCache:
               self._safeRemove(vibPath)
         newCache.clear()

      if self._cacheLocations:
         revertToOldCache(self._newCachedVibs, self._currCachedVibs)

      if self._esxioCachedVibLocations:
         revertToOldCache(self._esxioNewCachedVibs, self._esxioCurrCachedVibs)

   def finalize(self):
      """Finalize the cache by keeping new cached VIBs and remove the rest.
      """
      def updateCurrentCache(newCache, currCache):
         for vibId, vibPath in currCache.items():
            if vibId not in newCache:
               # Failure to remove a file would merely leave an unneeded VIB.
               self._safeRemove(vibPath)
         currCache = newCache.copy()
         newCache.clear()

      if self._cacheLocations:
         updateCurrentCache(self._newCachedVibs, self._currCachedVibs)

      if self._esxioCachedVibLocations:
         updateCurrentCache(self._esxioNewCachedVibs, self._esxioCurrCachedVibs)


