########################################################################
# Copyright (c) 2010-2024 Broadcom. All Rights Reserved.
# Broadcom Confidential. The term "Broadcom" refers to Broadcom Inc.
# and/or its subsidiaries.
########################################################################

"""This module defines a class for managing a collection of VIB depots,
   where each depot consists of index, vendor-index, and metadata.zip files.
"""
import logging
import os
import tempfile
import threading
import sys

if sys.version_info[0] >= 3:
   from urllib.request import url2pathname
   from urllib.parse import urlparse
else:
   from urllib import url2pathname
   from urlparse import urlparse

from . import Bulletin
from . import Depot
from . import Errors
from . import Vib
from . import VibCollection
from . import ImageProfile
from . import Downloader
from . import ReleaseCollection
from . import Notification
from .ConfigSchema import ConfigSchemaCollection
from .VibExports import VibExportCollection
from .Utils import PathUtils
from .Utils.Misc import isString
from . import Scan
from . import PERSONALITY_MANAGER_DEPOT_RECALL_ENABLED


log = logging.getLogger('DepotCollection')

def _readContent(f):
   """ Read content from the argument: a file descriptor or a Response object.
       The with clause closes the object.
   """
   with f:
      if hasattr(f, 'read'):
         indexxml = f.read()
      else:
         indexxml = b''
         for chunk in f.iter_content(1024):
            indexxml += chunk
   return indexxml

class OfflineBundleNotLocal(Exception):
   pass

class ProfileAlreadyExists(Exception):
   pass

class VibAlreadyExists(Exception):
   pass

class BaseImageAlreadyExists(Exception):
   pass

class AddonAlreadyExists(Exception):
   pass

class SolutionAlreadyExists(Exception):
   pass

class ManifestAlreadyExists(Exception):
   pass

## Not a real URL, this is used to describe the depot specially created
#  for custom channels created via AddChannel().
#  Note: it needs to use a supported scheme or urlparse() behavior is
#        unpredictable
CUSTOM_DEPOT_URL = 'file://custom/depot/'

class DepotCollection(object):
   """This class holds a collection of depots and their channels and
      metadata files.  It helps with constructing the depot hierarchy,
      managing the channels and metadata files, and
      querying and filtering on VIBs and image profiles within metadatas
      managed by this collection.

      Attributes:
         * depots   - A list of DepotIndex instances
         * channels - A dict of available channels, with the unique
                      channel ID as the key, and each value being an instance
                      of DepotChannel.
         * vibs     - A VibCollection of all Vib packages contained in all
                      the available channels.
         * profiles - An ImageProfileCollection for all profiles contained in
                      all the available channels.
         * vibscandata - An instance of VibScanner. Holds scan results for all
                         the VIBs.
         * baseimages  - A collection of base image objects.
         * addons      - A collection of addon objects.
         * solutions   - A collection of solution objects.
         * manifests   - A collection of manifest objects.
         * notifications - A collection of notification objects.
         * recallReference - A dict of notification IDs -> recalled release
                             units -> absurl of DepotIndex, VendorIndex,
                             MetadataNode of a recalled release unit.
                             e.g., recallReference: {
                             notification1.id: {
                               'recalledComponents': {
                                 (name1, version1): [(d1.url, v1.url, m1.url)],
                                 (name2, version2): [(d1.url, v2.url, m1.url),
                                                     (d1.url, v2.url, m2.url)],
                                 ...
                               },
                               ...
                             },
                             ...
                             }

      Class Variables:
         * INVENTORY_*      - types of inventory collections.
         * INVENTORY_CLASS  - a map from inventory types to their collection
                              class names.
         * RECALLED_*       - types of recalled release unit collections.
         * RECALL_INVENTORY - a map from types of recalled release unit
                              collections to related types of inventory
                              collections.
   """
   INVENTORY_VIBS = 'vibs'
   INVENTORY_PROFILES = 'profiles'
   INVENTORY_BULLETINS = 'bulletins'
   INVENTORY_BASEIMAGES = 'baseimages'
   INVENTORY_ADDONS = 'addons'
   INVENTORY_SOLUTIONS = 'solutions'
   INVENTORY_MANIFESTS = 'manifests'
   INVENTORY_CONFIGSCHEMAS = 'configSchemas'
   INVENTORY_NOTIFICATIONS = 'notifications'
   INVENTORY_VIBEXPORTS = 'vibExports'
   RECALLED_COMPONENTS = 'recalledComponents'
   RECALLED_ADDONS = 'recalledAddons'
   # Make sure components are at the end of the list, as we would like to first
   # recall release units that have a higher level than components, e.g., addons
   RECALL_INVENTORY = {RECALLED_ADDONS: INVENTORY_ADDONS,
                       RECALLED_COMPONENTS: INVENTORY_BULLETINS}

   RELEASE_OBJECT_TYPE = ('vib', 'bulletin', 'baseimage', 'addon', 'solution',
                          'manifest')

   INVENTORY_CLASS = {
      INVENTORY_VIBS: VibCollection.VibCollection,
      INVENTORY_PROFILES: ImageProfile.ImageProfileCollection,
      INVENTORY_BULLETINS: Bulletin.BulletinCollection,
      INVENTORY_BASEIMAGES: ReleaseCollection.BaseImageCollection,
      INVENTORY_ADDONS: ReleaseCollection.AddonCollection,
      INVENTORY_SOLUTIONS: ReleaseCollection.SolutionCollection,
      INVENTORY_MANIFESTS: ReleaseCollection.ManifestCollection,
      INVENTORY_CONFIGSCHEMAS: ConfigSchemaCollection,
      INVENTORY_VIBEXPORTS: VibExportCollection,
   }

   if PERSONALITY_MANAGER_DEPOT_RECALL_ENABLED:
      INVENTORY_CLASS[INVENTORY_NOTIFICATIONS] = \
         Notification.NotificationCollection

   def __init__(self, depots=None, cachedir=None):
      """Constructs a new DepotCollection.
         Parameters:
            * depots   - An initial list of DepotIndex instances
                         to populate the DepotCollection with.
            * cachedir - Path to a directory for caching depot metadata files
         Returns:
            An instance of DepotCollection.
      """
      self._depots = Depot.DepotTreeNode(children=depots)
      self._urlToChannelMap = dict()
      self.cachedir = cachedir
      self._channels = dict()
      self._vibscandata = Scan.VibScanner()
      self._lock = threading.RLock()
      self.recallReference = {}

      for inventoryName, cls in self.INVENTORY_CLASS.items():
         self._setInventory(inventoryName, cls())

      # Create a depot for custom channels. Make sure everything has a URL.
      self.customvendor = Depot.VendorIndex(name="Custom Depot", code="XXX",
                                            indexfile="custom-depot.xml",
                                            baseurl=CUSTOM_DEPOT_URL)
      self.customdepot = Depot.DepotIndex(
                                 url=self._TranslateDepotUrl(CUSTOM_DEPOT_URL),
                                 children=[self.customvendor])

   def _Lock(self):
      """ Wait for _lock is assigned: when this object is deeply copied, _lock
          is removed first since threading.RLock is not clonable.
      """
      while not self._lock:
         from time import sleep
         sleep(0.01)
      self._lock.acquire()

   def _Unlock(self):
      """ Wrapper that matches _Lock.
      """
      self._lock.release()

   def _safeget(self, attr):
      """Gets a class attribute with locking.
      """
      self._Lock()
      try:
         return getattr(self, attr)
      finally:
         self._Unlock()

   def _getInventorySafe(self, inventoryName):
      """Safely gets an internal inventory collection.
      """
      return self._safeget('_%s' % inventoryName)

   def _setInventory(self, inventoryName, val):
      """Sets an internal invetory collection without locking.
      """
      setattr(self, '_%s' % inventoryName, val)

   depots = property(lambda self: self._safeget('_depots').children)
   channels = property(lambda self: self._safeget('_channels'))
   vibscandata = property(lambda self: self._safeget('_vibscandata'))
   vibs = property(lambda self: self._getInventorySafe(
                                                self.INVENTORY_VIBS))
   profiles = property(lambda self: self._getInventorySafe(
                                                self.INVENTORY_PROFILES))
   bulletins = property(lambda self: self._getInventorySafe(
                                                self.INVENTORY_BULLETINS))
   baseimages = property(lambda self: self._getInventorySafe(
                                                self.INVENTORY_BASEIMAGES))
   addons = property(lambda self: self._getInventorySafe(
                                                self.INVENTORY_ADDONS))
   solutions = property(lambda self: self._getInventorySafe(
                                                self.INVENTORY_SOLUTIONS))
   manifests = property(lambda self: self._getInventorySafe(
                                                self.INVENTORY_MANIFESTS))
   configSchemas = property(lambda self: self._getInventorySafe(
                                                self.INVENTORY_CONFIGSCHEMAS))
   vibExports = property(lambda self: self._getInventorySafe(
                                                self.INVENTORY_VIBEXPORTS))
   if PERSONALITY_MANAGER_DEPOT_RECALL_ENABLED:
      notifications = property(lambda self: self._getInventorySafe(
                                                  self.INVENTORY_NOTIFICATIONS))

   @staticmethod
   def _TranslateDepotUrl(durl):
      # Translates offline .zip file paths to a zip:/ URI
      # Adds /index.xml to the end of depot URLs which don't have it
      # NOTE: does not translate local paths to file:/// URI's
      if durl.lower().endswith('.zip'):
         if os.path.splitdrive(durl)[0] or PathUtils.IsDatastorePath(durl):
            # If this is a Windows path starting with a drive letter, we're
            # already done. We specifically check this before trying urlparse,
            # because urlparse will think the drive letter is the URL scheme.
            # Likewise, if this is a [datastore] path, don't mess with it;
            # PathUtils.CreateZipUrl will handle it.
            path = durl
         else:
            scheme, _, path = urlparse(durl)[:3]
            if scheme == '':
               # durl was actually just a relative path. Convert it to an
               # absolute one.
               path = os.path.abspath(durl)
            elif scheme == 'file':
               # durl was a file URL. Use url2pathname to convert the URL path
               # to an OS path.
               path = url2pathname(path)
            else:
               # durl was a URL, but not a file URL. We don't support this.
               raise OfflineBundleNotLocal
         durl = PathUtils.CreateZipUrl(path, 'index.xml')
      elif not durl.lower().endswith('.xml'):
         # Note: forward slashes can be mixed in to Windows pathnames, it
         # still works.
         durl = durl.rstrip('/').rstrip('\\') + '/index.xml'

      return durl


   @staticmethod
   def genFileURL(filePath):
      """ Generate file URL as depot absolute URL:
             zip file -- zip:/...?index.xml
             non zip file -- file://...
      """
      filePath = DepotCollection._TranslateDepotUrl(filePath)
      if os.path.exists(filePath):
         filePath = PathUtils.FilepathToUrl(filePath)
      return filePath

   @staticmethod
   def getDepotFilePaths(depotPaths):
      """ Convert file URLs and zip URLs to file paths.
      """
      for i in range(len(depotPaths)):
         url = depotPaths[i]
         if url.startswith('file:'):
            depotPaths[i] = PathUtils.FileURLToPath(url)
         elif url.startswith('zip:'):
            depotPaths[i] = PathUtils.ZipURLToPath(url)
      return depotPaths

   def ConnectDepots(self, depotUrls, timeout=None,
                     ignoreerror=True, validate=False,
                     schemaVersionCheck=False):
      """Connects to depots and downloads the entire hierarchy of metadata
         files from index.xml down to every metadata.zip.  Parses all metadata
         files including VIB package and image profile metadata.  The
         vibs, profiles, and vibscandata properties will all be updated.
         Parameters:
            * depotUrls - The full remote URLs of the depot index.xml or local
                          offline bundle. If URL ends with '.zip', the URL will
                          be treated as pointing to offline bundle and local
                          file path is required. If the URL does not end
                          in .xml, it will be assumed to be a directory and
                          'index.xml' will be added.
            * timeout   - Network timeout in seconds
            * ignoreerror - If true, exception will be logged and will continue
                            to try to connect to the next depot URL.
            * validate - If True the function will enforce metadata validation
                         while parsing the metadata.zip file.
            * schemaVersionCheck - If True it enforces the json schema version
                                   check.
         Returns:
            A tuple.  First is a list of DepotIndex instances for the depots
            connected to.  The second element is a list of error instances
            for URLs that could not be connected to or parsed.  The error
            instances would be either MetadataDownloadError or
            MetadataFormatError.
            Note: an invalid offline .zip file (not in .zip format) yields
            MetadataDownloadError.
      """

      Downloader.Downloader.setEsxupdateFirewallRule('true')
      try:
         return self._connectDepots(depotUrls, timeout,
                                    ignoreerror, validate,
                                    schemaVersionCheck)
      finally:
         Downloader.Downloader.setEsxupdateFirewallRule('false')

   def _connectDepots(self, depotUrls, timeout=None,
                      ignoreerror=True, validate=False,
                      schemaVersionCheck=False):
      vendorurls = list()
      connected = list()
      errors = list()
      vendorToDepot = dict()
      for durl in depotUrls:
         try:
            durl = self._TranslateDepotUrl(durl)
         except OfflineBundleNotLocal:
            msg = ('local file path is required for bundle zip file :'
                   '(%s) seems to be a remote URI' % (durl))
            log.info(msg)
            e = ValueError(msg)
            errors.append(e)
            if not ignoreerror:
               raise e
            continue
         try:
            log.debug("Downloading depot index.xml from %s", durl)
            d = Downloader.Downloader(durl, timeout=timeout)
            do = d.Open()

            # Use the updated Downloader().url to handle redirecting urls.
            durl = d.url
            depotindex = self.ParseDepotIndex(durl, do)
         except Downloader.DownloaderError as e:
            msg = "Could not download from depot at %s, skipping (%s)" % (
                     durl, str(e))
            log.info(msg)
            err = Errors.MetadataDownloadError(durl, '', msg)
            errors.append(err)
            if not ignoreerror:
               raise err
            continue
         except Exception as e:
            log.info("Error parsing depot index.xml from %s: %s",
                     durl, str(e))
            errors.append(e)
            if not ignoreerror:
               raise
            continue
         connected.append(depotindex)
         vendorurls.extend(v.absurl for v in depotindex.children)
         depotUrl = self.genFileURL(durl)
         self._urlToChannelMap[depotUrl] = list()
         for v in depotindex.children:
            vendorToDepot[v.absurl] = depotUrl

      metaurls = list()
      for vurl in vendorurls:
         try:
            log.debug("Downloading vendor index.xml from %s", vurl)
            d = Downloader.Downloader(vurl, timeout=timeout)
            vindex = self.ParseVendorIndex(vurl, d.Open())
            if vindex:
               durl = vendorToDepot[vurl]
               channels = vindex.channels.values()
               self._urlToChannelMap.setdefault(durl, list()).extend(channels)
            else:
               log.debug('Empty depot for %s', vurl)
         except Downloader.DownloaderError as e:
            msg = ("Could not download from vendor index at %s, skipping (%s)"
                   % (vurl, str(e)))
            log.info(msg)
            err = Errors.MetadataDownloadError(vurl, '', msg)
            errors.append(err)
            if not ignoreerror:
               raise err
            continue
         except Exception as e:
            log.info("Error parsing vendor index.xml from %s: %s",
                      vurl, str(e))
            errors.append(e)
            if not ignoreerror:
               raise
            continue

         metaurls.extend(m.absurl for m in vindex.children)

      for metaurl in metaurls:
         with tempfile.NamedTemporaryFile() as f:
            try:
               log.debug("Downloading %s", metaurl)
               d = Downloader.Downloader(metaurl, local=f.name, fileobj=f,
                                         timeout=timeout)
               metaFile = d.Get()
               # Passing already opened file descriptor instead of
               # filepath to avoid re-opening.
               # The PowerCLI cmdlet failed to re-open already open
               # file while this isn't an issue on linux deployment.
               if metaFile == f.name:
                  metaFile = f
               metanode = self.ParseMetadata(
                                    metaurl, metaFile, validate=validate,
                                    schemaVersionCheck=schemaVersionCheck)
            except Downloader.DownloaderError as e:
               msg = ("Could not download %s, skipping (%s)"
                      % (metaurl, str(e)))
               log.info(msg)
               err = Errors.MetadataDownloadError(metaurl, '', msg)
               errors.append(err)
               if not ignoreerror:
                  raise err
            except Exception as e:
               log.info("Error parsing %s: %s", metaurl, str(e))
               errors.append(e)
               if not ignoreerror:
                  raise

      self._buildCollections(scanvibs=True)

      return connected, errors

   def ProcessNotification(self):
      """For each notification in MetadataNode object (isnotification=True),
         scan all MetadataNode objects (isnotification=False) in all depots
         to find whether the components/addons to be recalled exist. Remove
         them from original places (non-notification meta), and add them to
         new places (notification meta) where related notifications are.
         e.g., assume we only have one depot, within the depot, we only have
         one vendor. Current VendorIndex object contains two metas,
         mn1: isnotification=False, addons: addon1, addon2, bulletins: comp1,
         comp2, vibs:...;
         mn2: isnotification=True, notifications: notif1 (recalledComponents:
         comp1, recalledAddons: addon2).
         After running this method, these two MetadataNode objects become:
         mn1: isnotification=False, addons: addon1, bulletins: comp2, vibs:...;
         mn2: isnotification=True, notifications: notif1 (recalledComponents:
         comp1, recalledAddons: addon2), bulletins: comp1, addons: addon2.
         Also, bulletin/addon/notification collection and recallReference in
         DepotCollection will be updated accordingly.
         Current design for bulletins/vibs/addons/... stores two copies, one
         in MetadataNode objects, and the other in DepotCollection. Here for
         notifications, do the same; for recalled components/addons, only save
         them in MetadataNode objects.
      """
      self._Lock()
      try:
         for depot in self._depots.children:
            for vidx in depot.children:
               for meta in vidx.children:
                 # only look for meta with isnotification=True
                 # it stores notifications and related recalled components
                 if not meta.isnotification:
                    continue
                 for notifId, notification in meta.notifications.items():
                    # if a release unit with name + version in recalled
                    # release unit collection of a notification can be found,
                    # add it to current meta (notification meta) within
                    # _RecallFromMetadata
                    for recalledType, inventoryType in \
                          self.RECALL_INVENTORY.items():
                       recalledColl = getattr(notification, recalledType)
                       if not recalledColl:
                          continue
                       for name, version in recalledColl:
                          # Note: "name" is not available when adding base
                          # images to supported recalled collection.
                          self._RecallFromMetadata(name, version, inventoryType,
                             recalledType, getattr(meta, inventoryType),
                             self.recallReference, notifId)
      finally:
         self._Unlock()

   def _GetNameVersionPair(self, releaseUnit, inventoryType):
      """Given the inventory type of a release unit, return its name and version
         pair
         Parameters:
            * releaseUnit: a release unit object
            * inventoryType: type of the release unit
      """
      if inventoryType == self.INVENTORY_BULLETINS:
         releaseUnitName = releaseUnit.compNameStr
         releaseUnitVersion = releaseUnit.compVersionStr
      elif inventoryType == self.INVENTORY_ADDONS:
         releaseUnitName = releaseUnit.nameSpec.name
         releaseUnitVersion = releaseUnit.versionSpec.version.versionstring
      return releaseUnitName, releaseUnitVersion

   def _RecallFromMetadata(self, name, version, inventoryType, recalledType,
                           inventoryCollection, recallRef, notifId):
      """Given the name and version of a release unit, find all matching release
         units in all depots, record their sources (coming from which depot/
         vendor/meta), then remove them from related metas (non-notification
         metas) & the release unit collection in DepotCollection. For current
         work, only consider component (wrapped by bulletin) and addon.
         Parameters:
            * name, version: (non-display) name and version of the release unit
            * inventoryType: type of the release unit
            * recalledType: type of the recalled release unit
            * inventoryCollection: collection (dict) of current release unit in
                                   a notification meta
                                   (Note: meta within this method is a non-
                                    notification one))
            * recallRef: self.recallReference, a dict to store the information
                         of notification ids with recalled release units
                         together with their name, version, source (depot/vendor
                         /meta). See detailed explanation and example at the
                         beginning of DepotCollection.
            * notifId: notification id, will be used to update recallRef if
                       current notification with id=notifId has release units
                       to recall
      """
      for depot in self._depots.children:
         for vidx in depot.children:
            for meta in vidx.children:
              # only look for meta with isnotification=False
              # it stores pure and original metadata
              if meta.isnotification:
                 continue
              for releaseUnitId, releaseUnit in list(getattr(meta,
                    inventoryType).items()):
                 releaseUnitName, releaseUnitVersion = \
                    self._GetNameVersionPair(releaseUnit, inventoryType)
                 if releaseUnitName == name and releaseUnitVersion == version:
                    # add such release unit to notification meta to
                    # represent recall
                    # bulletins: need to assign every time (if same
                    # release unit is with different releaseDate,
                    # accept the later published one)
                    if inventoryType == self.INVENTORY_BULLETINS:
                       inventoryCollection.AddBulletin(releaseUnit)
                    # other release units: only assign for the first time
                    # (all attributes are equal for equal release units)
                    else:
                      if not releaseUnitId in inventoryCollection:
                         if inventoryType == self.INVENTORY_ADDONS:
                            inventoryCollection.AddAddon(releaseUnit)
                    # record its source (depot/vendor/meta) to recallRef
                    recallRef.setdefault(notifId, {}).setdefault(
                       recalledType, {}).setdefault((releaseUnitName,
                       releaseUnitVersion), []).append(
                       (depot.absurl, vidx.absurl, meta.absurl))
                    # delete from original meta (non-notification meta)
                    del getattr(meta, inventoryType)[releaseUnitId]
                    # delete from release unit collection in DepotCollection
                    getattr(self, '_' + inventoryType).pop(releaseUnitId, None)
                    # Only one release unit will match name + version in a
                    # meta. Or no match, and will not delete or add here.
                    break

   def _RemoveNotification(self, depot):
      """ When a depot is deleted, its notifications should also be deleted.
          Then we have to return recalled components/addons related to the
          notifications back to their original MetadataNode objects according
          to recallReference. So we need to delete a component/an addon from
          MetadataNode object (isNotification=True) with related notification,
          and add it back to its original place, a MetadataNode object
          (isNotification=False).
          Also, according to recallReference, if a component/an addon with
          name+version only has sources of current depot (to be deleted soon),
          the recalled component/addon should also be removed from that
          notification meta and recallReference.
          e.g., assume we only have two depots, and the first depot will be
          deleted. For each depot, we only have one vendor. Current VendorIndex
          object contains two MetadataNode objects,
          mn1: isnotification=False, addons: addon1, bulletins: comp2, vibs:...;
          mn2: isnotification=True, notifications: notif1 (recalledComponents:
          comp3, recalledAddons: addon2), addons: addon2, bulletins: comp3. -->
          a notification in depot1 recalls a component (comp3) and an addon
          (addon2) in depot2
          The VendorIndex object of the other depot contains only one
          MetadataNode object,
          mn3: isnotification=False, bulletins: comp1, vibs:...;
          After running this method, before deletion of the first depot,these
          three MetadataNode objects become:
          mn1: isnotification=False, addons: addon1, bulletins: comp2, vibs:...;
          mn2: isnotification=True, notifications: notif1 (recalledComponents:
          comp3, recalledAddons: addon2);
          mn3: isnotification=False, addons: addon2, bulletins: comp1, comp3,
          vibs:...
          No need to update release unit/notification collection, as they will
          be rebuilt later.
          Parameters:
            * depot: a depot within self._depots.children
      """
      # Step 1: return components/addons back to their original sources other
      # than current depot, since current depot will be deleted
      for vidx in depot.children:
         for meta in vidx.children:
            # only look for meta with isnotification=True
            # it stores notifications and related recalled components
            if not meta.isnotification:
               continue
            for notifId, notification in meta.notifications.items():
               # a component/an addon with such name/version could have been
               # recalled by other notifications, thus the key with
               # current notification id may not exist in
               # recallReference when we are updating this dictionary,
               # so first make sure all the following keys exist.
               if not notifId in self.recallReference:
                  continue
               for recalledType, inventoryType in self.RECALL_INVENTORY.items():
                  recalledColl = getattr(notification, recalledType)
                  if not recalledColl:
                     continue
                  for releaseUnitName, releaseUnitVersion in recalledColl:
                     if not recalledType in \
                           self.recallReference[notifId]:
                        continue
                     if not (releaseUnitName, releaseUnitVersion) in \
                           self.recallReference[notifId][recalledType]:
                        continue
                     # meta here is a notification one
                     for releaseUnitId, releaseUnit in list(getattr(meta,
                           inventoryType).items()):
                        name, version = self._GetNameVersionPair(
                                          releaseUnit, inventoryType)
                        if name == releaseUnitName and \
                              version == releaseUnitVersion:
                           # Since we will delete current depot later, we do
                           # not need to return recalled release units to the
                           # same depot. In real implementation, infoList is
                           # the same as self.recallReference[notifId][
                           # recalledReleaseUnit][(name, version)],
                           # since we are using micro depots: even metadata
                           # and notification from the same online depot will
                           # generate two separate micro depots. But if we
                           # generate depots manually, we can force metadata
                           # and notification to be under the same depot, e.g.,
                           # in unit tests, thus infoList contains less tuples.
                           infoList = [i for i in self.recallReference[notifId][
                              recalledType][(name, version)]
                              if i[0] != depot.absurl]
                           if infoList:
                              # add this release unit back to the sources (non-
                              # notification meta) according to infoList
                              self._RecoverMetadata(releaseUnit, inventoryType,
                                                    infoList)
                           # delete from this notification meta
                           del getattr(meta, inventoryType)[releaseUnitId]
                           # only one release unit will match name + version
                           # or no matching, and will not enter the for loop
                           break
               # since notifId is unique in the whole DepotCollection, deleting
               # a depot will cause all its notifications to be deleted, thus
               # to clear recallReference related to this notifId
               if notifId in self.recallReference:
                  del self.recallReference[notifId]

      # Step 2: if recallReference under certain (name, version) becomes empty
      # after filering out sources of current depot (which means that current
      # depot is their only source), delete this release unit (under current
      # notification id) from the notification meta with this notification,
      # as current depot is to be deleted soon.
      for notifId, recallRef in self.recallReference.items():
         for recalledType, recalledReleaseUnits in recallRef.items():
           for name, version in recalledReleaseUnits:
              # discard the sources whose depot is the same as current depot
              recalledReleaseUnits[(name, version)] = list(filter(lambda info:
                 info[0] != depot.absurl, recalledReleaseUnits[(
                 name, version)]))
              if not recalledReleaseUnits[(name, version)]:
                 # If (name, version) has no other sources other than
                 # current depot, the release unit with (name, version)
                 # needs to be deleted from the meta that has notification
                 # whose id is notifId.
                 self._RemoveFromMetadata(name, version, self.RECALL_INVENTORY[
                    recalledType], depot.absurl, notifId)

      # Step 3: update recallReference
      # delete the key if related value becomes empty
      self._ClearReference()

   def _RecoverMetadata(self, recalledReleaseUnit, inventoryType, infoList):
      """ According to the infoList, a list of tuples (depot.url, vendor.url,
          meta.url), add recalledReleaseUnit back to its original MetadataNode
          objects.
          A special case can be: the original depot is deleted, then no metadata
          can be returned back. In this way, we just delete the metadata from
          the MetadataNode object with notifications, as if it were deleted due
          to the deletion of its original depot. Nothing to do here.
          Parameters:
             * recalledReleaseUnit: a release unit object that is recalled
             * inventoryType: type of the release unit
             * infoList: a list of tuples (depot.url, vendor.url, meta.url) that
                         represents the sources of such release unit
      """
      for dUrl, vUrl, mUrl in infoList:
         for depot in self._depots.children:
            if depot.absurl == dUrl:
               for vidx in depot.children:
                  if vidx.absurl == vUrl:
                     for meta in vidx.children:
                        # only look for meta with isnotification=False
                        # it stores pure and original metadata
                        if meta.absurl == mUrl and not meta.isnotification:
                           if inventoryType == self.INVENTORY_BULLETINS:
                              meta.bulletins.AddBulletin(recalledReleaseUnit)
                           elif inventoryType == self.INVENTORY_ADDONS:
                              meta.addons.AddAddon(recalledReleaseUnit)

   def _RemoveFromMetadata(self, name, version, inventoryType,
                           currentDepotUrl, notifId):
      """Given the name and version of a release unit, and notifId, find the
         MetadataNode object that has notification whose id is notifId, then
         remove it from the notification meta. Here we do not need to add
         it back to its original meta (non-notification meta), as it is in
         current depot, which will be deleted soon.
         Parameters:
            * name, version: (non-display) name and version of the release unit
            * inventoryType: type of the release unit
            * currentDepotUrl: absurl of current depot, can skip current depot
                               during iteration of all depots
            * notifId: notification id, this notification recalled a release
                       unit with name+version, need to find where the related
                       MetadataNode object is.
      """
      for depot in self._depots.children:
         if depot.absurl == currentDepotUrl:
            continue
         for vidx in depot.children:
            for meta in vidx.children:
               # only look for meta with isnotification=True
               # it stores notifications and related recalled components
               if not meta.isnotification:
                  continue
               if not notifId in meta.notifications:
                  continue
               for releaseUnitId, releaseUnit in list(getattr(meta,
                     inventoryType).items()):
                  releaseUnitName, releaseUnitVersion = \
                     self._GetNameVersionPair(releaseUnit, inventoryType)
                  if releaseUnitName == name and releaseUnitVersion == version:
                     # delete from MetadataNode that has notification with
                     # id=notifId
                     del getattr(meta, inventoryType)[releaseUnitId]
                     # only one recalled component can match
                     return

   def _ClearReference(self):
      """If a depot is to be deleted, its notifications will also be deleted,
         thus: 1) under these notification ids within current depot, all
         recalled release units whose original depot is not current one need
         to be returned back; 2) if a recalled release unit under other
         notification ids only has sources of current depot, it should be
         removed from that notification meta, since current depot will be
         deleted and no release units can be returned. In the meantime, we
         need to update recallReference. If value becomes empty, delete its
         related key.
      """
      for notifId in list(self.recallReference):
         for recalledReleaseUnit in list(self.recallReference[notifId]):
            # right now recalledReleaseUnit can be self.RECALLED_COMPONENTS
            # or self.RECALLED_ADDONS
            for name, version in list(self.recallReference[notifId][
                  recalledReleaseUnit]):
               if not self.recallReference[notifId][recalledReleaseUnit][
                     (name, version)]:
                  del self.recallReference[notifId][recalledReleaseUnit][
                     (name, version)]
            if not self.recallReference[notifId][recalledReleaseUnit]:
               del self.recallReference[notifId][recalledReleaseUnit]
         if not self.recallReference[notifId]:
            del self.recallReference[notifId]

   def DisconnectDepots(self, depotUrls, isPman=False):
      """Disconnects from one or more depots.  The DepotIndexes specified
         will be removed from the depots property.
         Also, the vibs, profiles, and vibscandata properties will be updated.
         Parameters:
            * depotUrls   : The URLs of the depots to remove.
            * isPman      : Indicate the new behavior of generating URL; if not
                            set, keep old behavior. XXX: This is temporary; will
                            investigate and remove.
         Exceptions:
            IndexError - if one of the depotUrls is not found.
      """
      if isPman:
         depotUrls = [self.genFileURL(url) for url in depotUrls]
      toremove = set(depotUrls)
      delindex = list()
      self._Lock()
      needrebuild = False
      try:
         for i, depot in enumerate(self._depots.children):
            if depot.absurl in toremove:
               # Remove channels
               for vendoridx in depot.children:
                  for chan in vendoridx.channels.values():
                     try:
                        del self._channels[chan.channelId]
                     except KeyError as e:
                        if isPman:
                           # XXX: workaround of PR 2575639
                           log.warning('vLCM ignores the missing channel %s',
                                       str(e))
                        else:
                           raise
               delindex.append(i)
               toremove.remove(depot.absurl)
               needrebuild = True

         # Delete depots in reverse order
         # Besides, we also need to clear recallReference related to the depots
         # to be deleted. If the depots to be deleted contain notifications, we
         # need to return the recalled components from notification meta back to
         # their original non-notification meta in the depots that will not be
         # deleted currently.
         while delindex:
            delidx = delindex.pop()

            if PERSONALITY_MANAGER_DEPOT_RECALL_ENABLED:
               self._RemoveNotification(self._depots.children[delidx])

            del self._depots.children[delidx]
      finally:
         if isPman:
            for url in depotUrls:
               if url not in toremove:
                  del self._urlToChannelMap[url]
         if needrebuild:
            self._buildCollections(scanvibs=True)
         self._Unlock()
      if len(toremove):
         raise IndexError('URLs %s could not be found in existing depots' %
                          (', '.join(list(toremove))))

   def ParseDepotIndex(self, depotIndexUrl, localpath):
      """Parses the depot index.xml file.  The depot will be added to the
         DepotCollection list or the existing entry will be updated.
         NOTE: Please handle HTTP redirects BEFORE this method is invoked.
               depotIndexUrl should be the redirected URL.  This URL will
               be kept and used for comparison later, so changes in the URL
               may result in duplicate or inconsistent state.
         Parameters:
            * depotIndexUrl - The full remote URL of the depot index.xml file.
                              Used to compute the URLs of vendor-indexes.
            * localpath     - The local path where the index.xml to be parsed
                              resides, or a file object
         Returns:
            The instance of DepotIndex that was created or updated.  To obtain
            the URLs of the vendor indexes, look up the absurl attribute of each
            VendorIndex in the 'children' list attribute of the returned
            instance.
         Exceptions:
            MetadataFormatError - cannot parse metadata
            FileIOError         - cannot access localpath
      """
      try:
         if isString(localpath):
            f = open(localpath, 'r')
         else:
            f = localpath
         xmlData = _readContent(f)
         index = Depot.DepotIndex.FromXml(xmlData, url=depotIndexUrl)
      except EnvironmentError as e:
         raise Errors.FileIOError(localpath, str(e))

      self._Lock()
      try:
         dindex = self._depots.AddChild(index)
      finally:
         self._Unlock()
      return dindex

   def ParseVendorIndex(self, vendorIndexUrl, localpath):
      """Parses a vendor-index.xml file and its channels and metadata file info.
         Inserts the information in the appropriate place in the depot
         hierarchy. Updates the channels attribute with the channels found here.
         NOTE: Please handle HTTP redirects BEFORE this method is invoked.
               depotIndexUrl should be the redirected URL.  This URL will
               be kept and used for comparison later, so changes in the URL
               may result in duplicate or inconsistent state.
         Parameters:
            * vendorIndexUrl - The full remote URL of the vendor-index.xml file.
                               NOTE: This must match the absurl in the child
                               nodes as returned by ParseDepotIndex() above
                               in order for insertion to work correctly.
            * localpath      - The local path where the vendor-index.xml file
                               to be parsed resides, or a file object
         Returns:
            An instance of VendorIndex.  The child nodes contains information
            about metadata URL as well as channels.
         Exceptions:
            MetadataFormatError - cannot parse metadata
            FileIOError         - cannot access localpath
            IndexError - None of the depots contains the vendorIndexUrl passed
                         in.
      """
      index = None
      for depot in self._depots.children:
         #
         # For each depot, does it have the vendor-index at vendorIndexUrl?
         index = depot.GetChildFromUrl(vendorIndexUrl)
         if index:
            try:
               if isString(localpath):
                  f = open(localpath, 'r')
               else:
                  f = localpath
               xml = _readContent(f)
               vidx = Depot.VendorIndex.FromXml(xml, url=vendorIndexUrl)
            except EnvironmentError as e:
               raise Errors.FileIOError(localpath, str(e))

            self._Lock()
            try:
               index = depot.AddChild(vidx)
               self._AddChannels(index)
            finally:
               self._Unlock()
            return index

      # None of the depots contained vendorIndexUrl.  Just give up.
      raise IndexError("Vendor-index URL %s not found in any depot"
                       % vendorIndexUrl)

   def _AddChannels(self, vendorindex):
      # Prune channels in self._channels that don't belong
      vchannels = vendorindex.channels
      rm_list = []
      for channel in self._channels.values():
         if channel.vendorindex.absurl == vendorindex.absurl:
            if channel.name not in vchannels:
               rm_list.append(channel.channelId)
      for channelId in rm_list:
         del self._channels[channelId]

      # Adds the channels from vendorindex to self._channels
      for channel in vchannels.values():
         self._channels[channel.channelId] = channel

   def ParseMetadata(self, metadataUrl, localpath, validate=False,
                     schemaVersionCheck=False):
      """Parses a metadata.zip/notification.zip file. The VIB packages
         and image profiles found in the metadata.zip file will be
         subsequently available for query and scan operations. The vibs
         and profiles properties will be updated.
         Parameters:
            * metadataUrl - The full remote URL of the metadata.zip/
                            notification.zip file.
                            Will be used to compute VIB remotelocations.
                            NOTE: This must match the absurl in the child
                            nodes as returned by ParseVendorIndex() above
                            in order for insertion to work correctly.
            * localpath   - The local path where the metadata.zip/
                            notification.zip file to be parsed resides.
            * validate    - If True the function will enforce metadata
                            validation while parsing the metadata.zip/
                            notification.zip file.
            * schemaVersionCheck - If True the function will enforce schema
                             version check.
         Returns:
            An instance of Depot.MetadataNode
         Exceptions:
            MetadataFormatError - cannot parse metadata.zip/notification.zip
            MetadataIOError     - cannot access or read localpath
            IndexError          - None of the vendor-indexes contain the
                                  metadataUrl passed in.
      """
      # Find the right MetadataNode to replace
      # Note this assumes each MetadataNode has a unique absurl
      for depot in self._depots.children:
         for vidx in depot.children:
            metanode = vidx.GetChildFromUrl(metadataUrl)
            if metanode:
               self._Lock()
               try:
                  metanode.ReadMetadataZip(
                     localpath, validate=validate,
                     schemaVersionCheck=schemaVersionCheck)
                  metanode.SetVibRemoteLocations(metadataUrl)
               finally:
                  self._Unlock()
               self._buildCollections(scanvibs=False)
               return metanode

      raise IndexError("Metadata URL %s not found in the depot" % (metadataUrl))

   def GetChannelsByName(self, names):
      """Returns a set of channel IDs coresponding to the given channel names.
         Parameters:
            * names - an iterable of strings for the channel names to search.
         Returns:
            A set of channel IDs, or the empty set if none of the names
            correspond to an existing channel.  If two channels contain the same
            channel name, they will both be returned.
      """
      channels = set()
      for chanID, channel in self._channels.items():
         if channel.name in names:
            channels.add(chanID)
      return channels

   def GetChannelsByDepotUrl(self, urls):
      """Returns a set of channel IDs for channels in the given depots.
         Parameters:
            * urls - an iterable of depot URLs whose channels should be
                     returned.
         Returns:
            A set of channel IDs, or the empty set of none of the depot URLs
            are valid or contain channels.
      """
      channels = set()
      for url in urls:
         durl = self._TranslateDepotUrl(url)
         if PathUtils.IsAbsolutePath(durl):
            durl = PathUtils.FilepathToUrl(durl)
         depot = self._depots.GetChildFromUrl(durl)
         if depot:
            for vendor in depot.children:
               channels.update(set(c.channelId for c in
                                   vendor.channels.values()))
      return channels

   def _getByChannelId(self, channelId, inventoryName):
      """Returns all objects of one inventory type from a given channel
         as a collection.
         Parameters:
            * channelId     - The channel GUID, from the keys of self.channels
                              or channelId property of a DepotChannel.
            * inventoryName - One of INVENTORY_* constants.
      """
      collection = self.INVENTORY_CLASS[inventoryName]()
      for meta in self.channels[channelId].metadatas:
         collection += getattr(meta, inventoryName)
      return collection

   def _getByDepots(self, depotUrls, inventoryName):
      """Returns all same type objects of one inventory type from one
        or more depots.
         Parameters:
            * depotUrls     - The URLs for the interested depots.
            * inventoryName - One of INVENTORY_* constants.
      """
      collection = self.INVENTORY_CLASS[inventoryName]()
      depotFilter = set(self._TranslateDepotUrl(depotUrl)
                        for depotUrl in depotUrls)
      for depot in self._depots.children:
         if depot.absurl in depotFilter:
            for vendorindex in depot.children:
               for meta in vendorindex.children:
                  collection += getattr(meta, inventoryName)
      return collection

   def GetVibsByChannelId(self, channelId):
      """Returns all the VIBs from a given channel as a VibCollection.
         Parameters:
            * channelId - the channel GUID, from the keys of self.channels
                          or channelId property of a DepotChannel
         Returns:
            An instance of VibCollection.
         Raises:
            KeyError, if channelId is not in self.channels
      """
      return self._getByChannelId(channelId, self.INVENTORY_VIBS)

   def GetVibsByDepots(self, depotUrls):
      """Returns all the VIBs from one or more depots.
         Parameters:
            * depotUrls - the URLs for the interested depots.
         Returns:
            An instance of VibCollection.
      """
      return self._getByDepots(depotUrls, self.INVENTORY_VIBS)

   def GetProfilesByChannelId(self, channelId):
      """Returns all the image profiles from a given channel.
         Parameters:
            * channelId - the channel GUID, from the keys of self.channels
                          or channelId property of a DepotChannel
         Returns:
            An instance of ImageProfileCollection.
         Raises:
            KeyError, if channelId is not in self.channels
      """
      return self._getByChannelId(channelId, self.INVENTORY_PROFILES)

   def GetProfilesByDepots(self, depotUrls):
      """Returns all the image profiles from one or more depots.
         Parameters:
            * depotUrls - The URLs for the interested depots.
         Returns:
            An instance of ImageProfileCollection.
      """
      return self._getByDepots(depotUrls, self.INVENTORY_PROFILES)

   def GetBulletinsByChannelId(self, channelId):
      """Returns all the bulletins from a given channel.
         Parameters:
            * channelId - the channel GUID, from the keys of self.channels
                          or channelId property of a DepotChannel
         Returns:
            An instance of BulletinCollection.
         Raises:
            KeyError, if channelId is not in self.channels
      """
      return self._getByChannelId(channelId, self.INVENTORY_BULLETINS)

   def GetBulletinsByDepots(self, depotUrls):
      """Returns all bulletins from one or more depots.
         Parameters:
            * depotUrls - The URLs for the interested depots.
         Returns:
            An instance of BulletinCollection.
      """
      return self._getByDepots(depotUrls, self.INVENTORY_BULLETINS)

   def GetBaseImagesByChannelId(self, channelId):
      """Returns all the base images from a given channel.
         Parameters:
            * channelId - The channel GUID, from the keys of self.channels
                          or channelId property of a DepotChannel
         Returns:
            An instance of BaseImageCollection.
         Raises:
            KeyError - If channelId is not in self.channels
      """
      return self._getByChannelId(channelId, self.INVENTORY_BASEIMAGES)

   def GetBaseImagesByDepots(self, depotUrls):
      """Returns all the base images from one or more depots.
         Parameters:
            * depotUrls - The URLs for the interested depots.
         Returns:
            An instance of BaseImageCollection.
      """
      return self._getByDepots(depotUrls, self.INVENTORY_BASEIMAGES)

   def GetAddonsByChannelId(self, channelId):
      """Returns all the addons from a given channel.
         Parameters:
            * channelId - The channel GUID, from the keys of self.channels
                          or channelId property of a DepotChannel
         Returns:
            An instance of AddonCollection.
         Raises:
            KeyError - If channelId is not in self.channels
      """
      return self._getByChannelId(channelId, self.INVENTORY_ADDONS)

   def GetAddonsByDepots(self, depotUrls):
      """Returns all the addons from one or more depots.
         Parameters:
            * depotUrls - The URLs for the interested depots.
         Returns:
            An instance of AddonCollection.
      """
      return self._getByDepots(depotUrls, self.INVENTORY_ADDONS)

   def GetSolutionsByChannelId(self, channelId):
      """Returns all the solutions from a given channel.
         Parameters:
            * channelId - The channel GUID, from the keys of self.channels
                          or channelId property of a DepotChannel
         Returns:
            An instance of SolutionCollection.
         Raises:
            KeyError - If channelId is not in self.channels
      """
      return self._getByChannelId(channelId, self.INVENTORY_SOLUTIONS)

   def GetSolutionsByDepots(self, depotUrls):
      """Returns all the solutions from one or more depots.
         Parameters:
            * depotUrls - The URLs for the interested depots.
         Returns:
            An instance of SolutionCollection.
      """
      return self._getByDepots(depotUrls, self.INVENTORY_SOLUTIONS)

   def GetManifestsByChannelId(self, channelId):
      """Returns all the manifests from a given channel.
         Parameters:
            * channelId - The channel GUID, from the keys of self.channels
                          or channelId property of a DepotChannel
         Returns:
            An instance of ManifestCollection.
         Raises:
            KeyError - If channelId is not in self.channels
      """
      return self._getByChannelId(channelId, self.INVENTORY_MANIFESTS)

   def GetManifestsByDepots(self, depotUrls):
      """Returns all the manifests from one or more depots.
         Parameters:
            * depotUrls - The URLs for the interested depots
         Returns:
            An instance of ManifestCollection.
      """
      return self._getByDepots(depotUrls, self.INVENTORY_MANIFESTS)

   def ScanVibs(self):
      """Populates the vibscandata property by scanning all the VIB package
         dependencies of the internal vibs collection.
      """
      self._Lock()
      try:
         self._vibscandata = self._vibs.Scan()
      finally:
         self._Unlock()

   def AddChannel(self, name):
      """Adds a new channel to the depotcollection.  This is mostly designed
         for persisting user-created image profiles.  A special URL will be
         created for the channel.
         The channel will be added to a common depot created for the purpose
         of persisting custom channels.  The common depot and all custom
         channels can be removed using DisconnectDepots.
         If the channel already exists, then this is a NOP.
         Parameters:
            * name       - A friendly name for the new channel
         Returns:
            The new DepotChannel instance.
      """
      metaurl = CUSTOM_DEPOT_URL + name
      metanode = Depot.MetadataNode(productIds=[Depot.DEPOT_PRODUCT],
                                    channels=[name],
                                    url=metaurl)
      log.debug("Adding custom channel %s", metanode.absurl)

      with self._lock:
         self._depots.AddChild(self.customdepot)
         self.customvendor.AddChild(metanode)
         self._AddChannels(self.customvendor)
      return self.customvendor.channels[name]

   def AddProfile(self, profile, channelId, replace=True):
      """Adds or replaces an image profile in an existing channel.  The
         channel should have been added using AddChannel above.
         Parameters:
            * profile   - an ImageProfile instance to add/replace.
            * channelId - the channel ID of the channel to affect
            * replace   - if True, allows an existing image profile
                          to be replaced, where existing means an image profile
                          with the same ID.  If False, no profile with the same
                          ID can exist in this DepotCollection.
         Raises:
            KeyError    - if the channelId does not correspond to a valid
                          channel.
            ProfileAlreadyExists - if the image profile already exists and
                                   replace is False.
            ValueError  - The image profile to be replaced is read-only and
                          cannot be modified.
      """
      if profile.profileID not in self._profiles:
         # A new profile.  Add it to first meta of the channel.
         with self._lock:
            meta = self._channels[channelId].metadatas[0]
            meta.profiles[profile.profileID] = profile
            self._profiles[profile.profileID] = profile
      elif replace:
         # Profile exists already and we can replace it
         if not self._profiles[profile.profileID].readonly:
            with self._lock:
               for meta in self._channels[channelId].metadatas:
                  if profile.profileID in meta.profiles:
                     meta.profiles[profile.profileID] = profile
               self._profiles[profile.profileID] = profile
         else:
            raise ValueError("Image Profile '%s' is read-only and cannot be "
                             "updated"
                             % (self._profiles[profile.profileID].name))
      else:
         raise ProfileAlreadyExists("Image Profile '%s' already exists and "
                                    "replace=False"
                             % (self._profiles[profile.profileID].name))

   def RemoveProfile(self, profileID, channelname=''):
      """Removes an image profile from the collection for a channel. If
         channelname is empty, remove the image profile from all the channels.
         Parameters:
            * profileID - the ID of the profile to remove.
            * channelname - the name of the channel from which to remove the
                            profile.
         Raises:
            KeyError - the profile does not exist in the collection.
            ValueError - the profile is not found in the specified channel
      """
      with self._lock:
         keep = False
         deleted = False
         for chan in self._channels.values():
            for meta in chan.metadatas:
              if profileID in meta.profiles:
                 if len(channelname) == 0 or chan.name == channelname:
                    del meta.profiles[profileID]
                    deleted = True
                 else:
                    keep = True

         # Since we only support removing from customer channel. If the profile
         # is not in the specified channel but in other channels, which means
         # the profile cannot be removed as it is from a real depot.
         if keep and not deleted:
            raise ValueError("The image profile '%s' cannot be removed as it "
               "is from a read-only depot." % (self._profiles[profileID].name))

         # Not in any channels, remove it from top-level
         if not keep:
            del self._profiles[profileID]

   def _AddReleaseUnit(self, releaseUnit, channelId, replace,
                       inventoryType, errorType):
      """Common method for adding base image, addon, solution or manifest into
         depot collection.
      """
      def _setReleaseUnit(meta, releaseUnit):
         getattr(meta, inventoryType)[releaseUnit.releaseID] = releaseUnit

      releaseID = releaseUnit.releaseID
      if releaseID not in getattr(self, inventoryType):
         # Add/replace a release unit to first meta of the channel.
         with self._lock:
            meta = self._channels[channelId].metadatas[0]
            _setReleaseUnit(meta, releaseUnit)
            _setReleaseUnit(self, releaseUnit)
      elif replace:
         # Release unit exists already and we can replace it
         with self._lock:
            for meta in self._channels[channelId].metadatas:
               if releaseID in getattr(meta, inventoryType):
                  _setReleaseUnit(meta, releaseUnit)
            _setReleaseUnit(self, releaseUnit)
      else:
         raise errorType("%s'%s' already exists" %
                         (releaseUnit.releaseType, releaseID))

   def _RemoveReleaseUnit(self, releaseID, channelName, inventoryName):
      """Common method for removing base image, addon, solution or manifest from
         depot collection.
      """
      def _delReleaseUnit(meta, releaseID):
         del getattr(meta, inventoryName)[releaseID]

      with self._lock:
         keep = False
         deleted = False
         for chan in self._channels.values():
            for meta in chan.metadatas:
               if releaseID in getattr(meta, inventoryName):
                  if len(channelName) == 0 or chan.name == channelName:
                     _delReleaseUnit(meta, releaseID)
                     deleted = True
                  else:
                     keep = True

         # Since we only support removing from customer channel. If the release
         # unit is not in the specified channel but in other channels, which
         # means the release unit cannot be removed as it is from a real depot.
         if keep and not deleted:
            raise ValueError("The release unit '%s' cannot be removed as it is "
                             "from a read-only depot." % (releaseID))

         # Not in any channels, remove it from top level.
         if not keep and releaseID in getattr(self, inventoryName):
            _delReleaseUnit(self, releaseID)

   def AddBaseImage(self, baseimage, channelId, replace=True):
      """Adds or replaces a base image in an existing channel.  The
         channel should have been added using AddChannel above.
         Parameters:
            * baseimage - A BaseImage object to add/replace.
            * channelId - The channel ID of the channel to affect
            * replace   - If True, allows an existing base image to be replaced,
                          where existing means a base image with the same
                          releaseID.  If False, no base image with the same
                          ID can appear in this DepotCollection.
         Raises:
            KeyError - If the channelId does not correspond to a valid channel
            BaseImageAlreadyExists -
               If the base image already exists and replace is False.
      """
      self._AddReleaseUnit(baseimage, channelId, replace,
                           self.INVENTORY_BASEIMAGES,
                           BaseImageAlreadyExists)

   def RemoveBaseImage(self, releaseID, channelname=''):
      """Removes a base image from the collection for a channel. If channelname
         is empty, remove the base image from all the channels.
         Parameters:
            * releaseID   - The ID of the base image to remove.
            * channelname - The name of the channel from which to remove the
                 base image.
         Raises:
            KeyError   - The base image does not exist in the collection.
            ValueError - The base image is not found in the specified channel
      """
      self._RemoveReleaseUnit(releaseID, channelname, self.INVENTORY_BASEIMAGES)

   def AddAddon(self, addon, channelId, replace=True):
      """Adds or replaces an addon in an existing channel.  The
         channel should have been added using AddChannel above.
         Parameters:
            * addon     - An Addon object to add/replace.
            * channelId - The channel ID of the channel to affect
            * replace   - If True, allows an existing addon to be replaced,
                          where existing means an addon with the same
                          releaseID.  If False, no addon with the same
                          ID can appear in this DepotCollection.
         Raises:
            KeyError    - If the channelId does not correspond to a valid
                          channel.
            AddonAlreadyExists  - If the addon already exists and replace
                          is False.
      """
      self._AddReleaseUnit(addon, channelId, replace, self.INVENTORY_ADDONS,
                           AddonAlreadyExists)

   def RemoveAddon(self, releaseID, channelname=''):
      """Removes an addon from the collection for a channel. If channelname
         is empty, remove the addon from all the channels.
         Parameters:
            * releaseID   - The ID of the addon to remove.
            * channelname - The name of the channel from which to remove the
                  addon.
         Raises:
            KeyError   - The addon does not exist in the collection.
            ValueError - The addon is not found in the specified channel
      """
      self._RemoveReleaseUnit(releaseID, channelname, self.INVENTORY_ADDONS)

   def AddSolution(self, solution, channelId, replace=True):
      """Adds or replaces a solution in an existing channel.  The
         channel should have been added using AddChannel above.
         Parameters:
            * solution  - A Solution object to add/replace.
            * channelId - The channel ID of the channel to affect
            * replace   - If True, allows an existing solution to be replaced,
                          where existing means a solution with the same
                          releaseID.  If False, no solution with the same
                          ID can appear in this DepotCollection.
         Raises:
            KeyError    - If the channelId does not correspond to a valid
                          channel.
            SolutionAlreadyExists  - If the solution already exists and replace
                                     is False.
      """
      self._AddReleaseUnit(solution, channelId, replace,
                           self.INVENTORY_SOLUTIONS, SolutionAlreadyExists)

   def RemoveSolution(self, solutionID, channelname=''):
      """Removes a solution from the collection for a channel. If channelname
         is empty, remove the solution from all the channels.
         Parameters:
            * solutionID   - The ID of the solution to remove.
            * channelname  - The name of the channel from which to remove the
                             solution.
         Raises:
            KeyError   - The solution does not exist in the collection.
            ValueError - The solution is not found in the specified channel
      """
      self._RemoveReleaseUnit(solutionID, channelname, self.INVENTORY_SOLUTIONS)

   def AddManifest(self, manifest, channelId, replace=True):
      """Adds or replaces a manifest in an existing channel.  The
         channel should have been added using AddChannel above.
         Parameters:
            * manifest  - A Manifest object to add/replace.
            * channelId - The channel ID of the channel to affect
            * replace   - If True, allows an existing manifest to be replaced,
                          where existing means a manifest with the same
                          releaseID.  If False, no manifest with the same
                          ID can appear in this DepotCollection.
         Raises:
            KeyError    - If the channelId does not correspond to a valid
                          channel.
            ManifestAlreadyExists - If the manifest already exists and replace
                                    is False.
      """
      self._AddReleaseUnit(manifest, channelId, replace,
                           self.INVENTORY_MANIFESTS, ManifestAlreadyExists)

   def RemoveManifest(self, manifestID, channelname=''):
      """Removes a manifest from the collection for a channel. If channelname
         is empty, remove the manifest from all the channels.
         Parameters:
            * manifestID   - The ID of the manifest to remove.
            * channelname  - The name of the channel from which to remove the
                             manifest.
         Raises:
            KeyError   - The manifest does not exist in the collection.
            ValueError - The manifest is not found in the specified channel
      """
      self._RemoveReleaseUnit(manifestID, channelname, self.INVENTORY_MANIFESTS)

   def AddVib(self, vib, channelId, replace=True):
      """Adds or replaces a VIB in an existing channel. The channel should have
         have been added using AddChannel above.
         Parameters:
            * vib       - a BaseVib instance to add/replace.
            * channelId - the channel ID of the channel to affect
            * replace   - if True, allows an existing VIB with the same ID to be
                          replaced. If False, no VIB with the same ID can exist
                          in this DepotCollection.
         Raises:
            KeyError    - if the channelId does not correspond to a valid
                          channel.
            VibAlreadyExists - if the VIB already exists and replace is
                               False.
      """
      if vib.id not in self._vibs:
         # A new VIB. Add it to first meta of the channel.
         with self._lock:
            meta = self._channels[channelId].metadatas[0]
            meta.vibs[vib.id] = vib
            self._vibs[vib.id] = vib
      elif replace:
         # VIB exists already and we can replace it
         with self._lock:
            for meta in self._channels[channelId].metadatas:
               if vib.id in meta.vibs:
                  meta.vibs[vib.id] = vib
            self._vibs[vib.id] = vib
      else:
         raise VibAlreadyExists("VIB '%s' already exists and replace=False"
                                % (self._vibs[vib.id].name))
      self._buildCollections(scanvibs=True)

   def AddVibFromUrl(self, url, channelId, replace=True):
      if not urlparse(url).scheme and not os.path.isabs(url):
         msg = "Not a valid absolute path: %s" % url
         raise Errors.VibDownloadError(url, None, msg)
      try:
         d = Downloader.Downloader(url)
         vib = Vib.ArFileVib.FromFile(d.Open())
         vib.remotelocations = [url]
         # We don't know how much time is going to elapse before we try to
         # do anything with VIB payloads (like export an image profile), so
         # just close the file. We'll re-open it when it's needed.
         vib.Close()
      except Downloader.DownloaderError as e:
         raise Errors.VibDownloadError(url, None, str(e))
      self.AddVib(vib, channelId, replace)
      return vib

   def RemoveVib(self, vibID):
      """Removes an VIB from the collection.
         Parameters:
            * vibID - the ID of the VIB to remove.
         Raises:
            KeyError - the VIB does not exist in the collection.
      """
      with self._lock:
         del self._vibs[vibID]
         for chan in self._channels.values():
            for meta in chan.metadatas:
               try:
                  del meta.vibs[vibID]
               except KeyError:
                  pass
      self._buildCollections(scanvibs=True)

   def GetBaseImage(self, releaseID):
      ''' Get the base image with provided releaseID. '''
      return self.baseimages[releaseID]

   def GetAddon(self, releaseID):
      ''' Get the addon with provided release ID. '''
      return self.addons[releaseID]

   def _buildCollections(self, scanvibs=False):
      ''' Rebuilds collections of VIBs, profiles, bulletins and release units.
          If scanvibs is True, also rebuild the vibscandata.
      '''
      #
      # It is OK if there are duplicate metadata.zips in different channels.
      # The VibCollection Merge method would simply add the same Vib instances
      # together, which results in a VibCollection with the same contents.
      #
      self._Lock()
      try:
         for inventoryName, cls in self.INVENTORY_CLASS.items():
            collection = cls()
            # no channel support for self.INVENTORY_NOTIFICATIONS
            # use depot URLs
            if inventoryName == self.INVENTORY_NOTIFICATIONS:
               if PERSONALITY_MANAGER_DEPOT_RECALL_ENABLED:
                  depotUrls = [depot.absurl for depot in self._depots.children]
                  collection += self._getByDepots(depotUrls, inventoryName)
            else:
               for channel in self._channels:
                  collection += self._getByChannelId(channel, inventoryName)
            self._setInventory(inventoryName, collection)

         if scanvibs:
            self.ScanVibs()
      finally:
         self._Unlock()


   def CalculateMicroDepots(self, imageProfile, getURL=False):
      """ Calculate the micro depots that contain all image related objects in
          the provided image profile.

          Return value can be in url format or file path format, depending on
          the argument getURL.
      """
      depotPaths = set()
      knownCompIDs = set(imageProfile.components.GetComponentNameIds())
      knownCompIDs.update(imageProfile.reservedComponents.GetComponentNameIds())

      # Collect all release units into a set of (type, ID).
      releaseUnits = imageProfile.GetAllReleaseUnits()

      for url in self._urlToChannelMap:
         for channel in self._urlToChannelMap[url]:
            # Find and remove release units.
            if channel.RemoveMatchedReleaseUnits(releaseUnits):
               depotPaths.add(url)

            # Find and remove component IDs
            if channel.RemoveMatchedComponentIDs(knownCompIDs):
               depotPaths.add(url)

      if releaseUnits:
         msg = ('The following data are not in depots: \n\t%s' %
                '\n\t'.join([relID + ' (' + relType + ')'
                             for relType, relID in releaseUnits]))
         raise Errors.MetadataNotFoundError(releaseUnits, msg)

      if knownCompIDs:
         msg = ('The following componnets are not in depots: \n\t%s' %
                '\n\t'.join(knownCompIDs))
         raise Errors.MetadataNotFoundError(knownCompIDs, msg)

      if getURL:
         return list(depotPaths)
      return self.getDepotFilePaths(list(depotPaths))

   def GetRelatedVibs(self, imageProfile):
      """ Generate a VibCollection that only contains the vibs from the
          micro depots that overlap with the provided image profile.
      """
      relatedVibs = VibCollection.VibCollection()
      urls = self.CalculateMicroDepots(imageProfile, True)
      for url in urls:
         for channel in self._urlToChannelMap[url]:
            for meta in channel.metadatas:
               relatedVibs += meta.vibs
      return relatedVibs

   def GetReleaseObjects(self, depotUrl):
      """ Get all collections of release objects for a depot.
      """
      self._Lock()
      try:
         depotUrl = self.genFileURL(depotUrl)
         if depotUrl not in self._urlToChannelMap:
            raise Errors.DepotNotFoundError(
                     'Depot not found in depot collection', depotUrl)
         releaseObjects = {rot : self.INVENTORY_CLASS[rot + 's']()
                           for rot in self.RELEASE_OBJECT_TYPE}
         for channel in self._urlToChannelMap[depotUrl]:
            for meta in channel.metadatas:
               for rot in self.RELEASE_OBJECT_TYPE:
                  releaseObjects[rot] += meta.GetReleaseObjects(rot)
         return releaseObjects
      finally:
         self._Unlock()

   def GetDepotURLs(self):
      """ Get all depot URLs.
      """
      self._Lock()
      try:
         return self.getDepotFilePaths(list(self._urlToChannelMap.keys()))
      finally:
         self._Unlock()

   def GetVibConfigSchemas(self, vibs):
      """ Retrieve config schemas for the given VIBs.

         Parameters:
            vibs: A VibCollection containing the VIBs for which the config
                  schemas are retrieved for.
         Returns:
            The retrieved config schemas for the given VIBs
      """
      vibConfigSchemas = ConfigSchemaCollection()

      missingVibIds = set()
      for vib in vibs.values():
         csTag = vib.GetConfigSchemaTag()
         if csTag is not None:
            if csTag.schemaId in self.configSchemas:
               vibConfigSchemas.AddConfigSchema(self.configSchemas[csTag.schemaId])
            else:
               missingVibIds.add(vib.id)

      if missingVibIds:
         log.error("Failed to find config schema for VIB(s): %s",
                   ", ".join(missingVibIds))

      return vibConfigSchemas if vibConfigSchemas else None

   def GetVibExports(self, vibs):
      """ Retrieve vmsg files for given VIBs.

         Parameters:
            vibs: A VibCollection containing the VIBs for which the config
                  schemas are retrieved for.
         Returns:
            The retrieved collection of vib exports for the given VIBs
      """
      vibExports = VibExportCollection()

      for vib in vibs.values():
         csTag = vib.GetConfigSchemaTag()
         if csTag is not None and csTag.schemaId in self.vibExports:
            for export in self.vibExports[csTag.schemaId]:
               vibExports.AddVibExport(export)

      return vibExports if vibExports else None
