# Copyright 2018-2023 VMware, Inc.
# All rights reserved. -- VMware Confidential

"""ESX-native filesystem operations.
"""
import logging
from math import ceil
import os
import struct
from time import sleep

from systemStorage import IS_ESX
if IS_ESX:
   from systemStorage import (FS_TYPE_VFAT, FS_TYPE_VMFS, FS_TYPE_VMFS_L,
                              FS_TYPE_VMFSOS, MiB,
                              SYSTEM_DISK_SIZE_LARGE_MB,
                              SYSTEM_DISK_SIZE_MEDIUM_MB)
   from esxutils import runCli, isEsxInAVm, getHardwareUuid
   from vmware import vsi

log = logging.getLogger('esxfs')
log.setLevel(logging.DEBUG)

SIZEOF_FS_UUID = 16
FSS_VOLUMES_ROOT = os.path.join(os.path.sep, 'vmfs', 'volumes')

def reverseHexStr(s):
   octets = [s[i : i + 2] for i in range(0, len(s), 2)]
   octets.reverse()
   return "".join(octets)

class EsxFsUuid(bytes):
   """Disk volume UUID is 8-8-4-12 format in /vmfs/volumes.
   """

   def __init__(self, *args, **kwargs):
      super().__init__()
      try:
         uuid = struct.unpack("<IIH6s", self)
      except struct.error:
         raise ValueError("%s: invalid FS UUID" % super().__str__())
      self._timeLo, self._timeHi, self._rand, self._mac = uuid

   @classmethod
   def fromString(cls, uuidStr):
      """Convert from a contiguous hex string or '-' delimited format.

      e.g. "73ad723947c2afa89d19309c7a36eab8" is equivalent to
           "3972ad73-a8afc247-199d-309c7a36eab8".
      """
      if not uuidStr:
         raise ValueError("%s: invalid FS UUID" % uuidStr)

      if '-' in uuidStr:
         try:
            timeLo, timeHi, rand, mac = uuidStr.split('-')
         except ValueError:
            raise ValueError("%s: invalid FS UUID" % uuidStr)

         uuidStr = (reverseHexStr(timeLo) + reverseHexStr(timeHi) +
                    reverseHexStr(rand) + mac)

      return cls(bytes.fromhex(uuidStr))

   def __str__(self):
      """Return this FS UUID as a human-readable string.
      """
      return "%08x-%08x-%04x-%s" % (self._timeLo, self._timeHi, self._rand,
                                    ''.join(["%02x" % b for b in self._mac]))

if IS_ESX:

   class FssVolume(object):
      """Class representing an ESX filesystem volume.
      """

      def __init__(self, vsiVolumeInfo):
         self._vsiInfo = vsiVolumeInfo
         self._statvfs = None

      @classmethod
      def fromFssVolume(cls, volume, *args, **kwargs):
         return cls(volume._vsiInfo, *args, **kwargs)

      @property
      def name(self):
         """Volume name (typically the volume UUID or device name).

         E.g. '5cedcf2f-375b1ba1-3eca-000c29b7e105', or 'naa.58ce38ee2086b301:1'
         """
         return self._vsiInfo['volumeName']

      @property
      def label(self):
         """Volume label (e.g. 'datastore1').
         """
         return self._vsiInfo['volumeLabel']

      @property
      def uuid(self):
         """Volume UUID, or None if this volume doesn't have a UUID.
         """
         try:
            return EsxFsUuid.fromString(self.name)
         except ValueError:
            # Only certain ESX volumes have an FS UUID (typically the bootbanks,
            # OSDATA, and VMFS datastores). So we end up here in all other cases
            # (e.g. for partitions which were formatted by another OS than ESX).
            return None

      @property
      def path(self):
         """Absolute path to the volume mount point.

         E.g. '/vmfs/volumes/5cedcf2f-375b1ba1-3eca-000c29b7e105' or
              '/vmfs/volumes/naa.58ce38ee2086b301:1'.
         """
         return os.path.join(FSS_VOLUMES_ROOT, self.name)

      @property
      def devPath(self):
         """Volume device path (e.g. '/dev/disks/mpx.vmhba0:C0:T0:L0:7').
         """
         return os.path.join(os.path.sep, 'dev', 'disks',
                             self._vsiInfo['headExtent'])

      @property
      def partNum(self):
         """Volume partition number.
         """
         _, _, partNum = self._vsiInfo['headExtent'].rpartition(':')
         return int(partNum)

      @property
      def diskName(self):
         """Name of the disk containing this volume
         (e.g. 'mpx.vmhba0:C0:T0:L0').
         """
         diskName, _, _ = self._vsiInfo['headExtent'].rpartition(':')
         return diskName

      @property
      def diskPath(self):
         """Path to the disk containing this volume
         (e.g. '/dev/disks/mpx.vmhba0:C0:T0:L0').
         """
         return os.path.join(os.path.sep, 'dev', 'disks', self.diskName)

      @property
      def fsType(self):
         """Volume filesystem type.
         """
         # Known differences between VSI fs types and SystemStorage fs types.
         vsiTypeMap = {'VMFS': FS_TYPE_VMFS,
                       'VMFS-L': FS_TYPE_VMFS_L,
                       'VFFS' : FS_TYPE_VMFS_L,
                       'VMFSOS' : FS_TYPE_VMFSOS}

         # Convert if the type is different in VSI.
         fsType = self._vsiInfo['fsType']
         return vsiTypeMap.get(fsType, fsType)

      @property
      def isLocal(self):
         """True if this volume is on a local disk.
         """
         try:
            diskDev = vsi.get('/storage/scsifw/devices/%s/info' % self.diskName)
            return diskDev['isLocal']
         except Exception:
            return False

      @property
      def majorVersion(self):
         """Filesystem major version number.
         """
         return self._vsiInfo['majorVersion']

      @property
      def minorVersion(self):
         """Filesystem minor version number
         """
         return self._vsiInfo['minorVersion']

      @property
      def size(self):
         """Volume size in bytes, including filesystem metadata.
         """
         return os.stat(self.devPath).st_size

      @property
      def sizeInMB(self):
         """Volume size in MiB, including filesystem metadata.
         """
         return ceil(self.size / MiB)

      @property
      def statvfs(self):
         """Get and cache filesystem statistics.
         """
         if self._statvfs is None:
            self._statvfs = os.statvfs(self.path)
         return self._statvfs

      @property
      def fsSize(self):
         """Size of filesystem in bytes, excluding metadata.
         """
         return self.statvfs.f_blocks * self.statvfs.f_frsize

      @property
      def fsSizeInMB(self):
         """Size of filesystem in MiB, excluding metadata.
         """
         return ceil(self.fsSize / MiB)

      @property
      def fsFree(self):
         """Total amount of free space in bytes.
         """
         return self.statvfs.f_bfree * self.statvfs.f_frsize

      @property
      def fsPercentUsed(self):
         """Percentage of used space.
         """
         return ceil(100 - ((self.fsFree * 100) / self.fsSize))

      def getOpenFiles(self):
         """Return the set of all opened files on this volume.
         """
         VSI_FSS_OPEN_FILE_HANDLES = '/system/fsSwitch/fileHandles'

         filesInUse = set()

         for handle in vsi.list(VSI_FSS_OPEN_FILE_HANDLES):
            try:
               fsHandle = vsi.get("%s/%s" % (VSI_FSS_OPEN_FILE_HANDLES, handle))
            except Exception:
               # The handle could have been freed by the time we fetch it.
               continue

            if fsHandle['volumeName'] == self.name:
               filesInUse.add(fsHandle['fileName'])

         return filesInUse


   def getFssVolumes(diskName=None, fsTypes=None, uuid=None, localOnly=False):
      """List all mounted volumes matching the given filters.
      """
      VSI_FSVOL_PATH = "/system/fsSwitch/volumes"

      volumes = []
      if uuid is not None and type(uuid) is str:
         uuid = EsxFsUuid.fromString(uuid)
      for volUuid in vsi.list(VSI_FSVOL_PATH):
         volInfo = vsi.get("%s/%s" % (VSI_FSVOL_PATH, volUuid))
         volume = FssVolume(volInfo)

         # localOnly is not used in conjunction with name/uuid match
         if localOnly and not volume.isLocal:
            if diskName is None and uuid is None:
               continue

         if uuid is not None and volume.uuid != uuid:
            continue

         if diskName is not None and volume.diskName != diskName:
            continue

         if fsTypes is not None and volume.fsType not in fsTypes:
            continue

         volumes += [volume]

      return volumes

   def fsRescan():
      """Rescan all storage devices for new partitions and filesystems.
      """
      runCli(['storage', 'filesystem', 'rescan'])

   def earlyFsRescan():
      """Early boot rescan filesystems.
      """
      fsRescan()
      runCli(['storage', 'filesystem', 'automount'])
      runCli(['boot', 'storage',  'restore', '--explicit-mounts'])

   def getVolumeFullPath(volume):
      """Get full path of filesystem.
      """
      if volume.startswith(FSS_VOLUMES_ROOT):
         return volume
      return os.path.join(FSS_VOLUMES_ROOT, volume)

   def umountVFatVolume(volume, retries=3, waitTime=3):
      """Unmount the given VFAT filesystem volume.

      @retries: number of time to retry
      @waitTime: wait time (seconds) between retries.
      """
      from systemStorage import vfat

      volumeName = os.path.basename(volume.devPath)

      for i in range(1, retries + 1):
         try:
            vfat.umount(volumeName)
            break
         except Exception as e:
            log.warn('%s: failed to unmount VFAT volume on try %u: %s',
                     volume.devPath, i, str(e))
            try:
               openFiles = volume.getOpenFiles()
               if openFiles:
                  log.debug("%s: volume has opened file handles: %s",
                            volume.name, str(openFiles))
            except Exception:
               pass

            if i == retries:
               raise
            sleep(waitTime)

   def umountVfatFileSystems(diskName, retries=3, waitTime=3):
      """Unmount vFAT file systems on a disk.

      @retries: number of time to retry
      @waitTime: wait time (seconds) between retries.
      """
      for volume in getFssVolumes(diskName=diskName, fsTypes=[FS_TYPE_VFAT]):
         umountVFatVolume(volume, retries, waitTime)

   def umountVmfsFileSystems(diskName):
      """Unmount VMFS/VMFS-L file systems on a disk.
      """
      from systemStorage import vmfsl

      fsTypes = [FS_TYPE_VMFS, FS_TYPE_VMFS_L, FS_TYPE_VMFSOS]

      for volume in getFssVolumes(diskName=diskName, fsTypes=fsTypes):
         vmfsl.vmfsUnmount(volume.uuid)

   def umountFileSystems(diskName):
      """Unmount vFAT and VMFS/VMFS-L file systems on a disk.
      """
      umountVfatFileSystems(diskName)
      umountVmfsFileSystems(diskName)

   def waitFsMounted(volPath, timeout=10):
      """Wait until the filesystem referenced by volume path is accessible.

      @param timeout in seconds (default 10s).
      """
      mountPoint = os.path.realpath(volPath)

      for _ in range(timeout * 10):
         if os.path.exists(mountPoint):
            return
         sleep(.1)

      raise FileNotFoundError("%s: volume not mounted after %u-second timeout" %
                              (mountPoint, timeout))

   def waitFsUmounted(uuid, timeout=10):
      """Wait until the filesystem referenced by @uuid is umounted.
      """
      vsiNode = "/system/fsSwitch/volumes/%s" % str(uuid)

      for _ in range(timeout * 10):
         try:
            _ = vsi.get(vsiNode)
         except Exception:
            # Node doesn't exist because the filesystem has been unmounted.
            return
         sleep(.1)

      raise OSError("%s: failed to unmount volume after %u-second timeout" %
                    (str(uuid)), timeout)

   def getVolumeId(path):
      """Return the label (or UUID) part of a FSS volume path.

      e.g.: "/vmfs/volumes/datastore1/foo/bar" -> "datastore1"
            "/vmfs/volumes/5d97aaf7-5541916a" -> "5d97aaf7-5541916a"
      """
      path = os.path.normpath(path)

      if os.path.commonpath([FSS_VOLUMES_ROOT, path]) != FSS_VOLUMES_ROOT:
         raise ValueError("%s: invalid volume path "
                          "(not under FSS volumes root %s)" %
                          (path, FSS_VOLUMES_ROOT))

      try:
         return path.split(os.path.sep)[3]
      except IndexError:
         raise ValueError("%s: incomplete volume path "
                          "(missing volume label or UUID)" % path)

   def getSupportedOsdataPathDevice(path, checkFsFree=True):
      """Return the name of the storage device of the given path if it
         supports OSData. Otherwise, None is returned.

         If checkFsFree, then the free space on a VMFS datastore must be
         at least 31GB free space.
      """
      path = os.path.realpath(path)

      try:
         volId = getVolumeId(path)
         vols = getFssVolumes(uuid=volId)
         vol = vols[0]
      except Exception:
         return None

      fsTypes = {FS_TYPE_VMFS, FS_TYPE_VMFS_L, FS_TYPE_VMFSOS}

      if vol.fsType not in fsTypes:
         return None
      if vol.fsType == FS_TYPE_VMFS:
         # For VMFS datastore, path must be a subdirectory and
         # must be at least 31GB free on real hardware, and 10GB in a VM.
         if path.count(os.path.sep) < 4:
            return None
         minFree = (SYSTEM_DISK_SIZE_MEDIUM_MB if isEsxInAVm()
                                               else SYSTEM_DISK_SIZE_LARGE_MB)
         if checkFsFree and (vol.fsFree // MiB) < minFree:
            return None

      # Verify disk supports OSData.
      # Import is done here to avoid circular dependencies.
      from systemStorage.esxdisk import getDiskCollection

      disks = getDiskCollection()
      if vol.diskName in disks:
         disk = disks[vol.diskName]
         if disk.supportsOsdata:
            return vol.diskName

      return None


   def isNfsVolume(volumeId):
      """Return True if the given volume is configured to NFS.

      @volumeId can be a volume label or UUID.
      """
      import vmkctl
      storageInfo = vmkctl.StorageInfoImpl()
      try:
         for ptr in storageInfo.GetNetworkFileSystems():
            share = ptr.get()
            if volumeId == share.GetUuid() or volumeId == share.GetVolumeName():
               return True
      except Exception:
         return False
      return False

   def getOSDataOnDatastore(datastoreName):
      """Returns the OSData path if OSdata is created in a subdirectory in a
         datastore.

      @datastoreName is the name of the datastore.
      """
      uuid = getHardwareUuid()
      return os.path.join(FSS_VOLUMES_ROOT, datastoreName, '.osdata-' + uuid)

   def datastoreHasOSData(datastoreName):
      """Returns True if the datastore contains the OSdata directory.

      @datastoreName is the name of the datastore.
      """
      return os.path.isdir(getOSDataOnDatastore(datastoreName))
