File Manager - Server Admin

Editing: posix_util.py

# -*- coding: utf-8 -*-
# Copyright 2016 Google Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Helper file for POSIX methods."""

from __future__ import absolute_import
from __future__ import print_function
from __future__ import division
from __future__ import unicode_literals

from calendar import timegm
import getpass
import logging
import os
import re
import time

import six

from gslib.exception import CommandException
from gslib.tz_utc import UTC
from gslib.utils.metadata_util import CreateCustomMetadata
from gslib.utils.metadata_util import GetValueFromObjectCustomMetadata
from gslib.utils.system_util import IS_WINDOWS
from gslib.utils.unit_util import SECONDS_PER_DAY

# pylint: disable=g-import-not-at-top
if not IS_WINDOWS:
  import grp
  import pwd

if six.PY3:
  long = int

# Metadata attribute names for POSIX attributes.
ATIME_ATTR = 'goog-reserved-file-atime'
GID_ATTR = 'goog-reserved-posix-gid'
MODE_ATTR = 'goog-reserved-posix-mode'
MTIME_ATTR = 'goog-reserved-file-mtime'
UID_ATTR = 'goog-reserved-posix-uid'
# NA_TIME is a long value that takes the place of any invalid mtime.
NA_TIME = -1
# NA_ID is a value that takes the place of any invalid POSIX UID or GID.
NA_ID = -1
# NA_MODE is an octal value that takes the place of any invalid POSIX file mode.
NA_MODE = -0o1

MODE_REGEX = re.compile('^[0-7]{3}$')

# POSIX Permissions Bit-masks
# User.
U_R = 0o400
U_W = 0o200
U_X = 0o100

# Group.
G_R = 0o040
G_W = 0o020
G_X = 0o010

# Other.
O_R = 0o004
O_W = 0o002
O_X = 0o001

# The permissions newly created files get by default.
SYSTEM_POSIX_MODE = None
# A list of group IDs that the current user is a member of.
USER_GROUPS = set()

class POSIXAttributes(object):
  """Class to hold POSIX attributes for a file/object."""

def __init__(self,
               atime=NA_TIME,
               mtime=NA_TIME,
               uid=NA_ID,
               gid=NA_ID,
               mode=None):
    """Constructor for POSIXAttributes class which holds relevant data.

Args:
      atime: The access time of the file/object.
      mtime: The modification time of the file/object.
      uid: The user ID that owns the file.
      gid: The group ID that the user is in.
      mode: An instance of POSIXMode.
    """
    self.atime = atime
    self.mtime = mtime
    self.uid = uid
    self.gid = gid
    self.mode = POSIXMode(mode if mode else NA_MODE)

class POSIXMode(object):

def __init__(self, permissions):
    self.permissions = permissions

def ConvertModeToBase8(mode):
  """Converts a base-10 mode integer from os.stat to base-8."""
  # Strip out unnecessary bits in the mode. Mode is given as a base-10
  # integer. It must be converted to base-8.
  return int(oct(mode)[-3:])

def DeserializeFileAttributesFromObjectMetadata(obj_metadata, url_str):
  """Parses the POSIX attributes from the supplied metadata.

Args:
    obj_metadata: The metadata for an object.
    url_str: File/object path that provides context if a warning is thrown.

Returns:
    A POSIXAttribute object with the retrieved values or a default value for
    any attribute that could not be found.
  """
  posix_attrs = POSIXAttributes()
  # Parse atime.
  found, atime = GetValueFromObjectCustomMetadata(obj_metadata, ATIME_ATTR,
                                                  NA_TIME)
  try:
    atime = long(atime)
    if found and atime <= NA_TIME:
      WarnNegativeAttribute('atime', url_str)
      atime = NA_TIME
    elif atime > long(time.time()) + SECONDS_PER_DAY:
      WarnFutureTimestamp('atime', url_str)
      atime = NA_TIME
  except ValueError:
    WarnInvalidValue('atime', url_str)
    atime = NA_TIME
  posix_attrs.atime = atime
  # Parse gid.
  DeserializeIDAttribute(obj_metadata, GID_ATTR, url_str, posix_attrs)
  # Parse uid.
  DeserializeIDAttribute(obj_metadata, UID_ATTR, url_str, posix_attrs)
  found, mode = GetValueFromObjectCustomMetadata(obj_metadata, MODE_ATTR,
                                                 NA_MODE)
  if found and MODE_REGEX.match(mode):
    try:
      # Parse mode into a 3-digit base-8 number.
      posix_attrs.mode = POSIXMode(int(mode))
    except ValueError:
      WarnInvalidValue('mode', url_str)
  return posix_attrs

def SerializeFileAttributesToObjectMetadata(posix_attrs,
                                            custom_metadata,
                                            preserve_posix=False):
  """Takes a POSIXAttributes object and serializes it into custom metadata.

Args:
    posix_attrs: A POSIXAttributes object.
    custom_metadata: A custom metadata object to serialize values into.
    preserve_posix: Whether or not to preserve POSIX attributes other than
                    mtime.
  """
  # mtime will always be needed in the object metadata for rsync.
  if posix_attrs.mtime != NA_TIME:
    CreateCustomMetadata(entries={MTIME_ATTR: posix_attrs.mtime},
                         custom_metadata=custom_metadata)
  # Only add other POSIX attributes if the preserve_posix flag is set.
  if preserve_posix:
    if posix_attrs.atime != NA_TIME:
      CreateCustomMetadata(entries={ATIME_ATTR: posix_attrs.atime},
                           custom_metadata=custom_metadata)
    if posix_attrs.uid != NA_ID:
      CreateCustomMetadata(entries={UID_ATTR: posix_attrs.uid},
                           custom_metadata=custom_metadata)
    if posix_attrs.gid != NA_ID:
      CreateCustomMetadata(entries={GID_ATTR: posix_attrs.gid},
                           custom_metadata=custom_metadata)
    if posix_attrs.mode.permissions != NA_MODE:
      CreateCustomMetadata(entries={MODE_ATTR: posix_attrs.mode.permissions},
                           custom_metadata=custom_metadata)

def DeserializeIDAttribute(obj_metadata, attr, url_str, posix_attrs):
  """Parses the POSIX attributes from the supplied metadata into posix_attrs.

Args:
    obj_metadata: The metadata for an object.
    attr: Either GID_ATTR or UID_ATTR.
    url_str: File/object path that provides context if a warning is thrown.
    posix_attrs: POSIXAttribute object.
  """
  attr_name = attr.split('-')[-1]
  found, val = GetValueFromObjectCustomMetadata(obj_metadata, attr, NA_ID)
  try:
    val = int(val)
    if found and val <= NA_ID:
      WarnNegativeAttribute(attr_name, url_str)
      val = NA_ID
  except ValueError:
    WarnInvalidValue(attr_name, url_str)
    val = NA_ID
  setattr(posix_attrs, attr_name, val)

def NeedsPOSIXAttributeUpdate(src_atime, dst_atime, src_mtime, dst_mtime,
                              src_uid, dst_uid, src_gid, dst_gid, src_mode,
                              dst_mode):
  """Checks whether an update for any POSIX attribute is needed.

Args:
    src_atime: The source access time.
    dst_atime: The destination access time.
    src_mtime: The source modification time.
    dst_mtime: The destination modification time.
    src_uid: The source user ID.
    dst_uid: The destination user ID.
    src_gid: The source group ID.
    dst_gid: The destination group ID.
    src_mode: The source mode.
    dst_mode: The destination mode.

Returns:
    A tuple containing a POSIXAttribute object and a boolean for whether an
    update was needed.
  """
  posix_attrs = POSIXAttributes()
  has_src_atime = src_atime > NA_TIME
  has_dst_atime = dst_atime > NA_TIME
  has_src_mtime = src_mtime > NA_TIME
  has_dst_mtime = dst_mtime > NA_TIME
  has_src_uid = src_uid > NA_ID
  has_dst_uid = dst_uid > NA_ID
  has_src_gid = src_gid > NA_ID
  has_dst_gid = dst_gid > NA_ID
  has_src_mode = src_mode > NA_MODE
  has_dst_mode = dst_mode > NA_MODE
  if has_src_atime and not has_dst_atime:
    posix_attrs.atime = src_atime
  if has_src_mtime and not has_dst_mtime:
    posix_attrs.mtime = src_mtime
  if has_src_uid and not has_dst_uid:
    posix_attrs.uid = src_uid
  if has_src_gid and not has_dst_gid:
    posix_attrs.gid = src_gid
  if has_src_mode and not has_dst_mode:
    posix_attrs.mode.permissions = src_mode
  return posix_attrs, ((has_src_atime and not has_dst_atime) or
                       (has_src_mtime and not has_dst_mtime) or
                       (has_src_uid and not has_dst_uid) or
                       (has_src_gid and not has_dst_gid) or
                       (has_src_mode and not has_dst_mode))

def ValidatePOSIXMode(mode):
  """Validates whether the mode is valid.

In order for the mode to be valid either the user, group, or other byte must
  be >= 4.

Args:
    mode: The mode as a 3-digit, base-8 integer.

Returns:
    True/False
  """
  return MODE_REGEX.match(oct(mode)[-3:]) and (mode & U_R or mode & G_R or
                                               mode & O_R)

def ValidateFilePermissionAccess(url_str, uid=NA_ID, gid=NA_ID, mode=NA_MODE):
  """Validates that the user has file access if uid, gid, and mode are applied.

Args:
    url_str: The path to the object for which this is validating.
    uid: A POSIX user ID.
    gid: A POSIX group ID.
    mode: A 3-digit, number representing POSIX permissions, must be in base-8.

Returns:
    A (bool, str) tuple, True if and only if it's safe to copy the file, and a
    string containing details for the error.
  """
  # Windows doesn't use the POSIX system for file permissions, so all files will
  # validate.
  if IS_WINDOWS:
    return True, ''

uid_present = uid > NA_ID
  gid_present = int(gid) > NA_ID
  mode_present = mode > NA_MODE
  # No need to perform validation if Posix attrs are not being preserved.
  if not (uid_present or gid_present or mode_present):
    return True, ''

# The root user on non-Windows systems can access files regardless of their
  # permissions.
  if os.geteuid() == 0:
    return True, ''

mode_valid = ValidatePOSIXMode(int(str(mode), 8))
  if mode_present:
    if not mode_valid:
      return False, 'Mode for %s won\'t allow read access.' % url_str
  else:
    # Calculate the default mode if the mode doesn't exist.
    # Convert mode to a 3-digit, base-8 integer.
    mode = int(SYSTEM_POSIX_MODE)

if uid_present:
    try:
      pwd.getpwuid(uid)
    except (KeyError, OverflowError):
      return (False, 'UID for %s doesn\'t exist on current system. uid: %d' %
              (url_str, uid))
  if gid_present:
    try:
      grp.getgrgid(gid)
    except (KeyError, OverflowError):
      return (False, 'GID for %s doesn\'t exist on current system. gid: %d' %
              (url_str, gid))

# uid at this point must exist, but isn't necessarily the current user.
  # Likewise, gid must also exist at this point.
  uid_is_current_user = uid == os.getuid()

# By this point uid and gid must exist on the system. However, the uid might
  # not match the current user's or the current user might not be a member of
  # the group identified by gid. In this case, the 'other' byte of the
  # permissions could provide sufficient access.
  mode = int(str(mode), 8)
  # Check that if the uid is not present and the gid and mode are, so that we
  # won't orphan the file. For example if the mode is set to 007, we can orphan
  # the file because the uid would default to the current user's ID and if the
  # current user wouldn't have read access or better, the file will be orphaned
  # even though they might otherwise have access through the gid or other bytes.
  if not uid_present and gid_present and mode_present and not bool(mode & U_R):
    return (False, 'Insufficient access with uid/gid/mode for %s, gid: %d, '
            'mode: %s' % (url_str, gid, oct(mode)[-3:]))
  if uid_is_current_user:
    valid = bool(mode & U_R)
    return (valid, '' if valid else
            'Insufficient access with uid/gid/mode for %s, uid: %d, '
            'mode: %s' % (url_str, uid, oct(mode)[-3:]))
  elif int(gid) in USER_GROUPS:
    valid = bool(mode & G_R)
    return (valid, '' if valid else
            'Insufficient access with uid/gid/mode for %s, gid: %d, '
            'mode: %s' % (url_str, gid, oct(mode)[-3:]))
  elif mode & O_R:
    return True, ''
  elif not uid_present and not gid_present and mode_valid:
    return True, ''
  return False, 'There was a problem validating %s.' % url_str

def ParseAndSetPOSIXAttributes(path,
                               obj_metadata,
                               is_rsync=False,
                               preserve_posix=False):
  """Parses POSIX attributes from obj_metadata and sets them.

Attributes will only be set if they exist in custom metadata. This function
  should only be called after ValidateFilePermissionAccess has been called for
  the specific file/object so as not to orphan files.

Args:
    path: The local filesystem path for the file. Valid metadata attributes will
          be set for the file located at path, some attributes will only be set
          if preserve_posix is set to True.
    obj_metadata: The metadata for the file/object.
    is_rsync: Whether or not the caller is the rsync command. Used to determine
              if timeCreated should be used.
    preserve_posix: Whether or not all POSIX attributes should be set.
  """
  if obj_metadata is None:
    # This exception is meant for debugging purposes as it should never be
    # thrown unless there are unexpected code changes.
    raise CommandException('obj_metadata cannot be None for %s' % path)
  try:
    found_at, atime = GetValueFromObjectCustomMetadata(obj_metadata,
                                                       ATIME_ATTR,
                                                       default_value=NA_TIME)
    found_mt, mtime = GetValueFromObjectCustomMetadata(obj_metadata,
                                                       MTIME_ATTR,
                                                       default_value=NA_TIME)
    found_uid, uid = GetValueFromObjectCustomMetadata(obj_metadata,
                                                      UID_ATTR,
                                                      default_value=NA_ID)
    found_gid, gid = GetValueFromObjectCustomMetadata(obj_metadata,
                                                      GID_ATTR,
                                                      default_value=NA_ID)
    found_mode, mode = GetValueFromObjectCustomMetadata(obj_metadata,
                                                        MODE_ATTR,
                                                        default_value=NA_MODE)
    if found_mt:
      mtime = long(mtime)
      if not preserve_posix:
        atime_tmp = os.stat(path).st_atime
        os.utime(path, (atime_tmp, mtime))
        return
    elif is_rsync:
      mtime = ConvertDatetimeToPOSIX(obj_metadata.timeCreated)
      os.utime(path, (mtime, mtime))

if not preserve_posix:
      return

if found_at:
      atime = long(atime)
    if atime > NA_TIME and mtime > NA_TIME:
      # Set both atime and mtime.
      os.utime(path, (atime, mtime))
    elif atime > NA_TIME and mtime <= NA_TIME:
      # atime is valid but mtime isn't.
      mtime_tmp = os.stat(path).st_mtime
      os.utime(path, (atime, mtime_tmp))
    elif atime <= NA_TIME and mtime > NA_TIME:
      # mtime is valid but atime isn't
      atime_tmp = os.stat(path).st_atime
      os.utime(path, (atime_tmp, mtime))
    if IS_WINDOWS:
      # Windows doesn't use POSIX uid/gid/mode unlike other systems. So there's
      # no point continuing.
      return
    # Only root can change ownership.
    if found_uid and os.geteuid() == 0:
      uid = int(uid)
    else:
      uid = NA_ID
    if found_gid:
      gid = int(gid)
    if uid > NA_ID and gid > NA_ID:
      # Set both uid and gid.
      os.chown(path, uid, gid)
    elif uid > NA_ID and gid <= NA_ID:
      # uid is valid but gid isn't.
      os.chown(path, uid, -1)
    elif uid <= NA_ID and gid > NA_ID:
      # gid is valid but uid isn't.
      os.chown(path, -1, gid)

if found_mode:
      mode = int(str(mode), 8)
      os.chmod(path, mode)

except ValueError:
    raise CommandException('Check POSIX attribute values for %s' %
                           obj_metadata.name)

def WarnNegativeAttribute(attr_name, url_str):
  """Logs if an attribute has a negative value.

Args:
    attr_name: The name of the attribute to log.
    url_str: The path of the file for context.
  """
  logging.getLogger().warn('%s has a negative %s in its metadata', url_str,
                           attr_name)

def WarnInvalidValue(attr_name, url_str):
  """Logs if an attribute has an invalid value.

Args:
    attr_name: The name of the attribute to log.
    url_str: The path of the file for context.
  """
  logging.getLogger().warn('%s has an invalid %s in its metadata', url_str,
                           attr_name)

def WarnFutureTimestamp(attr_name, url_str):
  """Logs if an attribute has an invalid value.

Args:
    attr_name: The name of the attribute to log.
    url_str: The path of the file for context.
  """
  logging.getLogger().warn(
      '%s has an %s more than 1 day from current system'
      ' time', url_str, attr_name)

def ConvertDatetimeToPOSIX(dt):
  """Converts a datetime object to UTC and formats as POSIX.

Sanitize the timestamp returned in dt, and put it in UTC format. For more
  information see the UTC class.

Args:
    dt: A Python datetime object.

Returns:
    A POSIX timestamp according to UTC.
  """
  return long(timegm(dt.replace(tzinfo=UTC()).timetuple()))

def InitializeDefaultMode():
  """Records the default POSIX mode using os.umask."""
  global SYSTEM_POSIX_MODE
  if IS_WINDOWS:
    # os.umask returns 0 on Windows. Below math works out to 666.
    SYSTEM_POSIX_MODE = '666'
    return
  # umask returns the permissions that should not be granted, so they must be
  # subtracted from the maximum set of permissions.
  max_permissions = 0o777
  current_umask = os.umask(0o177)
  os.umask(current_umask)
  mode = max_permissions - current_umask
  # Files are not given execute privileges by default. Therefore we need to
  # subtract one from every odd permissions value. This is done via a bitmask.
  SYSTEM_POSIX_MODE = oct(mode & 0o666)[-3:]

def InitializeUserGroups():
  """Initializes the set of groups that the user is in.

Should only be called if the flag for preserving POSIX attributes is set.
  """
  global USER_GROUPS
  if IS_WINDOWS:
    return
  user_id = os.getuid()
  user_name = pwd.getpwuid(user_id).pw_name
  USER_GROUPS = set(
      # Primary group
      [pwd.getpwuid(user_id).pw_gid] +
      # Secondary groups
      [g.gr_gid for g in grp.getgrall() if user_name in g.gr_mem])

def InitializePreservePosixData():
  """Initializes POSIX data. Run once at the beginning of a copy."""
  InitializeDefaultMode()
  InitializeUserGroups()

Back to File Manager