Package oauth2client :: Module locked_file
[hide private]
[frames] | no frames]

Source Code for Module oauth2client.locked_file

  1  # Copyright 2014 Google Inc. All rights reserved. 
  2  # 
  3  # Licensed under the Apache License, Version 2.0 (the "License"); 
  4  # you may not use this file except in compliance with the License. 
  5  # You may obtain a copy of the License at 
  6  # 
  7  #      http://www.apache.org/licenses/LICENSE-2.0 
  8  # 
  9  # Unless required by applicable law or agreed to in writing, software 
 10  # distributed under the License is distributed on an "AS IS" BASIS, 
 11  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 
 12  # See the License for the specific language governing permissions and 
 13  # limitations under the License. 
 14   
 15  """Locked file interface that should work on Unix and Windows pythons. 
 16   
 17  This module first tries to use fcntl locking to ensure serialized access 
 18  to a file, then falls back on a lock file if that is unavialable. 
 19   
 20  Usage: 
 21      f = LockedFile('filename', 'r+b', 'rb') 
 22      f.open_and_lock() 
 23      if f.is_locked(): 
 24        print('Acquired filename with r+b mode') 
 25        f.file_handle().write('locked data') 
 26      else: 
 27        print('Acquired filename with rb mode') 
 28      f.unlock_and_close() 
 29  """ 
 30   
 31  from __future__ import print_function 
 32   
 33  __author__ = 'cache@google.com (David T McWherter)' 
 34   
 35  import errno 
 36  import logging 
 37  import os 
 38  import time 
 39   
 40  from oauth2client import util 
 41   
 42  logger = logging.getLogger(__name__) 
43 44 45 -class CredentialsFileSymbolicLinkError(Exception):
46 """Credentials files must not be symbolic links."""
47
48 49 -class AlreadyLockedException(Exception):
50 """Trying to lock a file that has already been locked by the LockedFile.""" 51 pass
52
53 54 -def validate_file(filename):
55 if os.path.islink(filename): 56 raise CredentialsFileSymbolicLinkError( 57 'File: %s is a symbolic link.' % filename)
58
59 -class _Opener(object):
60 """Base class for different locking primitives.""" 61
62 - def __init__(self, filename, mode, fallback_mode):
63 """Create an Opener. 64 65 Args: 66 filename: string, The pathname of the file. 67 mode: string, The preferred mode to access the file with. 68 fallback_mode: string, The mode to use if locking fails. 69 """ 70 self._locked = False 71 self._filename = filename 72 self._mode = mode 73 self._fallback_mode = fallback_mode 74 self._fh = None 75 self._lock_fd = None
76
77 - def is_locked(self):
78 """Was the file locked.""" 79 return self._locked
80
81 - def file_handle(self):
82 """The file handle to the file. Valid only after opened.""" 83 return self._fh
84
85 - def filename(self):
86 """The filename that is being locked.""" 87 return self._filename
88
89 - def open_and_lock(self, timeout, delay):
90 """Open the file and lock it. 91 92 Args: 93 timeout: float, How long to try to lock for. 94 delay: float, How long to wait between retries. 95 """ 96 pass
97
98 - def unlock_and_close(self):
99 """Unlock and close the file.""" 100 pass
101
102 103 -class _PosixOpener(_Opener):
104 """Lock files using Posix advisory lock files.""" 105
106 - def open_and_lock(self, timeout, delay):
107 """Open the file and lock it. 108 109 Tries to create a .lock file next to the file we're trying to open. 110 111 Args: 112 timeout: float, How long to try to lock for. 113 delay: float, How long to wait between retries. 114 115 Raises: 116 AlreadyLockedException: if the lock is already acquired. 117 IOError: if the open fails. 118 CredentialsFileSymbolicLinkError if the file is a symbolic link. 119 """ 120 if self._locked: 121 raise AlreadyLockedException('File %s is already locked' % 122 self._filename) 123 self._locked = False 124 125 validate_file(self._filename) 126 try: 127 self._fh = open(self._filename, self._mode) 128 except IOError as e: 129 # If we can't access with _mode, try _fallback_mode and don't lock. 130 if e.errno == errno.EACCES: 131 self._fh = open(self._filename, self._fallback_mode) 132 return 133 134 lock_filename = self._posix_lockfile(self._filename) 135 start_time = time.time() 136 while True: 137 try: 138 self._lock_fd = os.open(lock_filename, 139 os.O_CREAT|os.O_EXCL|os.O_RDWR) 140 self._locked = True 141 break 142 143 except OSError as e: 144 if e.errno != errno.EEXIST: 145 raise 146 if (time.time() - start_time) >= timeout: 147 logger.warn('Could not acquire lock %s in %s seconds', 148 lock_filename, timeout) 149 # Close the file and open in fallback_mode. 150 if self._fh: 151 self._fh.close() 152 self._fh = open(self._filename, self._fallback_mode) 153 return 154 time.sleep(delay)
155
156 - def unlock_and_close(self):
157 """Unlock a file by removing the .lock file, and close the handle.""" 158 if self._locked: 159 lock_filename = self._posix_lockfile(self._filename) 160 os.close(self._lock_fd) 161 os.unlink(lock_filename) 162 self._locked = False 163 self._lock_fd = None 164 if self._fh: 165 self._fh.close()
166
167 - def _posix_lockfile(self, filename):
168 """The name of the lock file to use for posix locking.""" 169 return '%s.lock' % filename
170 171 172 try: 173 import fcntl
174 175 - class _FcntlOpener(_Opener):
176 """Open, lock, and unlock a file using fcntl.lockf.""" 177
178 - def open_and_lock(self, timeout, delay):
179 """Open the file and lock it. 180 181 Args: 182 timeout: float, How long to try to lock for. 183 delay: float, How long to wait between retries 184 185 Raises: 186 AlreadyLockedException: if the lock is already acquired. 187 IOError: if the open fails. 188 CredentialsFileSymbolicLinkError if the file is a symbolic link. 189 """ 190 if self._locked: 191 raise AlreadyLockedException('File %s is already locked' % 192 self._filename) 193 start_time = time.time() 194 195 validate_file(self._filename) 196 try: 197 self._fh = open(self._filename, self._mode) 198 except IOError as e: 199 # If we can't access with _mode, try _fallback_mode and don't lock. 200 if e.errno in (errno.EPERM, errno.EACCES): 201 self._fh = open(self._filename, self._fallback_mode) 202 return 203 204 # We opened in _mode, try to lock the file. 205 while True: 206 try: 207 fcntl.lockf(self._fh.fileno(), fcntl.LOCK_EX) 208 self._locked = True 209 return 210 except IOError as e: 211 # If not retrying, then just pass on the error. 212 if timeout == 0: 213 raise e 214 if e.errno != errno.EACCES: 215 raise e 216 # We could not acquire the lock. Try again. 217 if (time.time() - start_time) >= timeout: 218 logger.warn('Could not lock %s in %s seconds', 219 self._filename, timeout) 220 if self._fh: 221 self._fh.close() 222 self._fh = open(self._filename, self._fallback_mode) 223 return 224 time.sleep(delay)
225
226 - def unlock_and_close(self):
227 """Close and unlock the file using the fcntl.lockf primitive.""" 228 if self._locked: 229 fcntl.lockf(self._fh.fileno(), fcntl.LOCK_UN) 230 self._locked = False 231 if self._fh: 232 self._fh.close()
233 except ImportError: 234 _FcntlOpener = None 235 236 237 try: 238 import pywintypes 239 import win32con 240 import win32file
241 242 - class _Win32Opener(_Opener):
243 """Open, lock, and unlock a file using windows primitives.""" 244 245 # Error #33: 246 # 'The process cannot access the file because another process' 247 FILE_IN_USE_ERROR = 33 248 249 # Error #158: 250 # 'The segment is already unlocked.' 251 FILE_ALREADY_UNLOCKED_ERROR = 158 252
253 - def open_and_lock(self, timeout, delay):
254 """Open the file and lock it. 255 256 Args: 257 timeout: float, How long to try to lock for. 258 delay: float, How long to wait between retries 259 260 Raises: 261 AlreadyLockedException: if the lock is already acquired. 262 IOError: if the open fails. 263 CredentialsFileSymbolicLinkError if the file is a symbolic link. 264 """ 265 if self._locked: 266 raise AlreadyLockedException('File %s is already locked' % 267 self._filename) 268 start_time = time.time() 269 270 validate_file(self._filename) 271 try: 272 self._fh = open(self._filename, self._mode) 273 except IOError as e: 274 # If we can't access with _mode, try _fallback_mode and don't lock. 275 if e.errno == errno.EACCES: 276 self._fh = open(self._filename, self._fallback_mode) 277 return 278 279 # We opened in _mode, try to lock the file. 280 while True: 281 try: 282 hfile = win32file._get_osfhandle(self._fh.fileno()) 283 win32file.LockFileEx( 284 hfile, 285 (win32con.LOCKFILE_FAIL_IMMEDIATELY| 286 win32con.LOCKFILE_EXCLUSIVE_LOCK), 0, -0x10000, 287 pywintypes.OVERLAPPED()) 288 self._locked = True 289 return 290 except pywintypes.error as e: 291 if timeout == 0: 292 raise e 293 294 # If the error is not that the file is already in use, raise. 295 if e[0] != _Win32Opener.FILE_IN_USE_ERROR: 296 raise 297 298 # We could not acquire the lock. Try again. 299 if (time.time() - start_time) >= timeout: 300 logger.warn('Could not lock %s in %s seconds' % ( 301 self._filename, timeout)) 302 if self._fh: 303 self._fh.close() 304 self._fh = open(self._filename, self._fallback_mode) 305 return 306 time.sleep(delay)
307
308 - def unlock_and_close(self):
309 """Close and unlock the file using the win32 primitive.""" 310 if self._locked: 311 try: 312 hfile = win32file._get_osfhandle(self._fh.fileno()) 313 win32file.UnlockFileEx(hfile, 0, -0x10000, pywintypes.OVERLAPPED()) 314 except pywintypes.error as e: 315 if e[0] != _Win32Opener.FILE_ALREADY_UNLOCKED_ERROR: 316 raise 317 self._locked = False 318 if self._fh: 319 self._fh.close()
320 except ImportError: 321 _Win32Opener = None
322 323 324 -class LockedFile(object):
325 """Represent a file that has exclusive access.""" 326 327 @util.positional(4)
328 - def __init__(self, filename, mode, fallback_mode, use_native_locking=True):
329 """Construct a LockedFile. 330 331 Args: 332 filename: string, The path of the file to open. 333 mode: string, The mode to try to open the file with. 334 fallback_mode: string, The mode to use if locking fails. 335 use_native_locking: bool, Whether or not fcntl/win32 locking is used. 336 """ 337 opener = None 338 if not opener and use_native_locking: 339 if _Win32Opener: 340 opener = _Win32Opener(filename, mode, fallback_mode) 341 if _FcntlOpener: 342 opener = _FcntlOpener(filename, mode, fallback_mode) 343 344 if not opener: 345 opener = _PosixOpener(filename, mode, fallback_mode) 346 347 self._opener = opener
348
349 - def filename(self):
350 """Return the filename we were constructed with.""" 351 return self._opener._filename
352
353 - def file_handle(self):
354 """Return the file_handle to the opened file.""" 355 return self._opener.file_handle()
356
357 - def is_locked(self):
358 """Return whether we successfully locked the file.""" 359 return self._opener.is_locked()
360
361 - def open_and_lock(self, timeout=0, delay=0.05):
362 """Open the file, trying to lock it. 363 364 Args: 365 timeout: float, The number of seconds to try to acquire the lock. 366 delay: float, The number of seconds to wait between retry attempts. 367 368 Raises: 369 AlreadyLockedException: if the lock is already acquired. 370 IOError: if the open fails. 371 """ 372 self._opener.open_and_lock(timeout, delay)
373
374 - def unlock_and_close(self):
375 """Unlock and close a file.""" 376 self._opener.unlock_and_close()
377