Python的Shelve模块是否有内置保护措施,以确保两个进程不会同时写入同一个文件?
Python shelve模块问题
原标题:
最佳回答
问题回答
我已经将
from contextlib import contextmanager
from fcntl import flock, LOCK_SH, LOCK_EX, LOCK_UN
import shelve
@contextmanager
def locking(lock_path, lock_mode):
with open(lock_path, w ) as lock:
flock(lock.fileno(), lock_mode) # block until lock is acquired
try:
yield
finally:
flock(lock.fileno(), LOCK_UN) # release
class DBManager(object):
def __init__(self, db_path):
self.db_path = db_path
def read(self):
with locking("%s.lock" % self.db_path, LOCK_SH):
with shelve.open(self.db_path, "r", 2) as db:
return dict(db)
def cas(self, old_db, new_db):
with locking("%s.lock" % self.db_path, LOCK_EX):
with shelve.open(self.db_path, "c", 2) as db:
if old_db != dict(db):
return False
db.clear()
db.update(new_db)
return True
根据最佳答案,一个书架上不宜有多个作者。我建议写一个包装器,以确保打开和访问书架元素的安全性。包装器代码大致如下:
def open(self, mode=READONLY):
if mode is READWRITE:
lockfilemode = "a"
lockmode = LOCK_EX
shelve_mode = c
else:
lockfilemode = "r"
lockmode = LOCK_SH
shelve_mode = r
self.lockfd = open(shelvefile+".lck", lockfilemode)
fcntl.flock(self.lockfd.fileno(), lockmode | LOCK_NB)
self.shelve = shelve.open(shelvefile, flag=shelve_mode, protocol=pickle.HIGHEST_PROTOCOL))
def close(self):
self.shelve.close()
fcntl.flock(self.lockfd.fileno(), LOCK_UN)
lockfd.close()
在 Ivo s 和 Samus_ s 的方法基础上,我实现了一个更简单的 shelve.open 包装器:
import fcntl
import shelve
import contextlib
import typing
@contextlib.contextmanager
def open_safe_shelve(db_path: str, flag: typing.Literal["r", "w", "c", "n"] = "c", protocol=None, writeback=False):
if flag in ("w", "c", "n"):
lockfile_lock_mode = fcntl.LOCK_EX
elif flag == "r":
lockfile_lock_mode = fcntl.LOCK_SH
else:
raise ValueError(f"Invalid mode: {flag}, only r , w , c , n are allowed.")
with open(f"{db_path}.lock", "w") as lock: # According to https://docs.python.org/3/library/fcntl.html#fcntl.flock, the file must be opened in write mode on some systems.
fcntl.flock(lock.fileno(), lockfile_lock_mode) # Block until lock is acquired.
try:
yield shelve.open(db_path, flag=flag, protocol=protocol, writeback=writeback)
finally:
fcntl.flock(lock.fileno(), fcntl.LOCK_UN) # Release lock
这样可以避免像Samus_的cas()
方法中一样需要检查字典是否自上次以来发生了更改。
请注意,此操作会阻塞直到锁定成功。如果你想在锁定被占用时抛出异常,可以使用lockfile_lock_mode | fcntl.LOCK_NB
作为锁标志。
它可以像书架一样正常使用。例如:
import time
import multiprocessing
def read(db_path: str):
print("Reading wants lock")
with open_safe_shelve(db_path, "r") as db:
print("Reading has lock")
print(f"foo: {db.get( foo , None)}")
time.sleep(10)
print(f"foo: {db.get( foo , None)}")
print("Reading giving up lock")
def write(db_path: str):
print("Writing wants lock")
with open_safe_shelve(db_path) as db:
print("Writing has lock")
db["foo"] = "bar"
print("Writing giving up lock")
if __name__ == "__main__":
db_path = "test_database"
read_process = multiprocessing.Process(target=read, args=(db_path,))
write_process = multiprocessing.Process(target=write, args=(db_path,))
read_process.start()
time.sleep(1)
write_process.start()
read_process.join()
write_process.join()
将输出(假定test_database.db
已存在):
Reading wants lock
Reading has lock
foo: None
Writing wants lock
# (sleeps for around 9 seconds)
foo: None
Reading giving up lock
Writing has lock
Writing giving up lock
相关问题
热门标签
- winforms
- combobox
- fogbugz
- java
- date
- internationalization
- asp.net
- iis
- url-rewriting
- urlrewriter
- c#
- enums
- ocaml
- haxe
- algorithm
- string
- viewstate
- .net
- c++
- c
- symbol-table
- mysql
- database
- postgresql
- licensing
- migration
- vb.net
- vb6
- declaration
- vb6-migration
- python
- psycopg2
- backup
- vmware
- virtualization
- gnu-screen
- authentication
- desktop
- excel
- xll
- cultureinfo
- regioninfo
- oracle
- client
- session
- download
- html
- virtual
- constructor
- scenarios
- perl
- full-text-search
- javascript
- ajax
- testing
- oop
- inheritance
- vim
- encapsulation
- information-hiding