English 中文(简体)
Python shelve模块问题
原标题:
  • 时间:2009-01-28 04:32:14
  •  标签:

Python的Shelve模块是否有内置保护措施,以确保两个进程不会同时写入同一个文件?

最佳回答

shelve模块使用底层数据库包(例如dbm、gdbm或bsddb)。

限制段落说(我的强调):

shelve 模块不支持同时读写访问已存储对象。(多个同时读取访问是安全的。)当程序打开架子板进行写入时,其他程序不应该打开它进行读取或写入。Unix 文件锁定可用于解决此问题,但在 Unix 版本间存在差异,需要了解所使用的数据库实现。

结论:这取决于操作系统和底层数据库。为了保持代码的可移植性,请不要建立在并发性之上。

问题回答

我已经将实现为上下文管理器,供有兴趣的人参考:

from contextlib import contextmanager
from fcntl import flock, LOCK_SH, LOCK_EX, LOCK_UN
import shelve

@contextmanager
def locking(lock_path, lock_mode):
    with open(lock_path,  w ) as lock:
        flock(lock.fileno(), lock_mode) # block until lock is acquired
        try:
            yield
        finally:
            flock(lock.fileno(), LOCK_UN) # release

class DBManager(object):
    def __init__(self, db_path):
        self.db_path = db_path

    def read(self):
        with locking("%s.lock" % self.db_path, LOCK_SH):
            with shelve.open(self.db_path, "r", 2) as db:
                return dict(db)

    def cas(self, old_db, new_db):
        with locking("%s.lock" % self.db_path, LOCK_EX):
            with shelve.open(self.db_path, "c", 2) as db:
                if old_db != dict(db):
                    return False
                db.clear()
                db.update(new_db)
                return True

根据最佳答案,一个书架上不宜有多个作者。我建议写一个包装器,以确保打开和访问书架元素的安全性。包装器代码大致如下:

def open(self, mode=READONLY):
    if mode is READWRITE:
        lockfilemode = "a" 
        lockmode = LOCK_EX
        shelve_mode =  c 
    else:
        lockfilemode = "r"
        lockmode = LOCK_SH
        shelve_mode =  r 
    self.lockfd = open(shelvefile+".lck", lockfilemode)
    fcntl.flock(self.lockfd.fileno(), lockmode | LOCK_NB)
    self.shelve = shelve.open(shelvefile, flag=shelve_mode, protocol=pickle.HIGHEST_PROTOCOL))
def close(self):
    self.shelve.close()
    fcntl.flock(self.lockfd.fileno(), LOCK_UN)
    lockfd.close()

在 Ivo s 和 Samus_ s 的方法基础上,我实现了一个更简单的 shelve.open 包装器:

import fcntl
import shelve
import contextlib
import typing


@contextlib.contextmanager
def open_safe_shelve(db_path: str, flag: typing.Literal["r", "w", "c", "n"] = "c", protocol=None, writeback=False):
    if flag in ("w", "c", "n"):
        lockfile_lock_mode = fcntl.LOCK_EX
    elif flag == "r":
        lockfile_lock_mode = fcntl.LOCK_SH
    else:
        raise ValueError(f"Invalid mode: {flag}, only  r ,  w ,  c ,  n  are allowed.")

    with open(f"{db_path}.lock", "w") as lock:  # According to https://docs.python.org/3/library/fcntl.html#fcntl.flock, the file must be opened in write mode on some systems.
        fcntl.flock(lock.fileno(), lockfile_lock_mode)  # Block until lock is acquired.
        try:
            yield shelve.open(db_path, flag=flag, protocol=protocol, writeback=writeback)
        finally:
            fcntl.flock(lock.fileno(), fcntl.LOCK_UN)  # Release lock

这样可以避免像Samus_的cas()方法中一样需要检查字典是否自上次以来发生了更改。

请注意,此操作会阻塞直到锁定成功。如果你想在锁定被占用时抛出异常,可以使用lockfile_lock_mode | fcntl.LOCK_NB作为锁标志。

它可以像书架一样正常使用。例如:

import time
import multiprocessing

def read(db_path: str):
    print("Reading wants lock")
    with open_safe_shelve(db_path, "r") as db:
        print("Reading has lock")
        print(f"foo: {db.get( foo , None)}")
        time.sleep(10)
        print(f"foo: {db.get( foo , None)}")
        print("Reading giving up lock")


def write(db_path: str):
    print("Writing wants lock")
    with open_safe_shelve(db_path) as db:
        print("Writing has lock")
        db["foo"] = "bar"
        print("Writing giving up lock")


if __name__ == "__main__":
    db_path = "test_database"
    read_process = multiprocessing.Process(target=read, args=(db_path,))
    write_process = multiprocessing.Process(target=write, args=(db_path,))
    read_process.start()
    time.sleep(1)
    write_process.start()
    read_process.join()
    write_process.join()

将输出(假定test_database.db 已存在):

Reading wants lock
Reading has lock
foo: None
Writing wants lock
# (sleeps for around 9 seconds)
foo: None
Reading giving up lock
Writing has lock
Writing giving up lock




相关问题
热门标签