Skip to content

prefect.records.memory

MemoryRecordStore

Bases: RecordStore

A record store that stores data in memory.

Source code in src/prefect/records/memory.py
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
class MemoryRecordStore(RecordStore):
    """
    A record store that stores data in memory.
    """

    _instance = None

    def __new__(cls, *args, **kwargs):
        if cls._instance is None:
            cls._instance = super().__new__(cls)
        return cls._instance

    def __init__(self):
        self._locks_dict_lock = threading.Lock()
        self._locks: Dict[str, _LockInfo] = {}
        self._records: Dict[str, TransactionRecord] = {}

    def read(
        self, key: str, holder: Optional[str] = None
    ) -> Optional[TransactionRecord]:
        holder = holder or self.generate_default_holder()

        if self.is_locked(key) and not self.is_lock_holder(key, holder):
            self.wait_for_lock(key)
        return self._records.get(key)

    def write(self, key: str, result: BaseResult, holder: Optional[str] = None) -> None:
        holder = holder or self.generate_default_holder()

        with self._locks_dict_lock:
            if self.is_locked(key) and not self.is_lock_holder(key, holder):
                raise ValueError(
                    f"Cannot write to transaction with key {key} because it is locked by another holder."
                )
        self._records[key] = TransactionRecord(key=key, result=result)

    def exists(self, key: str) -> bool:
        return key in self._records

    def supports_isolation_level(self, isolation_level: IsolationLevel) -> bool:
        return isolation_level in {
            IsolationLevel.READ_COMMITTED,
            IsolationLevel.SERIALIZABLE,
        }

    def _expire_lock(self, key: str):
        """
        Expire the lock for the given key.

        Used as a callback for the expiration timer of a lock.

        Args:
            key: The key of the lock to expire.
        """
        with self._locks_dict_lock:
            if key in self._locks:
                lock_info = self._locks[key]
                if lock_info["lock"].locked():
                    lock_info["lock"].release()
                if lock_info["expiration_timer"]:
                    lock_info["expiration_timer"].cancel()
                del self._locks[key]

    def acquire_lock(
        self,
        key: str,
        holder: Optional[str] = None,
        acquire_timeout: Optional[float] = None,
        hold_timeout: Optional[float] = None,
    ) -> bool:
        holder = holder or self.generate_default_holder()
        with self._locks_dict_lock:
            if key not in self._locks:
                lock = threading.Lock()
                lock.acquire()
                expiration_timer = None
                if hold_timeout is not None:
                    expiration_timer = threading.Timer(
                        hold_timeout, self._expire_lock, args=(key,)
                    )
                    expiration_timer.start()
                self._locks[key] = _LockInfo(
                    holder=holder, lock=lock, expiration_timer=expiration_timer
                )
                return True
            elif self._locks[key]["holder"] == holder:
                return True
            else:
                existing_lock_info = self._locks[key]

        if acquire_timeout is not None:
            existing_lock_acquired = existing_lock_info["lock"].acquire(
                timeout=acquire_timeout
            )
        else:
            existing_lock_acquired = existing_lock_info["lock"].acquire()

        if existing_lock_acquired:
            with self._locks_dict_lock:
                if (
                    expiration_timer := existing_lock_info["expiration_timer"]
                ) is not None:
                    expiration_timer.cancel()
                expiration_timer = None
                if hold_timeout is not None:
                    expiration_timer = threading.Timer(
                        hold_timeout, self._expire_lock, args=(key,)
                    )
                    expiration_timer.start()
                self._locks[key] = _LockInfo(
                    holder=holder,
                    lock=existing_lock_info["lock"],
                    expiration_timer=expiration_timer,
                )
            return True
        return False

    def release_lock(self, key: str, holder: Optional[str] = None) -> None:
        holder = holder or self.generate_default_holder()
        with self._locks_dict_lock:
            if key in self._locks and self._locks[key]["holder"] == holder:
                if (
                    expiration_timer := self._locks[key]["expiration_timer"]
                ) is not None:
                    expiration_timer.cancel()
                self._locks[key]["lock"].release()
                del self._locks[key]
            else:
                raise ValueError(
                    f"No lock held by {holder} for transaction with key {key}"
                )

    def is_locked(self, key: str) -> bool:
        return key in self._locks and self._locks[key]["lock"].locked()

    def is_lock_holder(self, key: str, holder: Optional[str] = None) -> bool:
        holder = holder or self.generate_default_holder()
        lock_info = self._locks.get(key)
        return (
            lock_info is not None
            and lock_info["lock"].locked()
            and lock_info["holder"] == holder
        )

    def wait_for_lock(self, key: str, timeout: Optional[float] = None) -> bool:
        if lock := self._locks.get(key, {}).get("lock"):
            if timeout is not None:
                lock_acquired = lock.acquire(timeout=timeout)
            else:
                lock_acquired = lock.acquire()
            if lock_acquired:
                lock.release()
            return lock_acquired
        return True