mirror of
https://github.com/markqvist/Reticulum.git
synced 2026-11-18 05:59:06 +00:00
851 lines
37 KiB
Python
851 lines
37 KiB
Python
# Reticulum License
|
|
#
|
|
# Copyright (c) 2016-2026 Mark Qvist
|
|
#
|
|
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
# of this software and associated documentation files (the "Software"), to deal
|
|
# in the Software without restriction, including without limitation the rights
|
|
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
# copies of the Software, and to permit persons to whom the Software is
|
|
# furnished to do so, subject to the following conditions:
|
|
#
|
|
# - The Software shall not be used in any kind of system which includes amongst
|
|
# its functions the ability to purposefully do harm to human beings.
|
|
#
|
|
# - The Software shall not be used, directly or indirectly, in the creation of
|
|
# an artificial intelligence, machine learning or language model training
|
|
# dataset, including but not limited to any use that contributes to the
|
|
# training or development of such a model or algorithm.
|
|
#
|
|
# - The above copyright notice and this permission notice shall be included in
|
|
# all copies or substantial portions of the Software.
|
|
#
|
|
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
# SOFTWARE.
|
|
|
|
import RNS
|
|
import hashlib
|
|
import struct
|
|
import sys
|
|
import threading
|
|
import os
|
|
import time
|
|
from array import array
|
|
|
|
class AnnounceRateTable:
|
|
# Columnar store for announce rate tracking. Layout:
|
|
# _index : dict[destination_hash -> slot]
|
|
# _keys : list[slot] -> destination_hash (for swap-remove bookkeeping)
|
|
# _last : array('d') last non-violating announce time
|
|
# _blocked: array('d') blocked_until
|
|
# _viol : array('i') rate_violations
|
|
# _ts : array('d') flat ring of max_timestamps doubles per slot
|
|
# _ts_len : array('B') valid timestamp count per slot
|
|
# _ts_head: array('B') ring write cursor per slot
|
|
|
|
def __init__(self, max_timestamps=16):
|
|
assert max_timestamps <= 255, "Timestamp ring size must fit in a single byte"
|
|
self.max_ts = max_timestamps
|
|
self._index = {}
|
|
self._keys = []
|
|
self._last = array('d')
|
|
self._blocked = array('d')
|
|
self._viol = array('i')
|
|
self._ts = array('d')
|
|
self._ts_len = array('B')
|
|
self._ts_head = array('B')
|
|
self._cursor = 0
|
|
|
|
def __len__(self): return len(self._keys)
|
|
def __contains__(self, destination_hash): return destination_hash in self._index
|
|
|
|
# Perform the announce-rate check-and-update for one announce.
|
|
# Caller must hold the announce_rate_table_lock.
|
|
def check_and_update(self, destination_hash, now, target, grace, penalty):
|
|
slot = self._index.get(destination_hash)
|
|
|
|
if slot is None:
|
|
slot = len(self._keys)
|
|
self._index[destination_hash] = slot
|
|
self._keys.append(destination_hash)
|
|
self._last.append(now)
|
|
self._blocked.append(0.0)
|
|
self._viol.append(0)
|
|
self._ts.extend([0.0]*self.max_ts)
|
|
self._ts[slot*self.max_ts] = now
|
|
self._ts_len.append(1)
|
|
self._ts_head.append(1 % self.max_ts)
|
|
return False
|
|
|
|
base = slot*self.max_ts
|
|
head = self._ts_head[slot]
|
|
self._ts[base+head] = now
|
|
self._ts_head[slot] = (head+1) % self.max_ts
|
|
if self._ts_len[slot] < self.max_ts: self._ts_len[slot] += 1
|
|
|
|
current_rate = now - self._last[slot]
|
|
rate_blocked = False
|
|
|
|
if now > self._blocked[slot]:
|
|
if current_rate < target: self._viol[slot] += 1
|
|
else:
|
|
v = self._viol[slot] - 1
|
|
self._viol[slot] = v if v > 0 else 0
|
|
|
|
if self._viol[slot] > grace:
|
|
self._blocked[slot] = self._last[slot] + target + penalty
|
|
rate_blocked = True
|
|
else: self._last[slot] = now
|
|
|
|
else: rate_blocked = True
|
|
|
|
return rate_blocked
|
|
|
|
# Caller must hold the table lock
|
|
def cull(self, now, ttl, max_scan=1024, max_cull=None):
|
|
if max_cull is None: max_cull = max_scan
|
|
horizon = now - ttl
|
|
culled = 0
|
|
scanned = 0
|
|
n = len(self._keys)
|
|
|
|
while scanned < max_scan and culled < max_cull and n > 0:
|
|
if self._cursor >= n: self._cursor = 0
|
|
slot = self._cursor
|
|
|
|
if self._last[slot] < horizon and self._blocked[slot] <= now:
|
|
last_i = n - 1
|
|
stale_key = self._keys[slot]
|
|
|
|
if slot != last_i:
|
|
moved_key = self._keys[last_i]
|
|
self._keys[slot] = moved_key
|
|
self._index[moved_key] = slot
|
|
self._last[slot] = self._last[last_i]
|
|
self._blocked[slot] = self._blocked[last_i]
|
|
self._viol[slot] = self._viol[last_i]
|
|
self._ts_len[slot] = self._ts_len[last_i]
|
|
self._ts_head[slot] = self._ts_head[last_i]
|
|
|
|
f = last_i*self.max_ts; t = slot*self.max_ts
|
|
self._ts[t:t+self.max_ts] = self._ts[f:f+self.max_ts]
|
|
|
|
self._keys.pop()
|
|
self._last.pop(); self._blocked.pop(); self._viol.pop()
|
|
self._ts_len.pop(); self._ts_head.pop()
|
|
del self._ts[len(self._ts)-self.max_ts:] # drop tail block of removed slot
|
|
del self._index[stale_key]
|
|
|
|
n -= 1
|
|
culled += 1
|
|
# Do not advance cursor, re-examine the slot a moved entry landed in
|
|
|
|
else:
|
|
self._cursor = slot + 1
|
|
scanned += 1
|
|
|
|
if self._cursor >= n: self._cursor = 0
|
|
|
|
return culled
|
|
|
|
# Hold table lock, or accept a possibly inconsistent snapshot
|
|
def items(self):
|
|
for slot, key in enumerate(self._keys):
|
|
n = self._ts_len[slot]
|
|
head = self._ts_head[slot]
|
|
base = slot*self.max_ts
|
|
if n < self.max_ts: ts = list(self._ts[base:base+n])
|
|
else: ts = list(self._ts[base+head:base+self.max_ts]) + list(self._ts[base:base+head])
|
|
yield key, { "last": self._last[slot], "rate_violations": self._viol[slot],
|
|
"blocked_until": self._blocked[slot], "timestamps": ts }
|
|
|
|
# Persists the announce rate table to a file, in
|
|
# compacted, little-endian binary format:
|
|
#
|
|
# Header (16 bytes)
|
|
# 0x00 6 Magic bytes
|
|
# 0x06 1 Format version
|
|
# 0x07 1 Flags, reserved, must be zero
|
|
# 0x08 1 Timestamp ring size (max_ts)
|
|
# 0x09 1 Destination hash length in bytes (keylen)
|
|
# 0x0A 2 Reserved, zero
|
|
# 0x0C 4 Entry count, uint32, little-endian
|
|
#
|
|
# Entries (columnar, in native slot order)
|
|
# Destination hash keys, followed by the _last, _blocked,
|
|
# _viol, _ts_len, _ts_head and _ts columns, all encoded as
|
|
# little-endian fixed-width values.
|
|
#
|
|
# Caller must hold the announce rate table lock
|
|
|
|
PERSIST_MAGIC = b"RNSART"
|
|
PERSIST_VERSION = 0x01
|
|
def persist(self, path, retain=None):
|
|
try:
|
|
count = len(self._keys)
|
|
|
|
if count > 0:
|
|
keylen = len(self._keys[0])
|
|
for key in self._keys:
|
|
if len(key) != keylen: raise ValueError("Inconsistent destination hash lengths in table")
|
|
else:
|
|
keylen = RNS.Identity.TRUNCATED_HASHLENGTH//8
|
|
|
|
if retain != None and count > 0:
|
|
now = time.time()
|
|
horizon = now - retain
|
|
keep = [slot for slot in range(count) if not (self._last[slot] < horizon and self._blocked[slot] <= now)]
|
|
else:
|
|
keep = None
|
|
|
|
if keep != None and len(keep) != count:
|
|
count = len(keep)
|
|
keys = [self._keys[slot] for slot in keep]
|
|
last = array("d", [self._last[slot] for slot in keep])
|
|
blocked = array("d", [self._blocked[slot] for slot in keep])
|
|
viol = array("i", [self._viol[slot] for slot in keep])
|
|
ts_len = array("B", [self._ts_len[slot] for slot in keep])
|
|
ts_head = array("B", [self._ts_head[slot] for slot in keep])
|
|
ts = array("d")
|
|
for slot in keep: ts.extend(self._ts[slot*self.max_ts:(slot+1)*self.max_ts])
|
|
else:
|
|
keys = self._keys
|
|
last = self._last
|
|
blocked = self._blocked
|
|
viol = self._viol
|
|
ts_len = self._ts_len
|
|
ts_head = self._ts_head
|
|
ts = self._ts
|
|
|
|
if last.itemsize != 8 or viol.itemsize != 4:
|
|
raise ValueError(f"Unsupported array itemsize at persist: d={last.itemsize}, i={viol.itemsize}")
|
|
|
|
header = self.PERSIST_MAGIC
|
|
header += bytes([self.PERSIST_VERSION, 0x00, self.max_ts & 0xFF, keylen & 0xFF])
|
|
header += struct.pack("<H", 0)
|
|
header += struct.pack("<I", count)
|
|
|
|
blob = header
|
|
blob += b"".join(keys)
|
|
blob += self._column_le(last)
|
|
blob += self._column_le(blocked)
|
|
blob += self._column_le(viol)
|
|
blob += ts_len.tobytes()
|
|
blob += ts_head.tobytes()
|
|
blob += self._column_le(ts)
|
|
|
|
tmp_path = path+".tmp"
|
|
with open(tmp_path, "wb") as file: file.write(blob)
|
|
os.replace(tmp_path, path)
|
|
return True
|
|
|
|
except Exception as e:
|
|
RNS.log(f"Could not persist announce rate table to {path}: {e}", RNS.LOG_ERROR)
|
|
return False
|
|
|
|
# Caller must hold the announce rate table lock
|
|
def load(self, path):
|
|
try:
|
|
with open(path, "rb") as file: blob = file.read()
|
|
except Exception: return False
|
|
|
|
try:
|
|
if len(blob) < 16: return False
|
|
if blob[0:6] != self.PERSIST_MAGIC: return False
|
|
if blob[6] != self.PERSIST_VERSION: return False
|
|
if blob[7] != 0x00: return False
|
|
|
|
file_max_ts = blob[8]
|
|
keylen = blob[9]
|
|
count = struct.unpack_from("<I", blob, 12)[0]
|
|
|
|
if file_max_ts < 1 or file_max_ts > 255: return False
|
|
if keylen < 1: return False
|
|
|
|
entry_size = keylen+22+file_max_ts*8
|
|
if len(blob) != 16+count*entry_size: return False
|
|
|
|
if file_max_ts == self.max_ts:
|
|
# Ring size matches, load directly
|
|
keys = [blob[16+slot*keylen : 16+(slot+1)*keylen] for slot in range(count)]
|
|
index = {}
|
|
for slot in range(count):
|
|
if keys[slot] in index: return False
|
|
index[keys[slot]] = slot
|
|
|
|
offset = 16+count*keylen
|
|
last = self._column_from_le("d", blob[offset:offset+count*8]); offset += count*8
|
|
blocked = self._column_from_le("d", blob[offset:offset+count*8]); offset += count*8
|
|
viol = self._column_from_le("i", blob[offset:offset+count*4]); offset += count*4
|
|
ts_len = array("B", blob[offset:offset+count]); offset += count
|
|
ts_head = array("B", blob[offset:offset+count]); offset += count
|
|
ts = self._column_from_le("d", blob[offset:offset+count*self.max_ts*8]); offset += count*self.max_ts*8
|
|
|
|
for slot in range(count):
|
|
if ts_len[slot] > self.max_ts or ts_head[slot] >= self.max_ts: return False
|
|
|
|
else:
|
|
# Different ring sizes from current running config,
|
|
# rebuild entries best effort.
|
|
keys = [blob[16+slot*keylen : 16+(slot+1)*keylen] for slot in range(count)]
|
|
index = {}
|
|
for slot in range(count):
|
|
if keys[slot] in index: return False
|
|
index[keys[slot]] = slot
|
|
|
|
offset = 16+count*keylen
|
|
last = self._column_from_le("d", blob[offset:offset+count*8]); offset += count*8
|
|
blocked = self._column_from_le("d", blob[offset:offset+count*8]); offset += count*8
|
|
viol = self._column_from_le("i", blob[offset:offset+count*4]); offset += count*4
|
|
f_ts_len = array("B", blob[offset:offset+count]); offset += count
|
|
f_ts_head = array("B", blob[offset:offset+count]); offset += count
|
|
f_ts = self._column_from_le("d", blob[offset:offset+count*file_max_ts*8]); offset += count*file_max_ts*8
|
|
|
|
for slot in range(count):
|
|
if f_ts_len[slot] > file_max_ts or f_ts_head[slot] >= file_max_ts: return False
|
|
|
|
ts_len = array("B"); ts_head = array("B"); ts = array("d")
|
|
for slot in range(count):
|
|
base = slot*file_max_ts
|
|
if f_ts_len[slot] < file_max_ts: chronological = list(f_ts[base:base+f_ts_len[slot]])
|
|
else: chronological = list(f_ts[base+f_ts_head[slot]:base+file_max_ts])+list(f_ts[base:base+f_ts_head[slot]])
|
|
chronological = chronological[-self.max_ts:]
|
|
ts_count = len(chronological)
|
|
|
|
ts_len.append(ts_count)
|
|
ts.extend(chronological)
|
|
if ts_count < self.max_ts: ts.extend([0.0]*(self.max_ts-ts_count))
|
|
ts_head.append(ts_count % self.max_ts)
|
|
|
|
self._index = index
|
|
self._keys = keys
|
|
self._last = last
|
|
self._blocked = blocked
|
|
self._viol = viol
|
|
self._ts = ts
|
|
self._ts_len = ts_len
|
|
self._ts_head = ts_head
|
|
self._cursor = 0
|
|
|
|
return True
|
|
|
|
except Exception as e:
|
|
RNS.log(f"Could not load announce rate table from {path}: {e}", RNS.LOG_ERROR)
|
|
return False
|
|
|
|
def _column_le(self, column):
|
|
if sys.byteorder == "big": swapped = array(column.typecode, column); swapped.byteswap(); return swapped.tobytes()
|
|
else: return column.tobytes()
|
|
|
|
def _column_from_le(self, typecode, data):
|
|
column = array(typecode); column.frombytes(data)
|
|
if sys.byteorder == "big": column.byteswap()
|
|
return column
|
|
|
|
class PacketHashlist:
|
|
# Bucketed byte-blob packet filter hashlist, with the primary
|
|
# design goal of memory coherence and density. All operations
|
|
# are safe to call without holding a lock.
|
|
|
|
def __init__(self, hash_width=32, bucket_count=16384):
|
|
assert bucket_count > 0 and bucket_count & (bucket_count-1) == 0, "Bucket count must be a power of two"
|
|
self.width = hash_width
|
|
self._nb = bucket_count
|
|
self._mask = bucket_count - 1
|
|
self._cur = [bytearray() for _ in range(bucket_count)]
|
|
self._prv = [bytearray() for _ in range(bucket_count)]
|
|
self._count = 0
|
|
|
|
def __len__(self):
|
|
return self._count
|
|
|
|
def __contains__(self, packet_hash):
|
|
bucket = (packet_hash[0] | (packet_hash[1] << 8)) & self._mask
|
|
return packet_hash in self._cur[bucket] or packet_hash in self._prv[bucket]
|
|
|
|
def add(self, packet_hash):
|
|
assert len(packet_hash) == self.width, "Packet hash width mismatch"
|
|
bucket = (packet_hash[0] | (packet_hash[1] << 8)) & self._mask
|
|
if packet_hash not in self._cur[bucket]:
|
|
self._cur[bucket].extend(packet_hash)
|
|
self._count += 1
|
|
|
|
def discard(self, packet_hash):
|
|
w = self.width
|
|
bucket = (packet_hash[0] | (packet_hash[1] << 8)) & self._mask
|
|
for generation in (self._cur, self._prv):
|
|
entries = generation[bucket]
|
|
pos = entries.find(packet_hash)
|
|
while pos != -1:
|
|
if pos % w == 0:
|
|
del entries[pos:pos+w]
|
|
if generation is self._cur: self._count -= 1
|
|
pos = entries.find(packet_hash, pos)
|
|
else:
|
|
pos = entries.find(packet_hash, pos+1)
|
|
|
|
def swap(self):
|
|
self._prv = self._cur
|
|
self._cur = [bytearray() for _ in range(self._nb)]
|
|
self._count = 0
|
|
|
|
# Current-generation buckets for persist operations
|
|
def buckets(self): return tuple(self._cur)
|
|
|
|
class KnownDestinationsStore:
|
|
# Write-behind SQLite storage backend for known destinations.
|
|
# The actual in-memory table is used directly for all transport
|
|
# operations; this serves purely as the store, and no read operations
|
|
# utilize the database directly.
|
|
|
|
def __init__(self, db_path, hash_width=16, read_only=False):
|
|
import sqlite3
|
|
self.sqlite3 = sqlite3
|
|
self.db_path = db_path
|
|
self.hash_width = hash_width
|
|
self.read_only = read_only
|
|
self._dirty_upsert = set()
|
|
self._dirty_delete = set()
|
|
self._ratchet_upserts = {}
|
|
self._ratchet_deletes = set()
|
|
self._kd_quarantine = {}
|
|
self._ratchet_quarantine = {}
|
|
self._flush_lock = threading.Lock()
|
|
|
|
if read_only: self._conn = sqlite3.connect(f"file:{db_path}?mode=ro", uri=True, check_same_thread=False, isolation_level=None, cached_statements=0)
|
|
else:
|
|
self._conn = sqlite3.connect(db_path, check_same_thread=False, isolation_level=None, cached_statements=0)
|
|
self._conn.execute("PRAGMA journal_mode=WAL")
|
|
self._conn.execute("PRAGMA synchronous=NORMAL")
|
|
self._conn.execute("PRAGMA journal_size_limit=8388608")
|
|
self.__create_schema()
|
|
|
|
def close(self):
|
|
if self._conn != None:
|
|
self._conn.close()
|
|
self._conn = None
|
|
|
|
def __create_schema(self):
|
|
db_version = self._conn.execute("PRAGMA user_version").fetchone()[0]
|
|
|
|
# Pre-release database cleanup
|
|
# TODO: Remove this in 1.6.1
|
|
if db_version > 0:
|
|
RNS.log(f"Pre-release database detected, rebuilding known destinations DB to release format", RNS.LOG_WARNING)
|
|
self._conn.execute("DROP TABLE IF EXISTS known_destinations")
|
|
|
|
self._conn.execute("""CREATE TABLE IF NOT EXISTS known_destinations (
|
|
destination_hash BLOB PRIMARY KEY,
|
|
last_heard REAL NOT NULL,
|
|
packet_hash BLOB,
|
|
public_key BLOB NOT NULL,
|
|
app_data BLOB,
|
|
used REAL NOT NULL DEFAULT 0,
|
|
stamp BLOB,
|
|
stamp_value REAL,
|
|
lr_cost REAL,
|
|
identity_hash BLOB NOT NULL)""")
|
|
|
|
self._conn.execute("""CREATE TABLE IF NOT EXISTS ratchets (
|
|
destination_hash BLOB PRIMARY KEY,
|
|
ratchet BLOB NOT NULL,
|
|
received REAL NOT NULL)""")
|
|
|
|
# Schema migrations
|
|
try:
|
|
self._conn.execute("BEGIN")
|
|
# No migrations yet
|
|
self._conn.execute("PRAGMA user_version = 0")
|
|
self._conn.execute("COMMIT")
|
|
self._conn.commit()
|
|
|
|
except Exception as e:
|
|
RNS.log(f"Known destinations database version migration failed: {e}", RNS.LOG_CRITICAL)
|
|
RNS.log(f"You will need to remove the known_destinations.db* files from your storage directory so it can be rebuilt", RNS.LOG_CRITICAL)
|
|
|
|
def __entry_row(self, destination_hash, entry):
|
|
stamp = entry[5] if len(entry) > 5 else None
|
|
stamp_value = entry[6] if len(entry) > 6 else 0
|
|
lr_cost = entry[7] if len(entry) > 7 else None
|
|
identity_hash = hashlib.sha256(entry[2]).digest()[:self.hash_width]
|
|
return (destination_hash, entry[0], entry[1], entry[2], entry[3], entry[4], stamp, stamp_value, lr_cost, identity_hash)
|
|
|
|
# Caller must hold Identity.known_destinations_lock
|
|
def mark_dirty(self, destination_hash):
|
|
if self.read_only: return
|
|
self._dirty_delete.discard(destination_hash)
|
|
self._kd_quarantine.pop(destination_hash, None)
|
|
self._dirty_upsert.add(destination_hash)
|
|
|
|
# Caller must hold Identity.known_destinations_lock
|
|
def mark_deleted(self, destination_hash):
|
|
if self.read_only: return
|
|
self._dirty_upsert.discard(destination_hash)
|
|
self._dirty_delete.add(destination_hash)
|
|
self._kd_quarantine.pop(destination_hash, None)
|
|
self._ratchet_quarantine.pop(destination_hash, None)
|
|
|
|
# Caller must hold Identity.known_destinations_lock
|
|
def take_pending(self):
|
|
if self.read_only: return set(), set(), {}, set()
|
|
upserts = self._dirty_upsert
|
|
deletes = self._dirty_delete
|
|
ratchet_upserts = self._ratchet_upserts
|
|
ratchet_deletes = self._ratchet_deletes
|
|
self._dirty_upsert = set()
|
|
self._dirty_delete = set()
|
|
self._ratchet_upserts = {}
|
|
self._ratchet_deletes = set()
|
|
return upserts, deletes, ratchet_upserts, ratchet_deletes
|
|
|
|
# Caller must hold the flush lock and have an already open transaction
|
|
def __wrap_exec(self, sql, rows):
|
|
rejects = []
|
|
|
|
# By default, run batched queries in chunks.
|
|
for i in range(0, len(rows), 4096):
|
|
chunk = rows[i:i+4096]
|
|
try:
|
|
self._conn.execute("SAVEPOINT chunk")
|
|
self._conn.executemany(sql, chunk)
|
|
self._conn.execute("RELEASE chunk")
|
|
|
|
# But on integrity and data errors, roll back the chunk
|
|
# and iterate per-row, so we can catch any violating
|
|
# entries and quarantine them. A later, sane update
|
|
# will then unquarantine and heal upon arrival.
|
|
#
|
|
# Other exceptions are intentionally not caught here,
|
|
# so they can propagate directly.
|
|
except (self.sqlite3.IntegrityError, self.sqlite3.DataError):
|
|
self._conn.execute("ROLLBACK TO chunk")
|
|
for row in chunk:
|
|
try:
|
|
self._conn.execute("SAVEPOINT row")
|
|
self._conn.execute(sql, row)
|
|
self._conn.execute("RELEASE row")
|
|
|
|
except (self.sqlite3.IntegrityError, self.sqlite3.DataError) as e:
|
|
self._conn.execute("ROLLBACK TO row")
|
|
rejects.append((row[0], str(e)))
|
|
|
|
self._conn.execute("RELEASE chunk")
|
|
|
|
return rejects
|
|
|
|
# May be called without any locks held
|
|
def apply(self, upserts, deletes, ratchet_upserts, ratchet_deletes, known_destinations):
|
|
if self.read_only: return 0
|
|
if len(upserts) == 0 and len(deletes) == 0 and len(ratchet_upserts) == 0 and len(ratchet_deletes) == 0: return 0
|
|
|
|
with self._flush_lock:
|
|
rows = []
|
|
removables = []
|
|
ratchet_rows = []
|
|
ratchet_removables = []
|
|
|
|
for destination_hash in upserts:
|
|
if destination_hash in self._kd_quarantine: continue
|
|
entry = known_destinations.get(destination_hash)
|
|
if entry != None: rows.append(self.__entry_row(destination_hash, entry))
|
|
|
|
for destination_hash in deletes:
|
|
if destination_hash not in known_destinations: removables.append((destination_hash,))
|
|
|
|
for destination_hash, ratchet_entry in ratchet_upserts.items():
|
|
# Ratchets of purged destinations must be removed as well
|
|
if destination_hash not in known_destinations: ratchet_removables.append((destination_hash,))
|
|
elif destination_hash not in self._ratchet_quarantine: ratchet_rows.append((destination_hash, *ratchet_entry))
|
|
|
|
for destination_hash in ratchet_deletes:
|
|
if destination_hash not in known_destinations: ratchet_removables.append((destination_hash,))
|
|
|
|
kd_rejects = []
|
|
ratchet_rejects = []
|
|
try:
|
|
self._conn.execute("BEGIN")
|
|
kd_rejects += self.__wrap_exec("INSERT INTO known_destinations (destination_hash, last_heard, packet_hash, public_key, app_data, used, stamp, stamp_value, lr_cost, identity_hash) VALUES (?,?,?,?,?,?,?,?,?,?) ON CONFLICT(destination_hash) DO UPDATE SET last_heard=excluded.last_heard, packet_hash=excluded.packet_hash, public_key=excluded.public_key, app_data=excluded.app_data, used=excluded.used, stamp=excluded.stamp, stamp_value=excluded.stamp_value, lr_cost=excluded.lr_cost, identity_hash=excluded.identity_hash", rows)
|
|
kd_rejects += self.__wrap_exec("DELETE FROM known_destinations WHERE destination_hash=?", removables)
|
|
ratchet_rejects += self.__wrap_exec("INSERT INTO ratchets VALUES (?,?,?) ON CONFLICT(destination_hash) DO UPDATE SET ratchet=excluded.ratchet, received=excluded.received", ratchet_rows)
|
|
ratchet_rejects += self.__wrap_exec("DELETE FROM ratchets WHERE destination_hash=?", ratchet_removables)
|
|
self._conn.commit()
|
|
|
|
except Exception:
|
|
# If an exception is not handled in the batch query wrapper,
|
|
# we will have to roll back the transaction, and mark all
|
|
# attempted entries as dirty. A failure state like such
|
|
# should not happen, and this is a last resort; at this
|
|
# point, writes may stall until an updated entry for
|
|
# the violating row(s) arrive and heal the invalid
|
|
# dirty entries, so this is raised for logging.
|
|
self._dirty_upsert |= {row[0] for row in rows}
|
|
self._dirty_delete |= {row[0] for row in removables}
|
|
self._ratchet_upserts |= {row[0]: row[1:] for row in ratchet_rows}
|
|
self._ratchet_deletes |= {row[0] for row in ratchet_removables}
|
|
self._conn.rollback()
|
|
raise
|
|
|
|
# Track failed persists and one-time log per failure. A later
|
|
# update of the destination clears the quarantine state and
|
|
# allows persist to retry with the (hopefully) valid data.
|
|
if len(kd_rejects) > 0 or len(ratchet_rejects) > 0:
|
|
for destination_hash, error in kd_rejects:
|
|
self._kd_quarantine[destination_hash] = (error, time.time())
|
|
RNS.log(f"Invalid known destination data for {RNS.prettyhexrep(destination_hash)} at persist. The contained exception was: {error}", RNS.LOG_ERROR)
|
|
|
|
for destination_hash, error in ratchet_rejects:
|
|
self._ratchet_quarantine[destination_hash] = (error, time.time())
|
|
RNS.log(f"Invalid ratchet data for {RNS.prettyhexrep(destination_hash)} at persist. The contained exception was: {error}", RNS.LOG_ERROR)
|
|
|
|
return len(rows)+len(removables)+len(ratchet_rows)+len(ratchet_removables)-len(kd_rejects)-len(ratchet_rejects)
|
|
|
|
def load_all(self):
|
|
cursor = self._conn.execute("SELECT destination_hash, last_heard, packet_hash, public_key, app_data, used, stamp, stamp_value, lr_cost FROM known_destinations")
|
|
while True:
|
|
rows = cursor.fetchmany(4096)
|
|
if not rows: break
|
|
for row in rows: yield row[0], [row[1], row[2], row[3], row[4], row[5], row[6], row[7], row[8]]
|
|
|
|
def remember_ratchet(self, destination_hash, ratchet, now):
|
|
if self.read_only: return False
|
|
self._ratchet_quarantine.pop(destination_hash, None)
|
|
|
|
pending = self._ratchet_upserts.get(destination_hash)
|
|
if pending != None:
|
|
if pending[0] == ratchet:
|
|
self._ratchet_deletes.discard(destination_hash)
|
|
return False
|
|
|
|
else:
|
|
with self._flush_lock:
|
|
row = self._conn.execute("SELECT ratchet FROM ratchets WHERE destination_hash=?", (destination_hash,)).fetchone()
|
|
|
|
if row != None and row[0] == ratchet:
|
|
# Discard any pending delete mark queued
|
|
# before this ratchet arrived
|
|
self._ratchet_deletes.discard(destination_hash)
|
|
return False
|
|
|
|
self._ratchet_upserts[destination_hash] = (ratchet, now)
|
|
self._ratchet_deletes.discard(destination_hash)
|
|
return True
|
|
|
|
def fetch_ratchet(self, destination_hash):
|
|
pending = self._ratchet_upserts.get(destination_hash)
|
|
if pending != None: return pending[0], pending[1]
|
|
if destination_hash in self._ratchet_deletes: return None
|
|
|
|
with self._flush_lock:
|
|
row = self._conn.execute("SELECT ratchet, received FROM ratchets WHERE destination_hash=?", (destination_hash,)).fetchone()
|
|
|
|
if row != None: return row[0], row[1]
|
|
|
|
return None
|
|
|
|
# Caller must hold Identity.known_destinations_lock
|
|
def mark_ratchet_deleted(self, destination_hash):
|
|
if self.read_only: return
|
|
self._ratchet_upserts.pop(destination_hash, None)
|
|
self._ratchet_deletes.add(destination_hash)
|
|
self._ratchet_quarantine.pop(destination_hash, None)
|
|
|
|
def expire_ratchets(self, cutoff):
|
|
if self.read_only: return []
|
|
|
|
with self._flush_lock:
|
|
# Drop expired entries in pending as well
|
|
expired = set(row[0] for row in self._conn.execute("SELECT destination_hash FROM ratchets WHERE received<?", (cutoff,)))
|
|
for destination_hash, entry in list(self._ratchet_upserts.items()):
|
|
if entry[1] < cutoff:
|
|
expired.add(destination_hash)
|
|
self._ratchet_upserts.pop(destination_hash, None)
|
|
|
|
expired.difference_update(self._ratchet_deletes)
|
|
if len(expired) > 0:
|
|
self._conn.execute("BEGIN")
|
|
for i in range(0, len(expired), 4096):
|
|
self._conn.executemany("DELETE FROM ratchets WHERE destination_hash=?", [(h,) for h in list(expired)[i:i+4096]])
|
|
self._conn.commit()
|
|
|
|
return list(expired)
|
|
|
|
def load_ratchets(self):
|
|
cursor = self._conn.execute("SELECT destination_hash, ratchet, received FROM ratchets")
|
|
while True:
|
|
rows = cursor.fetchmany(4096)
|
|
if not rows: break
|
|
for row in rows: yield row[0], row[1], row[2]
|
|
|
|
def import_legacy_ratchets(self, dir_path, expiry_cutoff, known_destinations, ratchet_width=32):
|
|
from .vendor import umsgpack
|
|
imported = 0
|
|
for filename in os.listdir(dir_path):
|
|
file_path = os.path.join(dir_path, filename)
|
|
try:
|
|
if not os.path.isfile(file_path): continue
|
|
with open(file_path, "rb") as rf:
|
|
ratchet_data = umsgpack.unpackb(rf.read())
|
|
|
|
destination_hash = bytes.fromhex(filename)
|
|
ratchet = ratchet_data["ratchet"]; received = ratchet_data["received"]
|
|
if len(ratchet) != ratchet_width: raise ValueError("Invalid ratchet width")
|
|
if received < expiry_cutoff: raise ValueError("Ratchet expired")
|
|
if destination_hash not in known_destinations: raise ValueError("Destination unknown")
|
|
|
|
self._ratchet_upserts[destination_hash] = (ratchet, received)
|
|
imported += 1
|
|
os.unlink(file_path)
|
|
|
|
except Exception as e:
|
|
RNS.log(f"Could not import ratchet for {filename}: {e}", RNS.LOG_WARNING)
|
|
try: os.unlink(file_path)
|
|
except: pass
|
|
|
|
pending = self._ratchet_upserts
|
|
self._ratchet_upserts = {}
|
|
self.apply(set(), set(), pending, set(), known_destinations)
|
|
|
|
return imported
|
|
|
|
def import_legacy(self, path):
|
|
from .vendor import umsgpack
|
|
with open(path, "rb") as file: legacy = umsgpack.load(file)
|
|
|
|
rows = []
|
|
now = time.time()
|
|
for destination_hash, entry in legacy.items():
|
|
if len(entry) < 5: continue
|
|
entry = list(entry)
|
|
while len(entry) < 8: entry.append(None if len(entry) in (5, 7) else 0)
|
|
if entry[0] == None or entry[0] <= 0: entry[0] = now
|
|
rows.append(self.__entry_row(destination_hash, entry))
|
|
|
|
self._conn.execute("BEGIN")
|
|
for i in range(0, len(rows), 4096): self._conn.executemany("INSERT OR REPLACE INTO known_destinations (destination_hash, last_heard, packet_hash, public_key, app_data, used, stamp, stamp_value, lr_cost, identity_hash) VALUES (?,?,?,?,?,?,?,?,?,?)", rows[i:i+4096])
|
|
self._conn.commit()
|
|
|
|
return len(rows)
|
|
|
|
class AnnounceCache:
|
|
|
|
def __init__(self, db_path):
|
|
import sqlite3
|
|
self.db_path = db_path
|
|
self._pending = {}
|
|
self._flush_lock = threading.Lock()
|
|
self._conn = sqlite3.connect(db_path, check_same_thread=False, isolation_level=None, cached_statements=0)
|
|
|
|
self._conn.execute("PRAGMA auto_vacuum=INCREMENTAL")
|
|
self._conn.execute("PRAGMA journal_mode=WAL")
|
|
self._conn.execute("PRAGMA synchronous=NORMAL")
|
|
self._conn.execute("PRAGMA journal_size_limit=8388608")
|
|
self.__create_schema()
|
|
|
|
def __create_schema(self):
|
|
self._conn.execute("""CREATE TABLE IF NOT EXISTS cached_announces (
|
|
packet_hash BLOB PRIMARY KEY,
|
|
received REAL NOT NULL,
|
|
raw BLOB NOT NULL,
|
|
interface_reference TEXT)""")
|
|
|
|
self._conn.execute("PRAGMA user_version = 1")
|
|
|
|
def store(self, packet_hash, raw, interface_reference):
|
|
self._pending[packet_hash] = (time.time(), raw, interface_reference)
|
|
|
|
def pending_count(self):
|
|
return len(self._pending)
|
|
|
|
def fetch(self, packet_hash):
|
|
pending = self._pending.get(packet_hash)
|
|
if pending != None: return pending[1], pending[2]
|
|
|
|
with self._flush_lock:
|
|
row = self._conn.execute("SELECT raw, interface_reference FROM cached_announces WHERE packet_hash=?", (packet_hash,)).fetchone()
|
|
|
|
if row != None: return row[0], row[1]
|
|
|
|
return None
|
|
|
|
def flush(self):
|
|
with self._flush_lock: return self.__apply_pending()
|
|
|
|
def __apply_pending(self):
|
|
pending = self._pending
|
|
self._pending = {}
|
|
if len(pending) == 0: return 0
|
|
|
|
try:
|
|
self._conn.execute("BEGIN")
|
|
items = list(pending.items())
|
|
for i in range(0, len(items), 4096):
|
|
self._conn.executemany("INSERT INTO cached_announces VALUES (?,?,?,?) ON CONFLICT(packet_hash) DO UPDATE SET received=excluded.received, raw=excluded.raw, interface_reference=excluded.interface_reference",
|
|
[(packet_hash, *entry) for packet_hash, entry in items[i:i+4096]])
|
|
self._conn.commit()
|
|
|
|
except Exception:
|
|
# Re-mark everything as pending, so the changes
|
|
# are retried on the next flush, then propagate.
|
|
self._pending.update(pending)
|
|
self._conn.rollback()
|
|
raise
|
|
|
|
return len(pending)
|
|
|
|
def clean(self, active_hashes):
|
|
with self._flush_lock:
|
|
self.__apply_pending()
|
|
|
|
try:
|
|
self._conn.execute("BEGIN")
|
|
self._conn.execute("DROP TABLE IF EXISTS temp.active_announces")
|
|
self._conn.execute("CREATE TEMP TABLE active_announces (packet_hash BLOB PRIMARY KEY)")
|
|
active_list = list(active_hashes)
|
|
for i in range(0, len(active_list), 4096):
|
|
self._conn.executemany("INSERT OR IGNORE INTO active_announces VALUES (?)", [(h,) for h in active_list[i:i+4096]])
|
|
|
|
changes_before = self._conn.total_changes
|
|
self._conn.execute("DELETE FROM cached_announces WHERE packet_hash NOT IN (SELECT packet_hash FROM active_announces)")
|
|
removed = self._conn.total_changes - changes_before
|
|
self._conn.commit()
|
|
|
|
except Exception:
|
|
self._conn.rollback()
|
|
raise
|
|
|
|
if removed > 0: self._conn.execute("PRAGMA incremental_vacuum")
|
|
|
|
return removed
|
|
|
|
def import_legacy(self, dir_path):
|
|
from .vendor import umsgpack
|
|
imported = 0
|
|
for filename in os.listdir(dir_path):
|
|
file_path = os.path.join(dir_path, filename)
|
|
try:
|
|
if not os.path.isfile(file_path): continue
|
|
with open(file_path, "rb") as file:
|
|
cached_data = umsgpack.unpackb(file.read())
|
|
|
|
packet_hash = bytes.fromhex(filename)
|
|
interface_reference = cached_data[1] if len(cached_data) > 1 else None
|
|
self.store(packet_hash, cached_data[0], interface_reference)
|
|
imported += 1
|
|
os.unlink(file_path)
|
|
|
|
except Exception:
|
|
try: os.unlink(file_path)
|
|
except Exception: pass
|
|
|
|
self.flush()
|
|
return imported
|
|
|
|
def close(self):
|
|
if self._conn != None:
|
|
self._conn.close()
|
|
self._conn = None
|