Files
skuld/ledger.py
T

447 lines
15 KiB
Python

"""The lot ledger of docs/spec.md section 5.
Every incoming quantity of BTC becomes a lot; every outgoing quantity consumes
sats from one or more lots and records what it consumed. `rebuild()` throws the
lots and the automatic consumptions away and replays every transaction, so the
ledger can never drift away from the transactions.
Hand-picked lots live in `lot_picks`, which is the owner's input and survives a
rebuild; `lot_consumptions` is derived from it and is therefore rebuilt too.
"""
from functools import cmp_to_key
import database
import rates
SATS_PER_BTC = rates.SATS_PER_BTC
LOT_METHODS = ("FIFO", "LIFO", "HIFO")
# Types that take sats out of an account. For a loss and a gift given a fee is
# only recorded. admin_cost is not here at all: it stands on its own and changes
# no holdings (section 3.10).
CONSUMING_TYPES = (
"trade",
"spend",
"loss",
"transfer_out",
"gift_given",
)
# A sats fee is taken on top of the amount consumed for these types. A transfer
# takes it on top of what is sent, so the sats that are sent are exactly the sats
# that arrive (docs/spec.md sections 3.2 and 3.3).
FEE_ADDS_TO_CONSUMPTION = ("trade", "spend", "transfer_out")
class LedgerError(Exception):
"""Raised when the ledger cannot be written as asked. Message is Nynorsk."""
def _cost_for(cost_ore, sats, take):
"""Cost basis of `take` sats out of a lot with `sats` sats and `cost_ore` ore.
Rounds half up, and gives the whole remainder when the last sat is taken, so
no ore is created or lost.
"""
if take >= sats:
return cost_ore
return (2 * cost_ore * take + sats) // (2 * sats)
def _lot_view(lot):
view = dict(lot)
sats = view.get("sats_remaining") or 0
view["btc_price_ore"] = (
view["cost_ore_remaining"] * SATS_PER_BTC // sats if sats else 0
)
return view
def _insert_lot(conn, market_id, acquired_date, origin_tx_id, source_lot_id, origin_seq,
sats, cost_ore):
cursor = conn.execute(
"INSERT INTO lots (market_id, acquired_date, origin_tx_id, source_lot_id, origin_seq,"
" sats_original, sats_remaining, cost_ore_original, cost_ore_remaining)"
" VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)",
(market_id, acquired_date, origin_tx_id, source_lot_id, origin_seq, sats, sats,
cost_ore, cost_ore),
)
return cursor.lastrowid
def _lot_method(conn):
row = conn.execute("SELECT value FROM settings WHERE key = 'lot_method'").fetchone()
method = row["value"] if row else "FIFO"
return method if method in LOT_METHODS else "FIFO"
def _open_lots(conn, market_id, date):
"""Lots of an account that existed at `date` and still hold sats."""
rows = conn.execute(
"SELECT l.id, l.market_id, l.acquired_date, l.origin_tx_id, l.origin_seq,"
" l.sats_remaining, l.cost_ore_remaining FROM lots l"
" JOIN transactions o ON o.id = l.origin_tx_id"
" WHERE l.market_id = ? AND o.date <= ? AND l.sats_remaining > 0"
" ORDER BY l.acquired_date, l.id",
(market_id, date),
).fetchall()
return [dict(row) for row in rows]
def _lot_by_origin(conn, market_id, origin_tx_id, origin_seq):
row = conn.execute(
"SELECT id, market_id, acquired_date, origin_tx_id, origin_seq, sats_remaining,"
" cost_ore_remaining FROM lots"
" WHERE market_id = ? AND origin_tx_id = ? AND origin_seq = ?",
(market_id, origin_tx_id, origin_seq),
).fetchone()
return dict(row) if row else None
def _sats_to_consume(tx):
"""How many sats a transaction takes out of its account.
A trade consumes only when BTC leaves the account (a sale). A buy carries
its sats fee on the incoming leg, where it lowers the sats received, so it
must not consume anything.
"""
if tx["type"] not in CONSUMING_TYPES:
return 0
sats = tx["leg_out_sats"] or 0
if sats <= 0:
return 0
if tx["type"] in FEE_ADDS_TO_CONSUMPTION:
sats += tx["fee_sats"] or 0
return sats
def _compare_hifo(left, right):
left_rate = left["cost_ore_remaining"] * right["sats_remaining"]
right_rate = right["cost_ore_remaining"] * left["sats_remaining"]
if left_rate != right_rate:
return -1 if left_rate > right_rate else 1
if left["acquired_date"] != right["acquired_date"]:
return -1 if left["acquired_date"] < right["acquired_date"] else 1
return left["id"] - right["id"]
def _ordered(lots, method):
if method == "LIFO":
return sorted(lots, key=lambda lot: (lot["acquired_date"], lot["id"]), reverse=True)
if method == "HIFO":
return sorted(lots, key=cmp_to_key(_compare_hifo))
return sorted(lots, key=lambda lot: (lot["acquired_date"], lot["id"]))
def _take(conn, lot, take, tx_id, auto):
cost_ore = _cost_for(lot["cost_ore_remaining"], lot["sats_remaining"], take)
conn.execute(
"UPDATE lots SET sats_remaining = sats_remaining - ?,"
" cost_ore_remaining = cost_ore_remaining - ? WHERE id = ?",
(take, cost_ore, lot["id"]),
)
conn.execute(
"INSERT INTO lot_consumptions (tx_id, lot_id, sats, cost_ore, auto)"
" VALUES (?, ?, ?, ?, ?)",
(tx_id, lot["id"], take, cost_ore, auto),
)
lot["sats_remaining"] -= take
lot["cost_ore_remaining"] -= cost_ore
return {"lot_id": lot["id"], "sats": take, "cost_ore": cost_ore, "auto": auto}
def pick_for_method(lots, sats, method):
"""Choose lots for `sats`, in the order the method prescribes.
Returns a list of (lot, sats) pairs covering as much as the lots hold, so a
caller can tell whether the account is short.
"""
chosen = []
remaining = int(sats)
for lot in _ordered(lots, method):
if remaining <= 0:
break
take = min(remaining, lot["sats_remaining"])
if take > 0:
chosen.append((lot, take))
remaining -= take
return chosen
def candidate_lots(market_id, date, conn=None):
"""The lots an account can consume at `date`, oldest acquisition first."""
own = conn is None
if own:
conn = database.connect()
try:
return [_lot_view(lot) for lot in _open_lots(conn, market_id, date)]
finally:
if own:
conn.close()
def create_lots_for(tx, conn=None):
"""Create the lots a transaction brings into its account. Returns their ids."""
own = conn is None
if own:
conn = database.connect()
try:
created = []
tx_type = tx["type"]
fee_sats = tx["fee_sats"] or 0
if tx_type == "trade" and tx["leg_in_sats"]:
sats = tx["leg_in_sats"] - fee_sats
# The kroner that left the account (docs/spec.md section 5.1).
nok_paid = tx["leg_out_ore"]
if nok_paid is None:
nok_paid = tx["leg_in_ore"]
cost_ore = nok_paid or 0
elif tx_type == "income" and tx["leg_in_sats"]:
sats = tx["leg_in_sats"] - fee_sats
cost_ore = tx["value_ore"] or 0
elif tx_type == "gift_received" and tx["leg_in_sats"]:
sats = tx["leg_in_sats"] - fee_sats
cost_ore = tx["value_ore"] or 0
else:
return created
if sats > 0:
created.append(
_insert_lot(conn, tx["market_id"], tx["date"], tx["id"], None, 0, sats,
cost_ore)
)
if own:
conn.commit()
return created
finally:
if own:
conn.close()
def consume(tx, picks=None, conn=None):
"""Consume the lots an outgoing transaction needs. Returns the parts consumed.
Hand-picked lots come first, in the order given, then the remainder by the
lot method. A pick that no longer resolves is skipped rather than fatal, so
a rebuild cannot be blocked by a stale pick.
"""
own = conn is None
if own:
conn = database.connect()
try:
need = _sats_to_consume(tx)
if need <= 0:
return []
parts = []
remaining = need
for pick in picks or []:
if remaining <= 0:
break
lot = _lot_by_origin(conn, tx["market_id"], pick["origin_tx_id"],
pick["origin_seq"])
if lot is None or lot["sats_remaining"] <= 0:
continue
take = min(int(pick["sats"]), remaining, lot["sats_remaining"])
if take > 0:
parts.append(_take(conn, lot, take, tx["id"], auto=0))
remaining -= take
if remaining > 0:
open_lots = _open_lots(conn, tx["market_id"], tx["date"])
for lot, take in pick_for_method(open_lots, remaining, _lot_method(conn)):
parts.append(_take(conn, lot, take, tx["id"], auto=1))
remaining -= take
if remaining > 0:
raise LedgerError(
"Kontoen har ikkje nok sats: det manglar %s sats." % rates.group_int(remaining)
)
if own:
conn.commit()
return parts
finally:
if own:
conn.close()
def holdings_at(date, conn=None):
"""Sats per account at the end of `date`, as {market_id: sats}."""
own = conn is None
if own:
conn = database.connect()
try:
rows = conn.execute(
"SELECT market_id, SUM(sats) AS sats FROM ("
" SELECT l.market_id AS market_id, l.sats_original AS sats FROM lots l"
" JOIN transactions o ON o.id = l.origin_tx_id WHERE o.date <= :date"
" UNION ALL"
" SELECT t.market_id AS market_id, -c.sats AS sats FROM lot_consumptions c"
" JOIN transactions t ON t.id = c.tx_id WHERE t.date <= :date"
") GROUP BY market_id",
{"date": date},
).fetchall()
return {row["market_id"]: row["sats"] for row in rows}
finally:
if own:
conn.close()
def sats_in_market(market_id, date, conn=None):
"""Sats held in one account at the end of `date`."""
own = conn is None
if own:
conn = database.connect()
try:
return holdings_at(date, conn).get(market_id, 0)
finally:
if own:
conn.close()
def kroner_in_market(market_id, date, conn=None):
"""Kroner held in one account at the end of `date` (docs/spec.md section 3.11).
Derived from the kroner legs of the account's own transactions and never
stored, so nothing can drift.
"""
own = conn is None
if own:
conn = database.connect()
try:
row = conn.execute(
"SELECT COALESCE(SUM(COALESCE(leg_in_ore, 0) - COALESCE(leg_out_ore, 0)), 0)"
" AS kroner FROM transactions WHERE market_id = ? AND date <= ?",
(market_id, date),
).fetchone()
return row["kroner"]
finally:
if own:
conn.close()
def sats_on_hand(conn=None):
"""Sats each account holds right now, as the ledger stands."""
own = conn is None
if own:
conn = database.connect()
try:
rows = conn.execute(
"SELECT market_id, SUM(sats_remaining) AS sats FROM lots GROUP BY market_id"
).fetchall()
return {row["market_id"]: row["sats"] for row in rows}
finally:
if own:
conn.close()
def _split_capped(total, weights):
"""Hand out `total` over the weights, in order, never more than a weight."""
cuts = []
left = max(int(total), 0)
for weight in weights:
take = min(left, weight)
cuts.append(take)
left -= take
return cuts
def _mirror_transfer(conn, tx):
"""Create the destination lots of a generated transfer_in.
The sent sats arrive in full, so the mirrored lots cover exactly the sent
amount: one lot per consumed part of the matching transfer_out, in order,
keeping the acquisition date and the cost per sat. The sats of a fee, and
their share of the cost, stay behind with the fee (docs/spec.md section 3.3).
"""
group = tx["transfer_group"]
if not group:
return []
source = conn.execute(
"SELECT id, leg_out_sats FROM transactions"
" WHERE type = 'transfer_out' AND transfer_group = ? ORDER BY id LIMIT 1",
(group,),
).fetchone()
if source is None:
return []
rows = conn.execute(
"SELECT c.sats, c.cost_ore, l.acquired_date, l.id AS source_lot_id"
" FROM lot_consumptions c JOIN lots l ON l.id = c.lot_id"
" WHERE c.tx_id = ? ORDER BY c.id",
(source["id"],),
).fetchall()
parts = [dict(row) for row in rows]
if not parts:
return []
mirrored = _split_capped(
source["leg_out_sats"] or 0, [part["sats"] for part in parts]
)
created = []
for index, part in enumerate(parts):
sats = mirrored[index]
if sats <= 0:
continue
created.append(
_insert_lot(conn, tx["market_id"], part["acquired_date"], tx["id"],
part["source_lot_id"], index, sats,
_cost_for(part["cost_ore"], part["sats"], sats))
)
return created
def _picks(conn, tx_id):
rows = conn.execute(
"SELECT origin_tx_id, origin_seq, sats FROM lot_picks WHERE tx_id = ? ORDER BY id",
(tx_id,),
).fetchall()
return [dict(row) for row in rows]
def _bank_market_id(conn):
"""The account that pays the costs (docs/spec.md section 3.11)."""
row = conn.execute(
"SELECT value FROM settings WHERE key = 'bank_market_id'"
).fetchone()
return int(row["value"]) if row else None
def _move_kroner(balances, tx, bank_id):
"""Track the derived kroner balance and refuse an over-drawn account.
Kroner are never stored, so the balance is carried along the replay. No real
account may go negative; the bank may, and its negative balance is how much
has been drawn from it (docs/spec.md section 3.11).
"""
market_id = tx["market_id"]
if market_id is None:
return
balance = balances.get(market_id, 0)
out_ore = tx["leg_out_ore"] or 0
if out_ore and balance < out_ore and market_id != bank_id:
raise LedgerError(
"Kontoen har ikkje nok kroner: det manglar %s kr."
% rates.kroner_whole(out_ore - balance)
)
balances[market_id] = balance + (tx["leg_in_ore"] or 0) - out_ore
def rebuild(conn=None):
"""Recompute every lot and every consumption from the transactions."""
own = conn is None
if own:
conn = database.connect()
try:
conn.execute("DELETE FROM lot_consumptions")
conn.execute("DELETE FROM lots")
rows = conn.execute("SELECT * FROM transactions ORDER BY date, id").fetchall()
balances = {}
bank_id = _bank_market_id(conn)
for row in rows:
tx = dict(row)
if tx["type"] == "transfer_in" and tx["generated"]:
_mirror_transfer(conn, tx)
else:
create_lots_for(tx, conn)
consume(tx, _picks(conn, tx["id"]), conn)
_move_kroner(balances, tx, bank_id)
conn.commit()
finally:
if own:
conn.close()