Add ledger.py with lots, consumption, holdings and rebuild
This commit is contained in:
1 parent
961edec209
commit
ea25da1d63
1 file changed
+377
@@ -0,0 +1,377 @@
|
||||
"""The lot ledger of docs/spec.md section 5.
|
||||
|
||||
Every incoming quantity of BTC becomes a lot; every outgoing quantity consumes
|
||||
sats from one or more lots and records what it consumed. `rebuild()` throws the
|
||||
lots and the automatic consumptions away and replays every transaction, so the
|
||||
ledger can never drift away from the transactions.
|
||||
|
||||
Hand-picked lots live in `lot_picks`, which is the owner's input and survives a
|
||||
rebuild; `lot_consumptions` is derived from it and is therefore rebuilt too.
|
||||
"""
|
||||
|
||||
from functools import cmp_to_key
|
||||
|
||||
import database
|
||||
import rates
|
||||
|
||||
SATS_PER_BTC = rates.SATS_PER_BTC
|
||||
LOT_METHODS = ("FIFO", "LIFO", "HIFO")
|
||||
|
||||
# Types that take sats out of an account. A sats fee is added to the amount
|
||||
# consumed only where docs/spec.md section 3.2 says it increases the sats used:
|
||||
# a sale and a spend. For a transfer the fee reduces what arrives instead, and
|
||||
# for loss, gift given and admin cost the fee is only recorded.
|
||||
CONSUMING_TYPES = (
|
||||
"trade",
|
||||
"spend",
|
||||
"loss",
|
||||
"transfer_out",
|
||||
"gift_given",
|
||||
"admin_cost",
|
||||
)
|
||||
FEE_ADDS_TO_CONSUMPTION = ("trade", "spend")
|
||||
|
||||
|
||||
class LedgerError(Exception):
|
||||
"""Raised when the ledger cannot be written as asked. Message is Nynorsk."""
|
||||
|
||||
|
||||
def _cost_for(cost_ore, sats, take):
|
||||
"""Cost basis of `take` sats out of a lot with `sats` sats and `cost_ore` ore.
|
||||
|
||||
Rounds half up, and gives the whole remainder when the last sat is taken, so
|
||||
no ore is created or lost.
|
||||
"""
|
||||
if take >= sats:
|
||||
return cost_ore
|
||||
return (2 * cost_ore * take + sats) // (2 * sats)
|
||||
|
||||
|
||||
def _lot_view(lot):
|
||||
view = dict(lot)
|
||||
sats = view.get("sats_remaining") or 0
|
||||
view["nok_per_btc_ore"] = (
|
||||
view["cost_ore_remaining"] * SATS_PER_BTC // sats if sats else 0
|
||||
)
|
||||
return view
|
||||
|
||||
|
||||
def _insert_lot(conn, market_id, acquired_date, origin_tx_id, source_lot_id, origin_seq,
|
||||
sats, cost_ore):
|
||||
cursor = conn.execute(
|
||||
"INSERT INTO lots (market_id, acquired_date, origin_tx_id, source_lot_id, origin_seq,"
|
||||
" sats_original, sats_remaining, cost_ore_original, cost_ore_remaining)"
|
||||
" VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)",
|
||||
(market_id, acquired_date, origin_tx_id, source_lot_id, origin_seq, sats, sats,
|
||||
cost_ore, cost_ore),
|
||||
)
|
||||
return cursor.lastrowid
|
||||
|
||||
|
||||
def _lot_method(conn):
|
||||
row = conn.execute("SELECT value FROM settings WHERE key = 'lot_method'").fetchone()
|
||||
method = row["value"] if row else "FIFO"
|
||||
return method if method in LOT_METHODS else "FIFO"
|
||||
|
||||
|
||||
def _open_lots(conn, market_id, date):
|
||||
"""Lots of an account that existed at `date` and still hold sats."""
|
||||
rows = conn.execute(
|
||||
"SELECT l.id, l.market_id, l.acquired_date, l.origin_tx_id, l.origin_seq,"
|
||||
" l.sats_remaining, l.cost_ore_remaining FROM lots l"
|
||||
" JOIN transactions o ON o.id = l.origin_tx_id"
|
||||
" WHERE l.market_id = ? AND o.date <= ? AND l.sats_remaining > 0"
|
||||
" ORDER BY l.acquired_date, l.id",
|
||||
(market_id, date),
|
||||
).fetchall()
|
||||
return [dict(row) for row in rows]
|
||||
|
||||
|
||||
def _lot_by_origin(conn, market_id, origin_tx_id, origin_seq):
|
||||
row = conn.execute(
|
||||
"SELECT id, market_id, acquired_date, origin_tx_id, origin_seq, sats_remaining,"
|
||||
" cost_ore_remaining FROM lots"
|
||||
" WHERE market_id = ? AND origin_tx_id = ? AND origin_seq = ?",
|
||||
(market_id, origin_tx_id, origin_seq),
|
||||
).fetchone()
|
||||
return dict(row) if row else None
|
||||
|
||||
|
||||
def _sats_to_consume(tx):
|
||||
"""How many sats a transaction takes out of its account."""
|
||||
if tx["type"] not in CONSUMING_TYPES:
|
||||
return 0
|
||||
sats = tx["leg_out_sats"] or 0
|
||||
if tx["type"] in FEE_ADDS_TO_CONSUMPTION:
|
||||
sats += tx["fee_sats"] or 0
|
||||
return sats
|
||||
|
||||
|
||||
def _compare_hifo(left, right):
|
||||
left_rate = left["cost_ore_remaining"] * right["sats_remaining"]
|
||||
right_rate = right["cost_ore_remaining"] * left["sats_remaining"]
|
||||
if left_rate != right_rate:
|
||||
return -1 if left_rate > right_rate else 1
|
||||
if left["acquired_date"] != right["acquired_date"]:
|
||||
return -1 if left["acquired_date"] < right["acquired_date"] else 1
|
||||
return left["id"] - right["id"]
|
||||
|
||||
|
||||
def _ordered(lots, method):
|
||||
if method == "LIFO":
|
||||
return sorted(lots, key=lambda lot: (lot["acquired_date"], lot["id"]), reverse=True)
|
||||
if method == "HIFO":
|
||||
return sorted(lots, key=cmp_to_key(_compare_hifo))
|
||||
return sorted(lots, key=lambda lot: (lot["acquired_date"], lot["id"]))
|
||||
|
||||
|
||||
def _take(conn, lot, take, tx_id, auto):
|
||||
cost_ore = _cost_for(lot["cost_ore_remaining"], lot["sats_remaining"], take)
|
||||
conn.execute(
|
||||
"UPDATE lots SET sats_remaining = sats_remaining - ?,"
|
||||
" cost_ore_remaining = cost_ore_remaining - ? WHERE id = ?",
|
||||
(take, cost_ore, lot["id"]),
|
||||
)
|
||||
conn.execute(
|
||||
"INSERT INTO lot_consumptions (tx_id, lot_id, sats, cost_ore, auto)"
|
||||
" VALUES (?, ?, ?, ?, ?)",
|
||||
(tx_id, lot["id"], take, cost_ore, auto),
|
||||
)
|
||||
lot["sats_remaining"] -= take
|
||||
lot["cost_ore_remaining"] -= cost_ore
|
||||
return {"lot_id": lot["id"], "sats": take, "cost_ore": cost_ore, "auto": auto}
|
||||
|
||||
|
||||
def pick_for_method(lots, sats, method):
|
||||
"""Choose lots for `sats`, in the order the method prescribes.
|
||||
|
||||
Returns a list of (lot, sats) pairs covering as much as the lots hold, so a
|
||||
caller can tell whether the account is short.
|
||||
"""
|
||||
chosen = []
|
||||
remaining = int(sats)
|
||||
for lot in _ordered(lots, method):
|
||||
if remaining <= 0:
|
||||
break
|
||||
take = min(remaining, lot["sats_remaining"])
|
||||
if take > 0:
|
||||
chosen.append((lot, take))
|
||||
remaining -= take
|
||||
return chosen
|
||||
|
||||
|
||||
def candidate_lots(market_id, date, conn=None):
|
||||
"""The lots an account can consume at `date`, oldest acquisition first."""
|
||||
own = conn is None
|
||||
if own:
|
||||
conn = database.connect()
|
||||
try:
|
||||
return [_lot_view(lot) for lot in _open_lots(conn, market_id, date)]
|
||||
finally:
|
||||
if own:
|
||||
conn.close()
|
||||
|
||||
|
||||
def create_lots_for(tx, conn=None):
|
||||
"""Create the lots a transaction brings into its account. Returns their ids."""
|
||||
own = conn is None
|
||||
if own:
|
||||
conn = database.connect()
|
||||
try:
|
||||
created = []
|
||||
tx_type = tx["type"]
|
||||
fee_sats = tx["fee_sats"] or 0
|
||||
fee_ore = tx["fee_ore"] or 0
|
||||
if tx_type == "trade" and tx["leg_in_sats"]:
|
||||
sats = tx["leg_in_sats"] - fee_sats
|
||||
cost_ore = (tx["leg_in_ore"] or 0) + fee_ore
|
||||
elif tx_type == "income" and tx["leg_in_sats"]:
|
||||
sats = tx["leg_in_sats"] - fee_sats
|
||||
cost_ore = max((tx["value_ore"] or 0) - fee_ore, 0)
|
||||
elif tx_type == "gift_received" and tx["leg_in_sats"]:
|
||||
sats = tx["leg_in_sats"] - fee_sats
|
||||
cost_ore = (tx["value_ore"] or 0) + fee_ore
|
||||
else:
|
||||
return created
|
||||
if sats > 0:
|
||||
created.append(
|
||||
_insert_lot(conn, tx["market_id"], tx["date"], tx["id"], None, 0, sats,
|
||||
cost_ore)
|
||||
)
|
||||
if own:
|
||||
conn.commit()
|
||||
return created
|
||||
finally:
|
||||
if own:
|
||||
conn.close()
|
||||
|
||||
|
||||
def consume(tx, picks=None, conn=None):
|
||||
"""Consume the lots an outgoing transaction needs. Returns the parts consumed.
|
||||
|
||||
Hand-picked lots come first, in the order given, then the remainder by the
|
||||
lot method. A pick that no longer resolves is skipped rather than fatal, so
|
||||
a rebuild cannot be blocked by a stale pick.
|
||||
"""
|
||||
own = conn is None
|
||||
if own:
|
||||
conn = database.connect()
|
||||
try:
|
||||
need = _sats_to_consume(tx)
|
||||
if need <= 0:
|
||||
return []
|
||||
parts = []
|
||||
remaining = need
|
||||
for pick in picks or []:
|
||||
if remaining <= 0:
|
||||
break
|
||||
lot = _lot_by_origin(conn, tx["market_id"], pick["origin_tx_id"],
|
||||
pick["origin_seq"])
|
||||
if lot is None or lot["sats_remaining"] <= 0:
|
||||
continue
|
||||
take = min(int(pick["sats"]), remaining, lot["sats_remaining"])
|
||||
if take > 0:
|
||||
parts.append(_take(conn, lot, take, tx["id"], auto=0))
|
||||
remaining -= take
|
||||
if remaining > 0:
|
||||
open_lots = _open_lots(conn, tx["market_id"], tx["date"])
|
||||
for lot, take in pick_for_method(open_lots, remaining, _lot_method(conn)):
|
||||
parts.append(_take(conn, lot, take, tx["id"], auto=1))
|
||||
remaining -= take
|
||||
if remaining > 0:
|
||||
raise LedgerError(
|
||||
"Kontoen har ikkje nok sats: det manglar %s sats." % rates.group_int(remaining)
|
||||
)
|
||||
if own:
|
||||
conn.commit()
|
||||
return parts
|
||||
finally:
|
||||
if own:
|
||||
conn.close()
|
||||
|
||||
|
||||
def holdings_at(date, conn=None):
|
||||
"""Sats per account at the end of `date`, as {market_id: sats}."""
|
||||
own = conn is None
|
||||
if own:
|
||||
conn = database.connect()
|
||||
try:
|
||||
rows = conn.execute(
|
||||
"SELECT market_id, SUM(sats) AS sats FROM ("
|
||||
" SELECT l.market_id AS market_id, l.sats_original AS sats FROM lots l"
|
||||
" JOIN transactions o ON o.id = l.origin_tx_id WHERE o.date <= :date"
|
||||
" UNION ALL"
|
||||
" SELECT t.market_id AS market_id, -c.sats AS sats FROM lot_consumptions c"
|
||||
" JOIN transactions t ON t.id = c.tx_id WHERE t.date <= :date"
|
||||
") GROUP BY market_id",
|
||||
{"date": date},
|
||||
).fetchall()
|
||||
return {row["market_id"]: row["sats"] for row in rows}
|
||||
finally:
|
||||
if own:
|
||||
conn.close()
|
||||
|
||||
|
||||
def sats_in_market(market_id, date, conn=None):
|
||||
"""Sats held in one account at the end of `date`."""
|
||||
own = conn is None
|
||||
if own:
|
||||
conn = database.connect()
|
||||
try:
|
||||
return holdings_at(date, conn).get(market_id, 0)
|
||||
finally:
|
||||
if own:
|
||||
conn.close()
|
||||
|
||||
|
||||
def _split_capped(total, weights):
|
||||
"""Hand out `total` over the weights, in order, never more than a weight."""
|
||||
cuts = []
|
||||
left = max(int(total), 0)
|
||||
for weight in weights:
|
||||
take = min(left, weight)
|
||||
cuts.append(take)
|
||||
left -= take
|
||||
return cuts
|
||||
|
||||
|
||||
def _split_even(total, weights):
|
||||
"""Hand out `total` over the weights so that the parts sum to exactly `total`."""
|
||||
total = int(total)
|
||||
weight_sum = sum(weights)
|
||||
if total <= 0 or not weights or weight_sum <= 0:
|
||||
return [0] * len(weights)
|
||||
cuts = [total * weight // weight_sum for weight in weights]
|
||||
cuts[-1] = total - sum(cuts[:-1])
|
||||
return cuts
|
||||
|
||||
|
||||
def _mirror_transfer(conn, tx):
|
||||
"""Create the destination lots of a generated transfer_in.
|
||||
|
||||
One lot per consumed part of the matching transfer_out, keeping the original
|
||||
acquisition date and cost. The sats fee lowers what arrives and the NOK fee
|
||||
raises the cost basis.
|
||||
"""
|
||||
group = tx["transfer_group"]
|
||||
if not group:
|
||||
return []
|
||||
source = conn.execute(
|
||||
"SELECT id FROM transactions WHERE type = 'transfer_out' AND transfer_group = ?"
|
||||
" ORDER BY id LIMIT 1",
|
||||
(group,),
|
||||
).fetchone()
|
||||
if source is None:
|
||||
return []
|
||||
rows = conn.execute(
|
||||
"SELECT c.sats, c.cost_ore, l.acquired_date, l.id AS source_lot_id"
|
||||
" FROM lot_consumptions c JOIN lots l ON l.id = c.lot_id"
|
||||
" WHERE c.tx_id = ? ORDER BY c.id",
|
||||
(source["id"],),
|
||||
).fetchall()
|
||||
parts = [dict(row) for row in rows]
|
||||
if not parts:
|
||||
return []
|
||||
weights = [part["sats"] for part in parts]
|
||||
sats_cuts = _split_capped(tx["fee_sats"] or 0, weights)
|
||||
ore_cuts = _split_even(tx["fee_ore"] or 0, weights)
|
||||
created = []
|
||||
for index, part in enumerate(parts):
|
||||
sats = part["sats"] - sats_cuts[index]
|
||||
if sats <= 0:
|
||||
continue
|
||||
created.append(
|
||||
_insert_lot(conn, tx["market_id"], part["acquired_date"], tx["id"],
|
||||
part["source_lot_id"], index, sats, part["cost_ore"] + ore_cuts[index])
|
||||
)
|
||||
return created
|
||||
|
||||
|
||||
def _picks(conn, tx_id):
|
||||
rows = conn.execute(
|
||||
"SELECT origin_tx_id, origin_seq, sats FROM lot_picks WHERE tx_id = ? ORDER BY id",
|
||||
(tx_id,),
|
||||
).fetchall()
|
||||
return [dict(row) for row in rows]
|
||||
|
||||
|
||||
def rebuild(conn=None):
|
||||
"""Recompute every lot and every consumption from the transactions."""
|
||||
own = conn is None
|
||||
if own:
|
||||
conn = database.connect()
|
||||
try:
|
||||
conn.execute("DELETE FROM lot_consumptions")
|
||||
conn.execute("DELETE FROM lots")
|
||||
rows = conn.execute("SELECT * FROM transactions ORDER BY date, id").fetchall()
|
||||
for row in rows:
|
||||
tx = dict(row)
|
||||
if tx["type"] == "transfer_in" and tx["generated"]:
|
||||
_mirror_transfer(conn, tx)
|
||||
else:
|
||||
create_lots_for(tx, conn)
|
||||
consume(tx, _picks(conn, tx["id"]), conn)
|
||||
conn.commit()
|
||||
finally:
|
||||
if own:
|
||||
conn.close()
|
||||
Reference in new issue
Block a user