"""The lot ledger of docs/spec.md section 5. Every incoming quantity of BTC becomes a lot; every outgoing quantity consumes sats from one or more lots and records what it consumed. `rebuild()` throws the lots and the automatic consumptions away and replays every transaction, so the ledger can never drift away from the transactions. Hand-picked lots live in `lot_picks`, which is the owner's input and survives a rebuild; `lot_consumptions` is derived from it and is therefore rebuilt too. """ from functools import cmp_to_key import database import rates SATS_PER_BTC = rates.SATS_PER_BTC LOT_METHODS = ("FIFO", "LIFO", "HIFO") # Types that take sats out of an account. A sats fee is added to the amount # consumed only where docs/spec.md section 3.2 says it increases the sats used: # a sale and a spend. For a transfer the fee reduces what arrives instead, and # for a loss and a gift given it is only recorded. admin_cost is not here at # all: it stands on its own and changes no holdings (section 3.10). CONSUMING_TYPES = ( "trade", "spend", "loss", "transfer_out", "gift_given", ) FEE_ADDS_TO_CONSUMPTION = ("trade", "spend") class LedgerError(Exception): """Raised when the ledger cannot be written as asked. Message is Nynorsk.""" def _cost_for(cost_ore, sats, take): """Cost basis of `take` sats out of a lot with `sats` sats and `cost_ore` ore. Rounds half up, and gives the whole remainder when the last sat is taken, so no ore is created or lost. """ if take >= sats: return cost_ore return (2 * cost_ore * take + sats) // (2 * sats) def _lot_view(lot): view = dict(lot) sats = view.get("sats_remaining") or 0 view["btc_price_ore"] = ( view["cost_ore_remaining"] * SATS_PER_BTC // sats if sats else 0 ) return view def _insert_lot(conn, market_id, acquired_date, origin_tx_id, source_lot_id, origin_seq, sats, cost_ore): cursor = conn.execute( "INSERT INTO lots (market_id, acquired_date, origin_tx_id, source_lot_id, origin_seq," " sats_original, sats_remaining, cost_ore_original, cost_ore_remaining)" " VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", (market_id, acquired_date, origin_tx_id, source_lot_id, origin_seq, sats, sats, cost_ore, cost_ore), ) return cursor.lastrowid def _lot_method(conn): row = conn.execute("SELECT value FROM settings WHERE key = 'lot_method'").fetchone() method = row["value"] if row else "FIFO" return method if method in LOT_METHODS else "FIFO" def _open_lots(conn, market_id, date): """Lots of an account that existed at `date` and still hold sats.""" rows = conn.execute( "SELECT l.id, l.market_id, l.acquired_date, l.origin_tx_id, l.origin_seq," " l.sats_remaining, l.cost_ore_remaining FROM lots l" " JOIN transactions o ON o.id = l.origin_tx_id" " WHERE l.market_id = ? AND o.date <= ? AND l.sats_remaining > 0" " ORDER BY l.acquired_date, l.id", (market_id, date), ).fetchall() return [dict(row) for row in rows] def _lot_by_origin(conn, market_id, origin_tx_id, origin_seq): row = conn.execute( "SELECT id, market_id, acquired_date, origin_tx_id, origin_seq, sats_remaining," " cost_ore_remaining FROM lots" " WHERE market_id = ? AND origin_tx_id = ? AND origin_seq = ?", (market_id, origin_tx_id, origin_seq), ).fetchone() return dict(row) if row else None def _sats_to_consume(tx): """How many sats a transaction takes out of its account. A trade consumes only when BTC leaves the account (a sale). A buy carries its sats fee on the incoming leg, where it lowers the sats received, so it must not consume anything. """ if tx["type"] not in CONSUMING_TYPES: return 0 sats = tx["leg_out_sats"] or 0 if sats <= 0: return 0 if tx["type"] in FEE_ADDS_TO_CONSUMPTION: sats += tx["fee_sats"] or 0 return sats def _compare_hifo(left, right): left_rate = left["cost_ore_remaining"] * right["sats_remaining"] right_rate = right["cost_ore_remaining"] * left["sats_remaining"] if left_rate != right_rate: return -1 if left_rate > right_rate else 1 if left["acquired_date"] != right["acquired_date"]: return -1 if left["acquired_date"] < right["acquired_date"] else 1 return left["id"] - right["id"] def _ordered(lots, method): if method == "LIFO": return sorted(lots, key=lambda lot: (lot["acquired_date"], lot["id"]), reverse=True) if method == "HIFO": return sorted(lots, key=cmp_to_key(_compare_hifo)) return sorted(lots, key=lambda lot: (lot["acquired_date"], lot["id"])) def _take(conn, lot, take, tx_id, auto): cost_ore = _cost_for(lot["cost_ore_remaining"], lot["sats_remaining"], take) conn.execute( "UPDATE lots SET sats_remaining = sats_remaining - ?," " cost_ore_remaining = cost_ore_remaining - ? WHERE id = ?", (take, cost_ore, lot["id"]), ) conn.execute( "INSERT INTO lot_consumptions (tx_id, lot_id, sats, cost_ore, auto)" " VALUES (?, ?, ?, ?, ?)", (tx_id, lot["id"], take, cost_ore, auto), ) lot["sats_remaining"] -= take lot["cost_ore_remaining"] -= cost_ore return {"lot_id": lot["id"], "sats": take, "cost_ore": cost_ore, "auto": auto} def pick_for_method(lots, sats, method): """Choose lots for `sats`, in the order the method prescribes. Returns a list of (lot, sats) pairs covering as much as the lots hold, so a caller can tell whether the account is short. """ chosen = [] remaining = int(sats) for lot in _ordered(lots, method): if remaining <= 0: break take = min(remaining, lot["sats_remaining"]) if take > 0: chosen.append((lot, take)) remaining -= take return chosen def candidate_lots(market_id, date, conn=None): """The lots an account can consume at `date`, oldest acquisition first.""" own = conn is None if own: conn = database.connect() try: return [_lot_view(lot) for lot in _open_lots(conn, market_id, date)] finally: if own: conn.close() def create_lots_for(tx, conn=None): """Create the lots a transaction brings into its account. Returns their ids.""" own = conn is None if own: conn = database.connect() try: created = [] tx_type = tx["type"] fee_sats = tx["fee_sats"] or 0 fee_ore = tx["fee_ore"] or 0 if tx_type == "trade" and tx["leg_in_sats"]: sats = tx["leg_in_sats"] - fee_sats # The NOK that left the account. docs/spec.md section 5.1 says # leg_in_ore, but a leg carries one currency only (the CHECK of # section 4), so a buy keeps it on leg_out_ore. nok_paid = tx["leg_out_ore"] if nok_paid is None: nok_paid = tx["leg_in_ore"] cost_ore = (nok_paid or 0) + fee_ore elif tx_type == "income" and tx["leg_in_sats"]: sats = tx["leg_in_sats"] - fee_sats cost_ore = max((tx["value_ore"] or 0) - fee_ore, 0) elif tx_type == "gift_received" and tx["leg_in_sats"]: sats = tx["leg_in_sats"] - fee_sats cost_ore = (tx["value_ore"] or 0) + fee_ore else: return created if sats > 0: created.append( _insert_lot(conn, tx["market_id"], tx["date"], tx["id"], None, 0, sats, cost_ore) ) if own: conn.commit() return created finally: if own: conn.close() def consume(tx, picks=None, conn=None): """Consume the lots an outgoing transaction needs. Returns the parts consumed. Hand-picked lots come first, in the order given, then the remainder by the lot method. A pick that no longer resolves is skipped rather than fatal, so a rebuild cannot be blocked by a stale pick. """ own = conn is None if own: conn = database.connect() try: need = _sats_to_consume(tx) if need <= 0: return [] parts = [] remaining = need for pick in picks or []: if remaining <= 0: break lot = _lot_by_origin(conn, tx["market_id"], pick["origin_tx_id"], pick["origin_seq"]) if lot is None or lot["sats_remaining"] <= 0: continue take = min(int(pick["sats"]), remaining, lot["sats_remaining"]) if take > 0: parts.append(_take(conn, lot, take, tx["id"], auto=0)) remaining -= take if remaining > 0: open_lots = _open_lots(conn, tx["market_id"], tx["date"]) for lot, take in pick_for_method(open_lots, remaining, _lot_method(conn)): parts.append(_take(conn, lot, take, tx["id"], auto=1)) remaining -= take if remaining > 0: raise LedgerError( "Kontoen har ikkje nok sats: det manglar %s sats." % rates.group_int(remaining) ) if own: conn.commit() return parts finally: if own: conn.close() def holdings_at(date, conn=None): """Sats per account at the end of `date`, as {market_id: sats}.""" own = conn is None if own: conn = database.connect() try: rows = conn.execute( "SELECT market_id, SUM(sats) AS sats FROM (" " SELECT l.market_id AS market_id, l.sats_original AS sats FROM lots l" " JOIN transactions o ON o.id = l.origin_tx_id WHERE o.date <= :date" " UNION ALL" " SELECT t.market_id AS market_id, -c.sats AS sats FROM lot_consumptions c" " JOIN transactions t ON t.id = c.tx_id WHERE t.date <= :date" ") GROUP BY market_id", {"date": date}, ).fetchall() return {row["market_id"]: row["sats"] for row in rows} finally: if own: conn.close() def sats_in_market(market_id, date, conn=None): """Sats held in one account at the end of `date`.""" own = conn is None if own: conn = database.connect() try: return holdings_at(date, conn).get(market_id, 0) finally: if own: conn.close() def sats_on_hand(conn=None): """Sats each account holds right now, as the ledger stands.""" own = conn is None if own: conn = database.connect() try: rows = conn.execute( "SELECT market_id, SUM(sats_remaining) AS sats FROM lots GROUP BY market_id" ).fetchall() return {row["market_id"]: row["sats"] for row in rows} finally: if own: conn.close() def _split_capped(total, weights): """Hand out `total` over the weights, in order, never more than a weight.""" cuts = [] left = max(int(total), 0) for weight in weights: take = min(left, weight) cuts.append(take) left -= take return cuts def _split_even(total, weights): """Hand out `total` over the weights so that the parts sum to exactly `total`.""" total = int(total) weight_sum = sum(weights) if total <= 0 or not weights or weight_sum <= 0: return [0] * len(weights) cuts = [total * weight // weight_sum for weight in weights] cuts[-1] = total - sum(cuts[:-1]) return cuts def _mirror_transfer(conn, tx): """Create the destination lots of a generated transfer_in. One lot per consumed part of the matching transfer_out, keeping the original acquisition date and cost. The sats fee lowers what arrives and the NOK fee raises the cost basis. """ group = tx["transfer_group"] if not group: return [] source = conn.execute( "SELECT id FROM transactions WHERE type = 'transfer_out' AND transfer_group = ?" " ORDER BY id LIMIT 1", (group,), ).fetchone() if source is None: return [] rows = conn.execute( "SELECT c.sats, c.cost_ore, l.acquired_date, l.id AS source_lot_id" " FROM lot_consumptions c JOIN lots l ON l.id = c.lot_id" " WHERE c.tx_id = ? ORDER BY c.id", (source["id"],), ).fetchall() parts = [dict(row) for row in rows] if not parts: return [] weights = [part["sats"] for part in parts] sats_cuts = _split_capped(tx["fee_sats"] or 0, weights) ore_cuts = _split_even(tx["fee_ore"] or 0, weights) created = [] for index, part in enumerate(parts): sats = part["sats"] - sats_cuts[index] if sats <= 0: continue created.append( _insert_lot(conn, tx["market_id"], part["acquired_date"], tx["id"], part["source_lot_id"], index, sats, part["cost_ore"] + ore_cuts[index]) ) return created def _picks(conn, tx_id): rows = conn.execute( "SELECT origin_tx_id, origin_seq, sats FROM lot_picks WHERE tx_id = ? ORDER BY id", (tx_id,), ).fetchall() return [dict(row) for row in rows] def rebuild(conn=None): """Recompute every lot and every consumption from the transactions.""" own = conn is None if own: conn = database.connect() try: conn.execute("DELETE FROM lot_consumptions") conn.execute("DELETE FROM lots") rows = conn.execute("SELECT * FROM transactions ORDER BY date, id").fetchall() for row in rows: tx = dict(row) if tx["type"] == "transfer_in" and tx["generated"]: _mirror_transfer(conn, tx) else: create_lots_for(tx, conn) consume(tx, _picks(conn, tx["id"]), conn) conn.commit() finally: if own: conn.close()