diff --git a/ledger.py b/ledger.py new file mode 100644 index 0000000..e3f3dfc --- /dev/null +++ b/ledger.py @@ -0,0 +1,377 @@ +"""The lot ledger of docs/spec.md section 5. + +Every incoming quantity of BTC becomes a lot; every outgoing quantity consumes +sats from one or more lots and records what it consumed. `rebuild()` throws the +lots and the automatic consumptions away and replays every transaction, so the +ledger can never drift away from the transactions. + +Hand-picked lots live in `lot_picks`, which is the owner's input and survives a +rebuild; `lot_consumptions` is derived from it and is therefore rebuilt too. +""" + +from functools import cmp_to_key + +import database +import rates + +SATS_PER_BTC = rates.SATS_PER_BTC +LOT_METHODS = ("FIFO", "LIFO", "HIFO") + +# Types that take sats out of an account. A sats fee is added to the amount +# consumed only where docs/spec.md section 3.2 says it increases the sats used: +# a sale and a spend. For a transfer the fee reduces what arrives instead, and +# for loss, gift given and admin cost the fee is only recorded. +CONSUMING_TYPES = ( + "trade", + "spend", + "loss", + "transfer_out", + "gift_given", + "admin_cost", +) +FEE_ADDS_TO_CONSUMPTION = ("trade", "spend") + + +class LedgerError(Exception): + """Raised when the ledger cannot be written as asked. Message is Nynorsk.""" + + +def _cost_for(cost_ore, sats, take): + """Cost basis of `take` sats out of a lot with `sats` sats and `cost_ore` ore. + + Rounds half up, and gives the whole remainder when the last sat is taken, so + no ore is created or lost. + """ + if take >= sats: + return cost_ore + return (2 * cost_ore * take + sats) // (2 * sats) + + +def _lot_view(lot): + view = dict(lot) + sats = view.get("sats_remaining") or 0 + view["nok_per_btc_ore"] = ( + view["cost_ore_remaining"] * SATS_PER_BTC // sats if sats else 0 + ) + return view + + +def _insert_lot(conn, market_id, acquired_date, origin_tx_id, source_lot_id, origin_seq, + sats, cost_ore): + cursor = conn.execute( + "INSERT INTO lots (market_id, acquired_date, origin_tx_id, source_lot_id, origin_seq," + " sats_original, sats_remaining, cost_ore_original, cost_ore_remaining)" + " VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", + (market_id, acquired_date, origin_tx_id, source_lot_id, origin_seq, sats, sats, + cost_ore, cost_ore), + ) + return cursor.lastrowid + + +def _lot_method(conn): + row = conn.execute("SELECT value FROM settings WHERE key = 'lot_method'").fetchone() + method = row["value"] if row else "FIFO" + return method if method in LOT_METHODS else "FIFO" + + +def _open_lots(conn, market_id, date): + """Lots of an account that existed at `date` and still hold sats.""" + rows = conn.execute( + "SELECT l.id, l.market_id, l.acquired_date, l.origin_tx_id, l.origin_seq," + " l.sats_remaining, l.cost_ore_remaining FROM lots l" + " JOIN transactions o ON o.id = l.origin_tx_id" + " WHERE l.market_id = ? AND o.date <= ? AND l.sats_remaining > 0" + " ORDER BY l.acquired_date, l.id", + (market_id, date), + ).fetchall() + return [dict(row) for row in rows] + + +def _lot_by_origin(conn, market_id, origin_tx_id, origin_seq): + row = conn.execute( + "SELECT id, market_id, acquired_date, origin_tx_id, origin_seq, sats_remaining," + " cost_ore_remaining FROM lots" + " WHERE market_id = ? AND origin_tx_id = ? AND origin_seq = ?", + (market_id, origin_tx_id, origin_seq), + ).fetchone() + return dict(row) if row else None + + +def _sats_to_consume(tx): + """How many sats a transaction takes out of its account.""" + if tx["type"] not in CONSUMING_TYPES: + return 0 + sats = tx["leg_out_sats"] or 0 + if tx["type"] in FEE_ADDS_TO_CONSUMPTION: + sats += tx["fee_sats"] or 0 + return sats + + +def _compare_hifo(left, right): + left_rate = left["cost_ore_remaining"] * right["sats_remaining"] + right_rate = right["cost_ore_remaining"] * left["sats_remaining"] + if left_rate != right_rate: + return -1 if left_rate > right_rate else 1 + if left["acquired_date"] != right["acquired_date"]: + return -1 if left["acquired_date"] < right["acquired_date"] else 1 + return left["id"] - right["id"] + + +def _ordered(lots, method): + if method == "LIFO": + return sorted(lots, key=lambda lot: (lot["acquired_date"], lot["id"]), reverse=True) + if method == "HIFO": + return sorted(lots, key=cmp_to_key(_compare_hifo)) + return sorted(lots, key=lambda lot: (lot["acquired_date"], lot["id"])) + + +def _take(conn, lot, take, tx_id, auto): + cost_ore = _cost_for(lot["cost_ore_remaining"], lot["sats_remaining"], take) + conn.execute( + "UPDATE lots SET sats_remaining = sats_remaining - ?," + " cost_ore_remaining = cost_ore_remaining - ? WHERE id = ?", + (take, cost_ore, lot["id"]), + ) + conn.execute( + "INSERT INTO lot_consumptions (tx_id, lot_id, sats, cost_ore, auto)" + " VALUES (?, ?, ?, ?, ?)", + (tx_id, lot["id"], take, cost_ore, auto), + ) + lot["sats_remaining"] -= take + lot["cost_ore_remaining"] -= cost_ore + return {"lot_id": lot["id"], "sats": take, "cost_ore": cost_ore, "auto": auto} + + +def pick_for_method(lots, sats, method): + """Choose lots for `sats`, in the order the method prescribes. + + Returns a list of (lot, sats) pairs covering as much as the lots hold, so a + caller can tell whether the account is short. + """ + chosen = [] + remaining = int(sats) + for lot in _ordered(lots, method): + if remaining <= 0: + break + take = min(remaining, lot["sats_remaining"]) + if take > 0: + chosen.append((lot, take)) + remaining -= take + return chosen + + +def candidate_lots(market_id, date, conn=None): + """The lots an account can consume at `date`, oldest acquisition first.""" + own = conn is None + if own: + conn = database.connect() + try: + return [_lot_view(lot) for lot in _open_lots(conn, market_id, date)] + finally: + if own: + conn.close() + + +def create_lots_for(tx, conn=None): + """Create the lots a transaction brings into its account. Returns their ids.""" + own = conn is None + if own: + conn = database.connect() + try: + created = [] + tx_type = tx["type"] + fee_sats = tx["fee_sats"] or 0 + fee_ore = tx["fee_ore"] or 0 + if tx_type == "trade" and tx["leg_in_sats"]: + sats = tx["leg_in_sats"] - fee_sats + cost_ore = (tx["leg_in_ore"] or 0) + fee_ore + elif tx_type == "income" and tx["leg_in_sats"]: + sats = tx["leg_in_sats"] - fee_sats + cost_ore = max((tx["value_ore"] or 0) - fee_ore, 0) + elif tx_type == "gift_received" and tx["leg_in_sats"]: + sats = tx["leg_in_sats"] - fee_sats + cost_ore = (tx["value_ore"] or 0) + fee_ore + else: + return created + if sats > 0: + created.append( + _insert_lot(conn, tx["market_id"], tx["date"], tx["id"], None, 0, sats, + cost_ore) + ) + if own: + conn.commit() + return created + finally: + if own: + conn.close() + + +def consume(tx, picks=None, conn=None): + """Consume the lots an outgoing transaction needs. Returns the parts consumed. + + Hand-picked lots come first, in the order given, then the remainder by the + lot method. A pick that no longer resolves is skipped rather than fatal, so + a rebuild cannot be blocked by a stale pick. + """ + own = conn is None + if own: + conn = database.connect() + try: + need = _sats_to_consume(tx) + if need <= 0: + return [] + parts = [] + remaining = need + for pick in picks or []: + if remaining <= 0: + break + lot = _lot_by_origin(conn, tx["market_id"], pick["origin_tx_id"], + pick["origin_seq"]) + if lot is None or lot["sats_remaining"] <= 0: + continue + take = min(int(pick["sats"]), remaining, lot["sats_remaining"]) + if take > 0: + parts.append(_take(conn, lot, take, tx["id"], auto=0)) + remaining -= take + if remaining > 0: + open_lots = _open_lots(conn, tx["market_id"], tx["date"]) + for lot, take in pick_for_method(open_lots, remaining, _lot_method(conn)): + parts.append(_take(conn, lot, take, tx["id"], auto=1)) + remaining -= take + if remaining > 0: + raise LedgerError( + "Kontoen har ikkje nok sats: det manglar %s sats." % rates.group_int(remaining) + ) + if own: + conn.commit() + return parts + finally: + if own: + conn.close() + + +def holdings_at(date, conn=None): + """Sats per account at the end of `date`, as {market_id: sats}.""" + own = conn is None + if own: + conn = database.connect() + try: + rows = conn.execute( + "SELECT market_id, SUM(sats) AS sats FROM (" + " SELECT l.market_id AS market_id, l.sats_original AS sats FROM lots l" + " JOIN transactions o ON o.id = l.origin_tx_id WHERE o.date <= :date" + " UNION ALL" + " SELECT t.market_id AS market_id, -c.sats AS sats FROM lot_consumptions c" + " JOIN transactions t ON t.id = c.tx_id WHERE t.date <= :date" + ") GROUP BY market_id", + {"date": date}, + ).fetchall() + return {row["market_id"]: row["sats"] for row in rows} + finally: + if own: + conn.close() + + +def sats_in_market(market_id, date, conn=None): + """Sats held in one account at the end of `date`.""" + own = conn is None + if own: + conn = database.connect() + try: + return holdings_at(date, conn).get(market_id, 0) + finally: + if own: + conn.close() + + +def _split_capped(total, weights): + """Hand out `total` over the weights, in order, never more than a weight.""" + cuts = [] + left = max(int(total), 0) + for weight in weights: + take = min(left, weight) + cuts.append(take) + left -= take + return cuts + + +def _split_even(total, weights): + """Hand out `total` over the weights so that the parts sum to exactly `total`.""" + total = int(total) + weight_sum = sum(weights) + if total <= 0 or not weights or weight_sum <= 0: + return [0] * len(weights) + cuts = [total * weight // weight_sum for weight in weights] + cuts[-1] = total - sum(cuts[:-1]) + return cuts + + +def _mirror_transfer(conn, tx): + """Create the destination lots of a generated transfer_in. + + One lot per consumed part of the matching transfer_out, keeping the original + acquisition date and cost. The sats fee lowers what arrives and the NOK fee + raises the cost basis. + """ + group = tx["transfer_group"] + if not group: + return [] + source = conn.execute( + "SELECT id FROM transactions WHERE type = 'transfer_out' AND transfer_group = ?" + " ORDER BY id LIMIT 1", + (group,), + ).fetchone() + if source is None: + return [] + rows = conn.execute( + "SELECT c.sats, c.cost_ore, l.acquired_date, l.id AS source_lot_id" + " FROM lot_consumptions c JOIN lots l ON l.id = c.lot_id" + " WHERE c.tx_id = ? ORDER BY c.id", + (source["id"],), + ).fetchall() + parts = [dict(row) for row in rows] + if not parts: + return [] + weights = [part["sats"] for part in parts] + sats_cuts = _split_capped(tx["fee_sats"] or 0, weights) + ore_cuts = _split_even(tx["fee_ore"] or 0, weights) + created = [] + for index, part in enumerate(parts): + sats = part["sats"] - sats_cuts[index] + if sats <= 0: + continue + created.append( + _insert_lot(conn, tx["market_id"], part["acquired_date"], tx["id"], + part["source_lot_id"], index, sats, part["cost_ore"] + ore_cuts[index]) + ) + return created + + +def _picks(conn, tx_id): + rows = conn.execute( + "SELECT origin_tx_id, origin_seq, sats FROM lot_picks WHERE tx_id = ? ORDER BY id", + (tx_id,), + ).fetchall() + return [dict(row) for row in rows] + + +def rebuild(conn=None): + """Recompute every lot and every consumption from the transactions.""" + own = conn is None + if own: + conn = database.connect() + try: + conn.execute("DELETE FROM lot_consumptions") + conn.execute("DELETE FROM lots") + rows = conn.execute("SELECT * FROM transactions ORDER BY date, id").fetchall() + for row in rows: + tx = dict(row) + if tx["type"] == "transfer_in" and tx["generated"]: + _mirror_transfer(conn, tx) + else: + create_lots_for(tx, conn) + consume(tx, _picks(conn, tx["id"]), conn) + conn.commit() + finally: + if own: + conn.close()