merge main into yahoo branch

README.md

@@ -63,6 +63,12 @@ symbols:
   RIO: RIO.L
   SAN: SAN.PA
 
+# Renames deal with stocks that have changed their symbol
+# for example by becoming a new company or by being delisted
+renames:
+  GPS: GAP
+  TUP: TUPBQ
+
 # Pies allows you split your aggregation into multiple CSVs
 # uncomment to use
 #pies:
@@ -95,6 +101,9 @@ The default currency is set to EUR, but you can use the dropdown to change it to
 
 ## Changelog
 
+- v0.2.7 - 2024-10-15 - Handle events related to T212 Card + Withdrawals
+- v0.2.6 - 2024-10-05 - Added "Renames" config for renamed/delisted stocks
+- v0.2.5 - 2024-02-08 - Added "Lending interest" field
 - v0.2.4 - 2023-08-01 - Fix stock splits for stocks that are untouched + Update dependencies
 - v0.2.3 - 2023-07-31 - Skip "Currency conversion" action + Deposits has changed fields
 - v0.2.2 - 2023-07-24 - Currency name not in headers anymore

cmd/aggregator/main.go

@@ -4,6 +4,7 @@ import (
 	"encoding/json"
 	"flag"
 	"fmt"
+	"math"
 	"os"
 	"strings"
 
@@ -72,23 +73,26 @@ func main() {
 		"pies":      len(cfg.Pies),
 		"splits":    len(cfg.Splits),
 		"symbols":   len(cfg.Symbols),
+		"renames":   len(cfg.Renames),
 	}).Info("Starting process.")
 
 	// loop through directory and find csv files
 	events := trading212.Collect(cfg.Input)
 
 	// aggregate events via Trading212 algorithm
-	stocks, totals := trading212.Aggregate(cfg.Splits, events)
+	stocks, totals := trading212.Aggregate(cfg.Splits, cfg.Renames, events)
 
 	log.WithFields(logrus.Fields{
-		"deposits":  totals.Deposits,
-		"invested":  totals.Invested,
-		"realized":  totals.Realized,
-		"dividends": totals.Dividends,
-		"fees":      totals.Fees,
-		"taxes":     totals.Taxes,
-		"cash":      totals.Cash,
-		"interest":  totals.Interest,
+		"deposits":            totals.Deposits,
+		"invested":            totals.Invested,
+		"realized":            totals.Realized,
+		"realized-with-costs": ceilFloat(totals.Realized-totals.Fees-totals.Taxes, 2),
+		"dividends":           totals.Dividends,
+		"fees":                totals.Fees,
+		"taxes":               totals.Taxes,
+		"cash":                totals.Cash,
+		"interest":            totals.Interest,
+		"withdrawals":         totals.Withdrawals,
 	}).Info("Completed aggregation.")
 
 	// write output
@@ -207,3 +211,8 @@ func writeOutputJSON(cfg fin.Config, outputName string, output interface{}) stri
 
 	return fn
 }
+
+func ceilFloat(f float64, precision int) float64 {
+	d := math.Pow(10, float64(precision))
+	return math.Ceil(f*d) / d
+}

config.go

@@ -17,6 +17,7 @@ type Config struct {
 	PieOnly  string            `yaml:"pie-only"`
 	Splits   []Splits          `yaml:"splits"`
 	Symbols  map[string]string `yaml:"symbols"`
+	Renames  map[string]string `yaml:"renames"`
 	Pies     []struct {
 		Name    string   `yaml:"name"`
 		Symbols []string `yaml:"symbols"`

config.yaml

@@ -21,6 +21,12 @@ symbols:
   RIO: RIO.L
   SAN: SAN.PA
 
+# Renames deal with stocks that have changed their symbol
+# for example by becoming a new company or by being delisted
+renames:
+  GPS: GAP
+  TUP: TUPBQ
+
 # Pies allows you split your aggregation into multiple CSVs
 # uncomment to use
 #pies:

models.go

@@ -18,12 +18,13 @@ type Aggregate struct {
 }
 
 type Totals struct {
-	Deposits  float64 // the money you deposited
-	Invested  float64 // the money you have invested, minus fees
-	Realized  float64 // gains you have realized by selling
-	Dividends float64 // amount of money you received from dividends
-	Fees      float64 // fees you paid
-	Cash      float64 // cash left in your portfolio
-	Taxes     float64 // taxes withheld from dividends
-	Interest  float64 // interest you received on cash
+	Deposits    float64 // the money you deposited
+	Invested    float64 // the money you have invested, minus fees
+	Realized    float64 // gains you have realized by selling
+	Dividends   float64 // amount of money you received from dividends
+	Fees        float64 // fees you paid
+	Withdrawals float64 // costs that are taken away from your cash i.e. Card debit
+	Cash        float64 // cash left in your portfolio
+	Taxes       float64 // taxes withheld from dividends
+	Interest    float64 // interest you received on cash, lent shares or card cashback
 }

testdata/gbp/trading212.csv

@@ -8,5 +8,5 @@ Deposit,2021-09-07 13:43:10,,,,,,,,,1000.00,,,1001.40,1.40,,"Transaction ID: xxx
 Market buy,2021-09-27 13:19:13,US02079K1079,ABEC,"Google",0.0041253700,2424.00,EUR,1.00000,,10.00,,,,,,,EOF5,,
 Dividend (Ordinary),2021-09-30 11:15:32,US5949181045,MSFT,"Microsoft",0.2709950000,0.48,USD,Not available,,0.11,0.02,USD,,,,,,,
 Market buy,2022-03-07 16:10:26,FR0000120578,SAN,"Sanofi",0.1117960000,89.18,EUR,1.00000,,10.00,,,,,,,EOF6,,0.03
-Market buy,2022-07-29 14:28:17,US02079K1079,ABEC,"Alphabet (Class C)",2.2887315000,113.60,EUR,1.00000,,260.00,,,,,,,EOF7,,
+Market buy,2022-07-29 14:28:17,US02079K1079,GOOG,"Alphabet (Class C)",2.2887315000,113.60,EUR,1.00000,,260.00,,,,,,,EOF7,,
 

testdata/multiple/3-wrong-order.csv

@@ -1,4 +1,4 @@
 Action,Time,ISIN,Ticker,Name,No. of shares,Price / share,Currency (Price / share),Exchange rate,Result,Total,Withholding tax,Currency (Withholding tax),Charge amount,Deposit fee,Stamp duty reserve tax,Notes,ID,Currency conversion fee,French transaction tax
 Market buy,2022-03-07 16:10:26,FR0000120578,SAN,"Sanofi",0.1117960000,89.18,EUR,1.00000,,10.00,,,,,,,EOF6,,0.03
-Market buy,2022-07-29 14:28:17,US02079K1079,ABEC,"Alphabet (Class C)",2.2887315000,113.60,EUR,1.00000,,260.00,,,,,,,EOF7,,
+Market buy,2022-07-29 14:28:17,US02079K1079,GOOG,"Alphabet (Class C)",2.2887315000,113.60,EUR,1.00000,,260.00,,,,,,,EOF7,,
 

testdata/trading212.csv

@@ -8,5 +8,5 @@ Deposit,2021-09-07 13:43:10,,,,,,,,,1000.00,,,1001.40,1.40,,"Transaction ID: xxx
 Market buy,2021-09-27 13:19:13,US02079K1079,ABEC,"Google",0.0041253700,2424.00,EUR,1.00000,,10.00,,,,,,,EOF5,,
 Dividend (Ordinary),2021-09-30 11:15:32,US5949181045,MSFT,"Microsoft",0.2709950000,0.48,USD,Not available,,0.11,0.02,USD,,,,,,,
 Market buy,2022-03-07 16:10:26,FR0000120578,SAN,"Sanofi",0.1117960000,89.18,EUR,1.00000,,10.00,,,,,,,EOF6,,0.03
-Market buy,2022-07-29 14:28:17,US02079K1079,ABEC,"Alphabet (Class C)",2.2887315000,113.60,EUR,1.00000,,260.00,,,,,,,EOF7,,
+Market buy,2022-07-29 14:28:17,US02079K1079,GOOG,"Alphabet (Class C)",2.2887315000,113.60,EUR,1.00000,,260.00,,,,,,,EOF7,,
 

trading212/aggregate.go

@@ -9,41 +9,58 @@ import (
 
 // Aggregate takes a map of events and aggregates them into a map of stocks and totals,
 // based on the Trading212 algorithm, along with stock splits.
-func Aggregate(splits []fin.Splits, events []TradeEvent) ([]fin.Aggregate, fin.Totals) {
+func Aggregate(splits []fin.Splits, renames map[string]string, events []TradeEvent) ([]fin.Aggregate, fin.Totals) {
 	var stocks = make(map[string]fin.Aggregate)
 	var stockNames []string
 	var totals fin.Totals
 	for _, e := range events {
-		// skip currency conversions
-		if e.Action == "Currency conversion" {
+		// skip every event we don't deal with
+		if e.IsSkippable() {
 			continue
 		}
-		// handle deposits
-		if e.Action == "Deposit" {
+		// handle deposits or additions
+		if e.Action == "Deposit" || e.Action == "Spending cashback" {
 			totals.Deposits += e.Total
+			totals.Withdrawals -= e.DepositFee
 			continue
 		}
-
 		// handle interest
-		if e.Action == "Interest on cash" || e.Action == "Lending interest" {
+		if e.IsInterest() {
 			totals.Interest += e.Total
 			continue
 		}
+		// handle money withdrawl
+		if e.IsMoneyWithdrawal() {
+			// we subtract the total, because it's stored as a negative number
+			totals.Withdrawals -= e.Total
+			continue
+		}
+
+		// if no action matches, but our symbol is empty, we continue too
+		if e.TickerSymbol == "" {
+			continue
+		}
+
+		// handle renamed stock symbols
+		symbol := e.TickerSymbol
+		if rn, ok := renames[symbol]; ok {
+			symbol = rn
+		}
 
 		// create entry if it doesn't exist
-		if _, ok := stocks[e.TickerSymbol]; !ok {
-			stocks[e.TickerSymbol] = fin.Aggregate{
-				Symbol: e.TickerSymbol,
+		if _, ok := stocks[symbol]; !ok {
+			stocks[symbol] = fin.Aggregate{
+				Symbol: symbol,
 			}
-			stockNames = append(stockNames, e.TickerSymbol)
+			stockNames = append(stockNames, symbol)
 		}
 
 		// calculate changes
-		a := stocks[e.TickerSymbol]
+		a := stocks[symbol]
 
 		// did a stock split happen today
 		for _, split := range splits {
-			if split.Symbol == e.TickerSymbol &&
+			if split.Symbol == symbol &&
 				split.Date > a.LastUpdate.Format("2006-01-02") && split.Date <= e.Time.Format("2006-01-02") {
 				a.ShareCount = a.ShareCount * split.Ratio
 			}
@@ -74,7 +91,8 @@ func Aggregate(splits []fin.Splits, events []TradeEvent) ([]fin.Aggregate, fin.T
 		totals.Taxes += e.Tax
 
 		// update totals
-		if a.ShareCount > 0 {
+		if floorFloat(a.ShareCount, 4) > 0 {
+			// if it's practically zero, reset it (float comparison issues)
 			a.AvgPrice = a.ShareCost / a.ShareCount
 		} else {
 			// during this event everything was sold
@@ -107,7 +125,7 @@ func Aggregate(splits []fin.Splits, events []TradeEvent) ([]fin.Aggregate, fin.T
 
 	// calculate cash left over in portfolio
 	moneyGained := totals.Deposits + totals.Realized + totals.Dividends
-	moneySpent := totals.Invested + totals.Fees
+	moneySpent := totals.Invested + totals.Fees + totals.Withdrawals
 	totals.Cash = moneyGained - moneySpent
 
 	// format money values to 2 decimals
@@ -128,6 +146,7 @@ func Aggregate(splits []fin.Splits, events []TradeEvent) ([]fin.Aggregate, fin.T
 	totals.Fees = floorFloat(totals.Fees, 2)
 	totals.Cash = floorFloat(totals.Cash, 2)
 	totals.Taxes = floorFloat(totals.Taxes, 2)
+	totals.Withdrawals = floorFloat(totals.Withdrawals, 2)
 
 	// sort and collate aggregates
 	sort.Strings(stockNames)

trading212/aggregate_test.go

@@ -10,7 +10,10 @@ import (
 
 func TestAggregate(t *testing.T) {
 	splits := []fin.Splits{
-		{Symbol: "ABEC", Date: "2022-07-16", Ratio: 20},
+		{Symbol: "GOOG", Date: "2022-07-16", Ratio: 20},
+	}
+	renames := map[string]string{
+		"ABEC": "GOOG",
 	}
 	tests := []struct {
 		name   string
@@ -22,36 +25,62 @@ func TestAggregate(t *testing.T) {
 			name:   "Regular test like our testdata",
 			events: defaultTestDataEvents,
 			want: []fin.Aggregate{
-				{Symbol: "ABEC", Name: "Alphabet (Class C)", ShareCount: 2.371231, AvgPrice: 113.86, PriceCurrency: "EUR", ShareCost: 270, ShareCostLocal: 270, ShareResult: 0, TotalDividend: 0, Fees: 0, Final: 0, LastUpdate: time.Date(2022, 7, 29, 14, 28, 17, 0, time.UTC)},
 				{Symbol: "FB", Name: "Meta Platforms", ShareCount: 0.086391, AvgPrice: 362, PriceCurrency: "USD", ShareCost: 31.27, ShareCostLocal: 26.67, ShareResult: 0, TotalDividend: 0, Fees: 0.04, Final: -0.04, LastUpdate: time.Date(2021, 8, 9, 18, 31, 41, 0, time.UTC)},
+				{Symbol: "GOOG", Name: "Alphabet (Class C)", ShareCount: 2.371231, AvgPrice: 113.86, PriceCurrency: "EUR", ShareCost: 270, ShareCostLocal: 270, ShareResult: 0, TotalDividend: 0, Fees: 0, Final: 0, LastUpdate: time.Date(2022, 7, 29, 14, 28, 17, 0, time.UTC)},
 				{Symbol: "MSFT", Name: "Microsoft", ShareCount: 0, AvgPrice: 0, PriceCurrency: "USD", ShareCost: 0, ShareCostLocal: 0, ShareResult: 2.61, TotalDividend: 0.11, Fees: 0.2, Final: 2.51, LastUpdate: time.Date(2021, 9, 30, 11, 15, 32, 0, time.UTC)},
 				{Symbol: "SAN", Name: "Sanofi", ShareCount: 0.111796, AvgPrice: 89.18, PriceCurrency: "EUR", ShareCost: 9.97, ShareCostLocal: 10, ShareResult: 0, TotalDividend: 0, Fees: 0.03, Final: -0.03, LastUpdate: time.Date(2022, 3, 7, 16, 10, 26, 0, time.UTC)},
 				{Symbol: "TSLA", Name: "Tesla", ShareCount: 0.076654, AvgPrice: 713.94, PriceCurrency: "USD", ShareCost: 54.72, ShareCostLocal: 46.67, ShareResult: 0, TotalDividend: 0, Fees: 0.07, Final: -0.08, LastUpdate: time.Date(2021, 8, 9, 18, 31, 41, 0, time.UTC)},
 			},
 			totals: &fin.Totals{
-				Deposits:  2000,
-				Invested:  353.2,
-				Realized:  2.61,
-				Dividends: 0.11,
-				Fees:      0.34,
-				Cash:      1649.17,
-				Taxes:     0.02,
+				Deposits:    2000,
+				Invested:    353.2,
+				Realized:    2.61,
+				Dividends:   0.11,
+				Fees:        0.34,
+				Cash:        1650.58,
+				Taxes:       0.02,
+				Withdrawals: -1.4,
 			},
 		},
 		{
 			name: "Test with a split",
 			events: []TradeEvent{
-				{Action: "Market buy", Time: DateTime{Time: time.Date(2021, 9, 27, 13, 19, 13, 0, time.UTC)}, TickerSymbol: "ABEC", ShareCount: 0.005, SharePrice: 2000.00, Total: 10.00, ID: "EOF1"},
-				{Action: "Market buy", Time: DateTime{Time: time.Date(2022, 9, 27, 13, 19, 13, 0, time.UTC)}, TickerSymbol: "ABEC", ShareCount: 0.125, SharePrice: 80.00, Total: 10.00, ID: "EOF2"},
+				{Action: "Market buy", Time: DateTime{Time: time.Date(2021, 9, 27, 13, 19, 13, 0, time.UTC)}, TickerSymbol: "GOOG", ShareCount: 0.005, SharePrice: 2000.00, Total: 10.00, ID: "EOF1"},
+				{Action: "Market buy", Time: DateTime{Time: time.Date(2022, 9, 27, 13, 19, 13, 0, time.UTC)}, TickerSymbol: "GOOG", ShareCount: 0.125, SharePrice: 80.00, Total: 10.00, ID: "EOF2"},
+			},
+			want: []fin.Aggregate{
+				{Symbol: "GOOG", ShareCount: 0.225, AvgPrice: 88.88, ShareCost: 20, ShareCostLocal: 20, ShareResult: 0, TotalDividend: 0, Fees: 0, Final: 0, LastUpdate: time.Date(2022, 9, 27, 13, 19, 13, 0, time.UTC)},
+			},
+		},
+		{
+			name: "Test float precision when selling",
+			events: []TradeEvent{
+				{Action: "Market buy", Time: DateTime{Time: time.Date(2021, 9, 27, 13, 19, 13, 0, time.UTC)}, TickerSymbol: "FB", ShareCount: 1.2345678, SharePrice: 2000.00, Total: 10.00, ID: "EOF1"},
+				{Action: "Market sell", Time: DateTime{Time: time.Date(2022, 9, 27, 13, 19, 13, 0, time.UTC)}, TickerSymbol: "FB", ShareCount: 1.2345, SharePrice: 2100.00, Result: 100, Total: 10.00, ID: "EOF2"},
 			},
 			want: []fin.Aggregate{
-				{Symbol: "ABEC", ShareCount: 0.225, AvgPrice: 88.88, ShareCost: 20, ShareCostLocal: 20, ShareResult: 0, TotalDividend: 0, Fees: 0, Final: 0, LastUpdate: time.Date(2022, 9, 27, 13, 19, 13, 0, time.UTC)},
+				{Symbol: "FB", ShareCount: 0, AvgPrice: 0, ShareCost: 0, ShareCostLocal: 0, ShareResult: 100, TotalDividend: 0, Fees: 0, Final: 100, LastUpdate: time.Date(2022, 9, 27, 13, 19, 13, 0, time.UTC)},
+			},
+		},
+		{
+			name: "Test operations with T212 card",
+			events: []TradeEvent{
+				{Action: "New card cost", Time: DateTime{Time: time.Date(2021, 9, 27, 13, 19, 13, 0, time.UTC)}, Total: -4.95, TotalCurrency: "EUR", ID: "EOF1"},
+				{Action: "Spending cashback", Time: DateTime{Time: time.Date(2022, 9, 27, 13, 19, 13, 0, time.UTC)}, Total: 0.22, TotalCurrency: "EUR", ID: "EOF2"},
+				{Action: "Deposit", Time: DateTime{Time: time.Date(2022, 9, 27, 13, 19, 13, 0, time.UTC)}, Total: 100, TotalCurrency: "EUR", ID: "EOF3"},
+				{Action: "Card debit", Time: DateTime{Time: time.Date(2022, 9, 27, 13, 19, 13, 0, time.UTC)}, Total: -15, TotalCurrency: "EUR", ID: "EOF4"},
+			},
+			want: []fin.Aggregate{},
+			totals: &fin.Totals{
+				Deposits:    100.22,
+				Cash:        80.27,
+				Withdrawals: 19.95, // New card cost + Card debit
 			},
 		},
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			aggregates, totals := Aggregate(splits, tt.events)
+			aggregates, totals := Aggregate(splits, renames, tt.events)
 			for idx, agg := range aggregates {
 				if !reflect.DeepEqual(agg, tt.want[idx]) {
 					t.Errorf("aggregate for %s is a mismatch \n%#v\n%#v", agg.Symbol, agg, tt.want[idx])
@@ -60,9 +89,17 @@ func TestAggregate(t *testing.T) {
 
 			if tt.totals != nil {
 				if !reflect.DeepEqual(totals, *tt.totals) {
-					t.Errorf("totals are a mismatch \n%#v\n%#v", totals, tt.totals)
+					t.Errorf("totals are a mismatch \n%#v\n%#v", totals, *tt.totals)
 				}
 			}
 		})
 	}
 }
+
+/*
+   trading212.TradeEvent{Action:"Deposit", Time:time.Date(2021, time.August, 9, 15, 25, 29, 0, time.UTC), ISIN:"", TickerSymbol:"", TickerName:"", ShareCount:0, SharePrice:0, ShareCurrency:"", ExchangeRate:"", ChargeAmount:1000, DepositFee:0, Result:0, ResultCurrency:"", Total:1000, TotalCurrency:"", Tax:0, TaxCurrency:"", StampDuty:0, StampDutyTax:0, Notes:"Transaction ID: xxx", ID:"d0ca160f-f407-4b9b-bb36-xxx", FXFee:0, FRFee:0, FinraFee:0, MerchantName:"", MerchantCategory:""}
+   trading212.TradeEvent{Action:"Deposit", Time:time.Date(2021, time.August, 9, 15, 25, 29, 0, time.UTC), ISIN:"", TickerSymbol:"", TickerName:"", ShareCount:0, SharePrice:0, ShareCurrency:"", ExchangeRate:"", ChargeAmount:0, DepositFee:0, Result:0, ResultCurrency:"", Total:1000, TotalCurrency:"", Tax:0, TaxCurrency:"", StampDuty:0, StampDutyTax:0, Notes:"Transaction ID: xxx", ID:"d0ca160f-f407-4b9b-bb36-xxx", FXFee:0, FRFee:0, FinraFee:0, MerchantName:"", MerchantCategory:""}
+
+            trading212.TradeEvent{Action:"Deposit", Time:time.Date(2021, time.September, 7, 13, 43, 10, 0, time.UTC), ISIN:"", TickerSymbol:"", TickerName:"", ShareCount:0, SharePrice:0, ShareCurrency:"", ExchangeRate:"", ChargeAmount:1001.4, DepositFee:1.4, Result:0, ResultCurrency:"", Total:1000, TotalCurrency:"", Tax:0, TaxCurrency:"", StampDuty:0, StampDutyTax:0, Notes:"Transaction ID: xxx", ID:"3e8f5274-1c62-46d6-baf4-xxx", FXFee:0, FRFee:0, FinraFee:0, MerchantName:"", MerchantCategory:""}
+            trading212.TradeEvent{Action:"Deposit", Time:time.Date(2021, time.September, 7, 13, 43, 10, 0, time.UTC), ISIN:"", TickerSymbol:"", TickerName:"", ShareCount:0, SharePrice:0, ShareCurrency:"", ExchangeRate:"", ChargeAmount:1000, DepositFee:0, Result:0, ResultCurrency:"", Total:1000, TotalCurrency:"", Tax:0, TaxCurrency:"", StampDuty:0, StampDutyTax:0, Notes:"Transaction ID: xxx", ID:"3e8f5274-1c62-46d6-baf4-xxx", FXFee:0, FRFee:0, FinraFee:0, MerchantName:"", MerchantCategory:""}
+*/