Merge pull request #13 from planetary-social/substack

Merge substack into main.

.env.sample

@@ -2,11 +2,10 @@ CGO_ENABLED=1
 GOARCH=amd64
 GOOS=linux
 PORT=8080
-SECRET=test
 DB_DIR="/db/rsslay.sqlite"
 DEFAULT_PROFILE_PICTURE_URL="https://i.imgur.com/MaceU96.png"
 SECRET="CHANGE_ME"
-VERSION=0.4.7
+VERSION=0.5.3
 REPLAY_TO_RELAYS=false
 RELAYS_TO_PUBLISH_TO=""
 NITTER_INSTANCES=""
@@ -21,4 +20,5 @@ INFO_RELAY_NAME="rsslay"
 INFO_CONTACT=""
 MAX_CONTENT_LENGTH=250
 LOG_LEVEL=WARN
-DELETE_FAILING_FEEDS=false
+DELETE_FAILING_FEEDS=false
+REDIS_CONNECTION_STRING=""

Dockerfile

@@ -28,7 +28,7 @@ ENV PORT="8080"
 ENV DB_DIR="/db/rsslay.sqlite"
 ENV DEFAULT_PROFILE_PICTURE_URL="https://i.imgur.com/MaceU96.png"
 ENV SECRET="CHANGE_ME"
-ENV VERSION=0.4.7
+ENV VERSION=0.5.3
 ENV REPLAY_TO_RELAYS=false
 ENV RELAYS_TO_PUBLISH_TO=""
 ENV NITTER_INSTANCES=""
@@ -44,6 +44,7 @@ ENV INFO_CONTACT=""
 ENV MAX_CONTENT_LENGTH=250
 ENV LOG_LEVEL="WARN"
 ENV DELETE_FAILING_FEEDS=false
+ENV REDIS_CONNECTION_STRING=""
 
 COPY --from=build /rsslay .
 COPY --from=build /app/web/assets/ ./web/assets/

Dockerfile.fly

@@ -30,7 +30,7 @@ ENV PORT="8080"
 ENV DB_DIR="/db/rsslay.sqlite"
 ENV DEFAULT_PROFILE_PICTURE_URL="https://i.imgur.com/MaceU96.png"
 ENV SECRET="CHANGE_ME"
-ENV VERSION=0.4.7
+ENV VERSION=0.5.3
 ENV REPLAY_TO_RELAYS=false
 ENV RELAYS_TO_PUBLISH_TO=""
 ENV NITTER_INSTANCES=""
@@ -46,6 +46,7 @@ ENV INFO_CONTACT=""
 ENV MAX_CONTENT_LENGTH=250
 ENV LOG_LEVEL="WARN"
 ENV DELETE_FAILING_FEEDS=false
+ENV REDIS_CONNECTION_STRING=""
 
 COPY --from=litefs /usr/local/bin/litefs /usr/local/bin/litefs
 COPY --from=build /rsslay /usr/local/bin/rsslay

Dockerfile.railwayapp

@@ -57,6 +57,7 @@ ARG INFO_CONTACT
 ARG MAX_CONTENT_LENGTH
 ARG LOG_LEVEL
 ARG DELETE_FAILING_FEEDS
+ARG REDIS_CONNECTION_STRING
 
 LABEL org.opencontainers.image.title="rsslay"
 LABEL org.opencontainers.image.source=https://github.com/piraces/rsslay
@@ -68,7 +69,7 @@ ENV PORT=$PORT
 ENV DB_DIR=$DB_DIR
 ENV DEFAULT_PROFILE_PICTURE_URL=$DEFAULT_PROFILE_PICTURE_URL
 ENV SECRET=$SECRET
-ENV VERSION=0.4.7
+ENV VERSION=0.5.3
 ENV REPLAY_TO_RELAYS=false
 ENV RELAYS_TO_PUBLISH_TO=""
 ENV NITTER_INSTANCES=""
@@ -84,6 +85,7 @@ ENV INFO_CONTACT=""
 ENV MAX_CONTENT_LENGTH=250
 ENV LOG_LEVEL="WARN"
 ENV DELETE_FAILING_FEEDS=false
+ENV REDIS_CONNECTION_STRING=$REDIS_CONNECTION_STRING
 
 COPY --from=build /rsslay .
 COPY --from=build /app/web/assets/ ./web/assets/

README.md

@@ -30,19 +30,11 @@ Checkout the [wiki entry](https://github.com/piraces/rsslay/wiki/API) for furthe
 
 ## Mirroring events ("replaying")
 
-Actually `rsslay` makes usage of a method named `AttemptReplayEvents` which is made to send the events to other relays of confidence to attempt to make the events and the profile more reachable (they are just mirror relays)...
-
-Currently used relays:
-- wss://relay.nostrgraph.net
-- wss://e.nos.lol
-- wss://nos.lol
-- wss://nostr.mom
-- wss://relay.nostr.band
-- wss://nostr.mutinywallet.com
+_**Note:** since v0.5.3 its recommended to set `REPLAY_TO_RELAYS` to false. There is no need to perform replays to other relays, the main rsslay should be able to handle the events._
 
-This is needed nowadays, with further improvements in relays implementations or clients it may not be needed in the future.
+Actually `rsslay` makes usage of a method named `AttemptReplayEvents` which is made to send the events to other relays of confidence to attempt to make the events and the profile more reachable (they are just mirror relays)...
 
-Maybe in the future with other implementations we can avoid that, but nowadays its needed.
+Currently used relays: none.
 
 ## Feeds from Twitter via Nitter instances
 
@@ -66,6 +58,18 @@ If you want to run your own instance, you are covered!
 Several options (including "one-click" ones) are available.
 Checkout [the wiki](https://github.com/piraces/rsslay/wiki/Deploy-your-own-instance).
 
+## Caching
+
+Since version v0.5.1, rsslay uses cache by default (in-memory with [BigCache](https://github.com/allegro/bigcache) by default or with [Redis](https://redis.io/) if configured) enabled by default to improve performance.
+In the case of the main instance `rsslay.nostr.moe`, Redis is used in HA mode to improve performance for multiple requests for the same feed.
+
+**Nevertheless, there is a caveat using this approach that is that cached feeds do not refresh for 30 minutes (but I personally think it is worth for the performance gain).**
+
+## Metrics
+
+Since version v0.5.1, rsslay uses [Prometheus](https://prometheus.io/) instrumenting with metrics exposed on `/metrics` path.
+So with this you can mount your own [Graphana](https://grafana.com/) dashboards and look into rsslay insights!
+
 # Contributing
 
 Feel free to [open an issue](https://github.com/piraces/rsslay/issues/new), provide feedback in [discussions](https://github.com/piraces/rsslay/discussions), or fork the repo and open a PR with your contribution!

SECURITY.md

@@ -9,10 +9,10 @@ Only the latest minor version is currently being supported with security updates
 
 As of now the following versions are covered with security updates:
 
-| Version   | Supported          |
-| -------   | ------------------ |
-| 0.4.x     | :white_check_mark: |
-| < 0.4.0   | :x:                |
+| Version | Supported          |
+|---------| ------------------ |
+| 0.5.x   | :white_check_mark: |
+| < 0.5.0 | :x:                |
 
 ## Reporting a Vulnerability
 

cmd/rsslay/main.go

@@ -7,6 +7,14 @@ import (
 	"errors"
 	"flag"
 	"fmt"
+	"log"
+	"net/http"
+	"os"
+	"path"
+	"sync"
+	"time"
+
+	"github.com/eko/gocache/lib/v4/cache"
 	"github.com/fiatjaf/relayer"
 	_ "github.com/fiatjaf/relayer"
 	"github.com/hashicorp/logutils"
@@ -16,17 +24,14 @@ import (
 	"github.com/nbd-wtf/go-nostr"
 	"github.com/nbd-wtf/go-nostr/nip11"
 	"github.com/piraces/rsslay/internal/handlers"
+	"github.com/piraces/rsslay/pkg/custom_cache"
 	"github.com/piraces/rsslay/pkg/events"
 	"github.com/piraces/rsslay/pkg/feed"
+	"github.com/piraces/rsslay/pkg/metrics"
 	"github.com/piraces/rsslay/pkg/replayer"
 	"github.com/piraces/rsslay/scripts"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"golang.org/x/exp/slices"
-	"log"
-	"net/http"
-	"os"
-	"path"
-	"sync"
-	"time"
 )
 
 // Command line flags.
@@ -55,13 +60,16 @@ type Relay struct {
 	Contact                         string   `envconfig:"INFO_CONTACT" default:"~"`
 	MaxContentLength                int      `envconfig:"MAX_CONTENT_LENGTH" default:"250"`
 	DeleteFailingFeeds              bool     `envconfig:"DELETE_FAILING_FEEDS" default:"false"`
+	RedisConnectionString           string   `envconfig:"REDIS_CONNECTION_STRING" default:""`
 
 	updates            chan nostr.Event
 	lastEmitted        sync.Map
 	db                 *sql.DB
 	healthCheck        *health.Health
 	mutex              sync.Mutex
 	routineQueueLength int
+	converterSelector  *feed.ConverterSelector
+	cache              *cache.Cache[string]
 }
 
 var relayInstance = &Relay{
@@ -93,13 +101,20 @@ func ConfigureLogging() {
 	log.SetOutput(filter)
 }
 
+func ConfigureCache() {
+	if relayInstance.RedisConnectionString != "" {
+		custom_cache.RedisConnectionString = &relayInstance.RedisConnectionString
+	}
+	custom_cache.InitializeCache()
+}
+
 func (r *Relay) Name() string {
 	return r.RelayName
 }
 
 func (r *Relay) OnInitialized(s *relayer.Server) {
 	s.Router().Path("/").HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
-		handlers.HandleWebpage(writer, request, r.db)
+		handlers.HandleWebpage(writer, request, r.db, &r.MainDomainName)
 	})
 	s.Router().Path("/create").HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
 		handlers.HandleCreateFeed(writer, request, r.db, &r.Secret, dsn)
@@ -117,6 +132,7 @@ func (r *Relay) OnInitialized(s *relayer.Server) {
 	s.Router().Path("/.well-known/nostr.json").HandlerFunc(func(writer http.ResponseWriter, request *http.Request) {
 		handlers.HandleNip05(writer, request, r.db, &r.OwnerPublicKey, &r.EnableAutoNIP05Registration)
 	})
+	s.Router().Path("/metrics").Handler(promhttp.Handler())
 }
 
 func (r *Relay) Init() error {
@@ -128,16 +144,21 @@ func (r *Relay) Init() error {
 		log.Printf("[INFO] Running VERSION %s:\n - DSN=%s\n - DB_DIR=%s\n\n", r.Version, *dsn, r.DatabaseDirectory)
 	}
 
+	ConfigureCache()
 	r.db = InitDatabase(r)
 
 	go r.UpdateListeningFilters()
 
+	longFormConverter := feed.NewLongFormConverter()
+	r.converterSelector = feed.NewConverterSelector(longFormConverter)
+
 	return nil
 }
 
 func (r *Relay) UpdateListeningFilters() {
 	for {
 		time.Sleep(20 * time.Minute)
+		metrics.ListeningFiltersOps.Inc()
 
 		filters := relayer.GetListeningFilters()
 		log.Printf("[DEBUG] Checking for updates; %d filters active", len(filters))
@@ -151,9 +172,11 @@ func (r *Relay) UpdateListeningFilters() {
 						continue
 					}
 
+					converter := r.converterSelector.Select(parsedFeed)
+
 					for _, item := range parsedFeed.Items {
 						defaultCreatedAt := time.Unix(time.Now().Unix(), 0)
-						evt := feed.ItemToTextNote(pubkey, item, parsedFeed, defaultCreatedAt, entity.URL, relayInstance.MaxContentLength)
+						evt := converter.Convert(pubkey, item, parsedFeed, defaultCreatedAt, entity.URL)
 						last, ok := r.lastEmitted.Load(entity.URL)
 						if last == nil {
 							last = uint32(time.Now().Unix())
@@ -162,7 +185,7 @@ func (r *Relay) UpdateListeningFilters() {
 							_ = evt.Sign(entity.PrivateKey)
 							r.updates <- evt
 							r.lastEmitted.Store(entity.URL, last.(uint32))
-							parsedEvents = append(parsedEvents, replayer.EventWithPrivateKey{Event: evt, PrivateKey: entity.PrivateKey})
+							parsedEvents = append(parsedEvents, replayer.EventWithPrivateKey{Event: &evt, PrivateKey: entity.PrivateKey})
 						}
 					}
 				}
@@ -175,6 +198,7 @@ func (r *Relay) UpdateListeningFilters() {
 func (r *Relay) AttemptReplayEvents(events []replayer.EventWithPrivateKey) {
 	if relayInstance.ReplayToRelays && relayInstance.routineQueueLength < relayInstance.MaxSubroutines && len(events) > 0 {
 		r.routineQueueLength++
+		metrics.ReplayRoutineQueueLength.Set(float64(r.routineQueueLength))
 		replayer.ReplayEventsToRelays(&replayer.ReplayParameters{
 			MaxEventsToReplay:        relayInstance.MaxEventsToReplay,
 			RelaysToPublish:          relayInstance.RelaysToPublish,
@@ -188,6 +212,7 @@ func (r *Relay) AttemptReplayEvents(events []replayer.EventWithPrivateKey) {
 }
 
 func (r *Relay) AcceptEvent(_ *nostr.Event) bool {
+	metrics.InvalidEventsRequests.Inc()
 	return false
 }
 
@@ -201,17 +226,21 @@ type store struct {
 
 func (b store) Init() error { return nil }
 func (b store) SaveEvent(_ *nostr.Event) error {
+	metrics.InvalidEventsRequests.Inc()
 	return errors.New("blocked: we don't accept any events")
 }
 
 func (b store) DeleteEvent(_, _ string) error {
+	metrics.InvalidEventsRequests.Inc()
 	return errors.New("blocked: we can't delete any events")
 }
 
 func (b store) QueryEvents(filter *nostr.Filter) ([]nostr.Event, error) {
 	var parsedEvents []nostr.Event
 	var eventsToReplay []replayer.EventWithPrivateKey
 
+	metrics.QueryEventsRequests.Inc()
+
 	if filter.IDs != nil || len(filter.Tags) > 0 {
 		return parsedEvents, nil
 	}
@@ -223,8 +252,10 @@ func (b store) QueryEvents(filter *nostr.Filter) ([]nostr.Event, error) {
 			continue
 		}
 
+		converter := relayInstance.converterSelector.Select(parsedFeed)
+
 		if filter.Kinds == nil || slices.Contains(filter.Kinds, nostr.KindSetMetadata) {
-			evt := feed.EntryFeedToSetMetadata(pubkey, parsedFeed, entity.URL, relayInstance.EnableAutoNIP05Registration, relayInstance.DefaultProfilePictureUrl)
+			evt := feed.EntryFeedToSetMetadata(pubkey, parsedFeed, entity.URL, relayInstance.EnableAutoNIP05Registration, relayInstance.DefaultProfilePictureUrl, relayInstance.MainDomainName)
 
 			if filter.Since != nil && evt.CreatedAt < *filter.Since {
 				continue
@@ -235,14 +266,16 @@ func (b store) QueryEvents(filter *nostr.Filter) ([]nostr.Event, error) {
 
 			_ = evt.Sign(entity.PrivateKey)
 			parsedEvents = append(parsedEvents, evt)
-			eventsToReplay = append(eventsToReplay, replayer.EventWithPrivateKey{Event: evt, PrivateKey: entity.PrivateKey})
+			if relayInstance.ReplayToRelays {
+				eventsToReplay = append(eventsToReplay, replayer.EventWithPrivateKey{Event: &evt, PrivateKey: entity.PrivateKey})
+			}
 		}
 
-		if filter.Kinds == nil || slices.Contains(filter.Kinds, nostr.KindTextNote) {
+		if filter.Kinds == nil || slices.Contains(filter.Kinds, nostr.KindTextNote) || slices.Contains(filter.Kinds, feed.KindLongFormTextContent) {
 			var last uint32 = 0
 			for _, item := range parsedFeed.Items {
 				defaultCreatedAt := time.Unix(time.Now().Unix(), 0)
-				evt := feed.ItemToTextNote(pubkey, item, parsedFeed, defaultCreatedAt, entity.URL, relayInstance.MaxContentLength)
+				evt := converter.Convert(pubkey, item, parsedFeed, defaultCreatedAt, entity.URL)
 
 				// Feed need to have a date for each entry...
 				if evt.CreatedAt == nostr.Timestamp(defaultCreatedAt.Unix()) {
@@ -258,12 +291,18 @@ func (b store) QueryEvents(filter *nostr.Filter) ([]nostr.Event, error) {
 
 				_ = evt.Sign(entity.PrivateKey)
 
+				if !filter.Matches(&evt) {
+					continue
+				}
+
 				if evt.CreatedAt > nostr.Timestamp(int64(last)) {
 					last = uint32(evt.CreatedAt)
 				}
 
 				parsedEvents = append(parsedEvents, evt)
-				eventsToReplay = append(eventsToReplay, replayer.EventWithPrivateKey{Event: evt, PrivateKey: entity.PrivateKey})
+				if relayInstance.ReplayToRelays {
+					eventsToReplay = append(eventsToReplay, replayer.EventWithPrivateKey{Event: &evt, PrivateKey: entity.PrivateKey})
+				}
 			}
 
 			relayInstance.lastEmitted.Store(entity.URL, last)
@@ -280,6 +319,7 @@ func (r *Relay) InjectEvents() chan nostr.Event {
 }
 
 func (r *Relay) GetNIP11InformationDocument() nip11.RelayInformationDocument {
+	metrics.RelayInfoRequests.Inc()
 	infoDocument := nip11.RelayInformationDocument{
 		Name:          relayInstance.Name(),
 		Description:   "Relay that creates virtual nostr profiles for each RSS feed submitted, powered by the relayer framework",

fly.toml

@@ -6,6 +6,10 @@ kill_timeout = 5
 [experimental]
   enable_consul = true
 
+[metrics]
+  port = 8080
+  path = "/metrics"
+
 [env]
   DB_DIR = "/var/lib/litefs/db"
   DEFAULT_PROFILE_PICTURE_URL = "https://i.imgur.com/MaceU96.png"
@@ -23,9 +27,9 @@ kill_timeout = 5
   NITTER_INSTANCES = "birdsite.xanny.family,notabird.site,nitter.moomoo.me,nitter.fly.dev"
   OWNER_PUBLIC_KEY = "4ac24d2ee822a34a9881eff526bf71f39704419837e4c14b34642d82e111ed39"
   PORT = "8080"
-  RELAYS_TO_PUBLISH_TO = "wss://relay.nostrgraph.net,wss://e.nos.lol,wss://nostr.mom,wss://relay.nostr.band,wss://nos.lol,wss://nostr.mutinywallet.com"
-  REPLAY_TO_RELAYS = "true"
-  VERSION = "0.4.7"
+  RELAYS_TO_PUBLISH_TO = ""
+  REPLAY_TO_RELAYS = "false"
+  VERSION = "0.5.3"
 
 [[mounts]]
   source = "rsslay_data_machines"