refactor article generation to use async queues, add new poll endpoint

dao/dao.go

@@ -34,6 +34,7 @@ type storedEdition struct {
 	Date       string
 	StartTime  time.Time
 	EndTime    time.Time
+	Created    time.Time
 	Sources    []domain.Source
 	Articles   []string
 	Categories []string
@@ -42,6 +43,7 @@ type storedEdition struct {
 
 func GetEditionForTime(ctx context.Context, t time.Time, allowRecent bool) (*domain.Edition, error) {
 	iter := client.Collection("editions").Documents(ctx)
+	candidates := []*domain.Edition{}
 	var maxEdition storedEdition
 	for {
 		doc, err := iter.Next()
@@ -61,13 +63,28 @@ func GetEditionForTime(ctx context.Context, t time.Time, allowRecent bool) (*dom
 		}
 		if s.EndTime.After(t) {
 			e, err := editionFromStored(ctx, s)
-			return e, err
+			if err != nil {
+				return nil, err
+			}
+			candidates = append(candidates, e)
 		}
 	}
-	if maxEdition.ID != "" && allowRecent {
-		return editionFromStored(ctx, maxEdition)
+	if len(candidates) == 0 {
+		if maxEdition.ID != "" && allowRecent {
+			return editionFromStored(ctx, maxEdition)
+		}
+	}
+
+	selected := &domain.Edition{}
+	for _, e := range candidates {
+		if e.Created.After(selected.Created) {
+			selected = e
+		}
+	}
+	if selected.ID == "" {
+		return nil, nil
 	}
-	return nil, nil
+	return selected, nil
 }
 
 func SetEdition(ctx context.Context, e *domain.Edition) error {
@@ -90,6 +107,7 @@ func editionToStored(e *domain.Edition) storedEdition {
 		Sources:    e.Sources,
 		StartTime:  e.StartTime,
 		EndTime:    e.EndTime,
+		Created:    e.Created,
 		Categories: e.Categories,
 		Metadata:   e.Metadata,
 	}
@@ -110,6 +128,7 @@ func editionFromStored(ctx context.Context, s storedEdition) (*domain.Edition, e
 		Sources:    s.Sources,
 		StartTime:  s.StartTime,
 		EndTime:    s.EndTime,
+		Created:    s.Created,
 		Categories: s.Categories,
 		Metadata:   s.Metadata,
 	}
@@ -162,3 +181,42 @@ func GetArticle(ctx context.Context, id string) (*domain.Article, error) {
 	mu.Unlock()
 	return &a, err
 }
+
+func GetArticleByURL(ctx context.Context, url string) (*domain.Article, error) {
+	iter := client.Collection("articles").Where("Link", "==", url).Documents(ctx)
+	docs, err := iter.GetAll()
+	if err != nil {
+		return nil, err
+	}
+	if len(docs) == 0 {
+		return nil, nil
+	}
+	a := domain.Article{}
+	err = docs[0].DataTo(&a)
+	mu.Lock()
+	articleCache[a.ID] = a
+	mu.Unlock()
+	return &a, err
+}
+
+func GetArticlesByTime(ctx context.Context, start, end time.Time) ([]domain.Article, error) {
+	iter := client.Collection("articles").
+		Where("Timestamp", ">", start).
+		Where("Timestamp", "<", end).
+		Documents(ctx)
+	docs, err := iter.GetAll()
+	if err != nil {
+		return nil, err
+	}
+
+	out := []domain.Article{}
+	for _, doc := range docs {
+		a := domain.Article{}
+		err = doc.DataTo(&a)
+		mu.Lock()
+		articleCache[a.ID] = a
+		mu.Unlock()
+		out = append(out, a)
+	}
+	return out, nil
+}

domain/edition.go

@@ -2,13 +2,8 @@ package domain
 
 import (
 	"context"
-	"sort"
+	"github.com/arussellsaw/news/idgen"
 	"time"
-	"unicode/utf8"
-
-	"github.com/pkg/errors"
-
-	"github.com/google/uuid"
 )
 
 var (
@@ -26,6 +21,7 @@ type Edition struct {
 
 	StartTime time.Time
 	EndTime   time.Time
+	Created   time.Time
 
 	Metadata map[string]string
 
@@ -56,7 +52,6 @@ top:
 				}
 				return ""
 			}()
-			a.Trim(size)
 			a.Layout = Layout{}
 			return a
 		}
@@ -87,9 +82,10 @@ func NewEdition(ctx context.Context, now time.Time) (*Edition, error) {
 	}
 
 	e := Edition{
-		ID:         uuid.New().String(),
+		ID:         idgen.New("edt"),
 		Sources:    sources,
 		Categories: cats,
+		Created:    time.Now(),
 	}
 
 	e.Date = time.Now().Format("Monday January 02 2006")
@@ -105,49 +101,5 @@ func NewEdition(ctx context.Context, now time.Time) (*Edition, error) {
 		e.Name = "Evening Edition"
 	}
 
-	articles, err := FetchArticles(ctx)
-	if err != nil {
-		return nil, errors.Wrap(err, "error fetching articles")
-	}
-
-	newArticles := []Article{}
-L:
-	for _, a := range articles {
-		if time.Since(a.Timestamp) > 100*time.Hour {
-			continue
-		}
-		for _, e := range a.Content {
-			if !utf8.Valid([]byte(e.Value)) {
-				continue L
-			}
-		}
-		newArticles = append(newArticles, a)
-	}
-	e.Articles = newArticles
-
-	bySource := make(map[string][]Article)
-	for _, a := range e.Articles {
-		bySource[a.Source.Name] = append(bySource[a.Source.Name], a)
-	}
-	newArticles = nil
-	for _, as := range bySource {
-		sort.Slice(as, func(i, j int) bool {
-			return as[i].Timestamp.After(as[j].Timestamp)
-		})
-	}
-top:
-	for s, as := range bySource {
-		newArticles = append(newArticles, as[0])
-		bySource[s] = as[1:]
-		if len(bySource[s]) == 0 {
-			delete(bySource, s)
-			goto top
-		}
-	}
-	if len(bySource) != 0 {
-		goto top
-	}
-	e.Articles = newArticles
-
 	return &e, nil
 }