158 lines
3.0 KiB
Go
158 lines
3.0 KiB
Go
package main
|
|
|
|
import (
|
|
"crypto/hmac"
|
|
"crypto/sha256"
|
|
"encoding/hex"
|
|
"encoding/json"
|
|
"fmt"
|
|
"net/http"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/PuerkitoBio/goquery"
|
|
"github.com/nbd-wtf/go-nostr"
|
|
strip "github.com/grokify/html-strip-tags-go"
|
|
"github.com/mmcdole/gofeed"
|
|
"github.com/rif/cache2go"
|
|
)
|
|
|
|
var (
|
|
fp = gofeed.NewParser()
|
|
feedCache = cache2go.New(512, time.Minute*19)
|
|
client = &http.Client{
|
|
Timeout: 5 * time.Second,
|
|
}
|
|
)
|
|
|
|
type Entity struct {
|
|
PrivateKey string
|
|
URL string
|
|
}
|
|
|
|
var types = []string{
|
|
"rss+xml",
|
|
"atom+xml",
|
|
"feed+json",
|
|
"text/xml",
|
|
"application/xml",
|
|
}
|
|
|
|
func getFeedURL(url string) string {
|
|
resp, err := client.Get(url)
|
|
if err != nil || resp.StatusCode >= 300 {
|
|
return ""
|
|
}
|
|
|
|
ct := resp.Header.Get("Content-Type")
|
|
for _, typ := range types {
|
|
if strings.Contains(ct, typ) {
|
|
return url
|
|
}
|
|
}
|
|
|
|
if strings.Contains(ct, "text/html") {
|
|
doc, err := goquery.NewDocumentFromReader(resp.Body)
|
|
if err != nil {
|
|
return ""
|
|
}
|
|
|
|
for _, typ := range types {
|
|
href, _ := doc.Find(fmt.Sprintf("link[type*='%s']", typ)).Attr("href")
|
|
if href == "" {
|
|
continue
|
|
}
|
|
if !strings.HasPrefix(href, "http") {
|
|
href, _ = urljoin(url, href)
|
|
}
|
|
return href
|
|
}
|
|
}
|
|
|
|
return ""
|
|
}
|
|
|
|
func parseFeed(url string) (*gofeed.Feed, error) {
|
|
if feed, ok := feedCache.Get(url); ok {
|
|
return feed.(*gofeed.Feed), nil
|
|
}
|
|
|
|
feed, err := fp.ParseURL(url)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
// cleanup a little so we don't store too much junk
|
|
for i := range feed.Items {
|
|
feed.Items[i].Content = ""
|
|
}
|
|
feedCache.Set(url, feed)
|
|
|
|
return feed, nil
|
|
}
|
|
|
|
func feedToSetMetadata(pubkey string, feed *gofeed.Feed) nostr.Event {
|
|
metadata := map[string]string{
|
|
"name": feed.Title,
|
|
"about": feed.Description + "\n\n" + feed.Link,
|
|
}
|
|
if feed.Image != nil {
|
|
metadata["picture"] = feed.Image.URL
|
|
}
|
|
content, _ := json.Marshal(metadata)
|
|
|
|
createdAt := time.Now()
|
|
if feed.PublishedParsed != nil {
|
|
createdAt = *feed.PublishedParsed
|
|
}
|
|
|
|
evt := nostr.Event{
|
|
PubKey: pubkey,
|
|
CreatedAt: createdAt,
|
|
Kind: nostr.KindSetMetadata,
|
|
Tags: nostr.Tags{},
|
|
Content: string(content),
|
|
}
|
|
evt.ID = string(evt.Serialize())
|
|
|
|
return evt
|
|
}
|
|
|
|
func itemToTextNote(pubkey string, item *gofeed.Item) nostr.Event {
|
|
content := ""
|
|
if item.Title != "" {
|
|
content = "**" + item.Title + "**\n\n"
|
|
}
|
|
content += strip.StripTags(item.Description)
|
|
if len(content) > 250 {
|
|
content += content[0:249] + "…"
|
|
}
|
|
content += "\n\n" + item.Link
|
|
|
|
createdAt := time.Now()
|
|
if item.UpdatedParsed != nil {
|
|
createdAt = *item.UpdatedParsed
|
|
}
|
|
if item.PublishedParsed != nil {
|
|
createdAt = *item.PublishedParsed
|
|
}
|
|
|
|
evt := nostr.Event{
|
|
PubKey: pubkey,
|
|
CreatedAt: createdAt,
|
|
Kind: nostr.KindTextNote,
|
|
Tags: nostr.Tags{},
|
|
Content: content,
|
|
}
|
|
evt.ID = string(evt.Serialize())
|
|
|
|
return evt
|
|
}
|
|
|
|
func privateKeyFromFeed(url string) string {
|
|
m := hmac.New(sha256.New, []byte(relay.Secret))
|
|
m.Write([]byte(url))
|
|
r := m.Sum(nil)
|
|
return hex.EncodeToString(r)
|
|
}
|