Implement spidering improvements

- pkg/app/relay/server.go
  - Modified the time ticker used for spidering with a custom duration value
  - Added a log message to indicate when the spider is running
- pkg/app/config/config.go
  - Added a new configuration parameter `SpiderTime` of type `time.Duration` to specify how often the spider should run
This commit is contained in:
2025-08-05 04:39:48 +01:00
parent 96eab2270d
commit df8e66d9a7
2 changed files with 21 additions and 19 deletions

View File

@@ -39,6 +39,7 @@ type C struct {
PublicReadable bool `env:"ORLY_PUBLIC_READABLE" default:"true" usage:"allow public read access to regardless of whether the client is authed"`
SpiderSeeds []string `env:"ORLY_SPIDER_SEEDS" usage:"seeds to use for the spider (relays that are looked up initially to find owner relay lists) (comma separated)" default:"wss://profiles.nostr1.com/,wss://relay.nostr.band/,wss://relay.damus.io/,wss://nostr.wine/,wss://nostr.land/,wss://theforest.nostr1.com/"`
SpiderType string `env:"ORLY_SPIDER_TYPE" usage:"whether to spider, and what degree of spidering: none, directory, follows (follows means to the second degree of the follow graph)" default:"directory"`
SpiderTime time.Duration `env:"ORLY_SPIDER_FREQUENCY" usage:"how often to run the spider, uses notation 0h0m0s" default:"1h"`
Owners []string `env:"ORLY_OWNERS" usage:"list of users whose follow lists designate whitelisted users who can publish events, and who can read if public readable is false (comma separated)"`
Private bool `env:"ORLY_PRIVATE" usage:"do not spider for user metadata because the relay is private and this would leak relay memberships" default:"false"`
Whitelist []string `env:"ORLY_WHITELIST" usage:"only allow connections from this list of IP addresses"`

View File

@@ -207,6 +207,7 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
func (s *Server) Start(
host string, port int, started ...chan bool,
) (err error) {
log.I.F("running spider every %v", s.C.SpiderTime)
if len(s.C.Owners) > 0 {
// start up spider
if err = s.Spider(s.C.Private); chk.E(err) {
@@ -216,7 +217,7 @@ func (s *Server) Start(
}
}
// start up a spider run to trigger every 30 minutes
ticker := time.NewTicker(time.Hour)
ticker := time.NewTicker(s.C.SpiderTime)
go func() {
for {
select {