mleku/orly
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Implement spidering improvements

Browse Source 
- pkg/app/relay/server.go
  - Modified the time ticker used for spidering with a custom duration value
  - Added a log message to indicate when the spider is running
- pkg/app/config/config.go
  - Added a new configuration parameter `SpiderTime` of type `time.Duration` to specify how often the spider should run
This commit is contained in:
mleku
2025-08-05 04:39:48 +01:00
parent 96eab2270d
commit df8e66d9a7
2 changed files with 21 additions and 19 deletions
Download Patch File Download Diff File Expand all files Collapse all files

37
pkg/app/config/config.go
View File

@@ -26,24 +26,25 @@ import (
// and default values. It defines parameters for app behaviour, storage // and default values. It defines parameters for app behaviour, storage
// locations, logging, and network settings used across the relay service. // locations, logging, and network settings used across the relay service.
type C struct { type C struct {
AppName string `env:"ORLY_APP_NAME" default:"orly"` AppName string `env:"ORLY_APP_NAME" default:"orly"`
Config string `env:"ORLY_CONFIG_DIR" usage:"location for configuration file, which has the name '.env' to make it harder to delete, and is a standard environment KEY=value<newline>... style" default:"~/.config/orly"` Config string `env:"ORLY_CONFIG_DIR" usage:"location for configuration file, which has the name '.env' to make it harder to delete, and is a standard environment KEY=value<newline>... style" default:"~/.config/orly"`
State string `env:"ORLY_STATE_DATA_DIR" usage:"storage location for state data affected by dynamic interactive interfaces" default:"~/.local/state/orly"` State string `env:"ORLY_STATE_DATA_DIR" usage:"storage location for state data affected by dynamic interactive interfaces" default:"~/.local/state/orly"`
DataDir string `env:"ORLY_DATA_DIR" usage:"storage location for the event store" default:"~/.local/cache/orly"` DataDir string `env:"ORLY_DATA_DIR" usage:"storage location for the event store" default:"~/.local/cache/orly"`
Listen string `env:"ORLY_LISTEN" default:"0.0.0.0" usage:"network listen address"` Listen string `env:"ORLY_LISTEN" default:"0.0.0.0" usage:"network listen address"`
Port int `env:"ORLY_PORT" default:"3334" usage:"port to listen on"` Port int `env:"ORLY_PORT" default:"3334" usage:"port to listen on"`
LogLevel string `env:"ORLY_LOG_LEVEL" default:"info" usage:"debug level: fatal error warn info debug trace"` LogLevel string `env:"ORLY_LOG_LEVEL" default:"info" usage:"debug level: fatal error warn info debug trace"`
DbLogLevel string `env:"ORLY_DB_LOG_LEVEL" default:"info" usage:"debug level: fatal error warn info debug trace"` DbLogLevel string `env:"ORLY_DB_LOG_LEVEL" default:"info" usage:"debug level: fatal error warn info debug trace"`
Pprof string `env:"ORLY_PPROF" usage:"enable pprof on 127.0.0.1:6060" enum:"cpu,memory,allocation"` Pprof string `env:"ORLY_PPROF" usage:"enable pprof on 127.0.0.1:6060" enum:"cpu,memory,allocation"`
AuthRequired bool `env:"ORLY_AUTH_REQUIRED" default:"false" usage:"require authentication for all requests"` AuthRequired bool `env:"ORLY_AUTH_REQUIRED" default:"false" usage:"require authentication for all requests"`
PublicReadable bool `env:"ORLY_PUBLIC_READABLE" default:"true" usage:"allow public read access to regardless of whether the client is authed"` PublicReadable bool `env:"ORLY_PUBLIC_READABLE" default:"true" usage:"allow public read access to regardless of whether the client is authed"`
SpiderSeeds []string `env:"ORLY_SPIDER_SEEDS" usage:"seeds to use for the spider (relays that are looked up initially to find owner relay lists) (comma separated)" default:"wss://profiles.nostr1.com/,wss://relay.nostr.band/,wss://relay.damus.io/,wss://nostr.wine/,wss://nostr.land/,wss://theforest.nostr1.com/"` SpiderSeeds []string `env:"ORLY_SPIDER_SEEDS" usage:"seeds to use for the spider (relays that are looked up initially to find owner relay lists) (comma separated)" default:"wss://profiles.nostr1.com/,wss://relay.nostr.band/,wss://relay.damus.io/,wss://nostr.wine/,wss://nostr.land/,wss://theforest.nostr1.com/"`
SpiderType string `env:"ORLY_SPIDER_TYPE" usage:"whether to spider, and what degree of spidering: none, directory, follows (follows means to the second degree of the follow graph)" default:"directory"` SpiderType string `env:"ORLY_SPIDER_TYPE" usage:"whether to spider, and what degree of spidering: none, directory, follows (follows means to the second degree of the follow graph)" default:"directory"`
Owners []string `env:"ORLY_OWNERS" usage:"list of users whose follow lists designate whitelisted users who can publish events, and who can read if public readable is false (comma separated)"` SpiderTime time.Duration `env:"ORLY_SPIDER_FREQUENCY" usage:"how often to run the spider, uses notation 0h0m0s" default:"1h"`
Private bool `env:"ORLY_PRIVATE" usage:"do not spider for user metadata because the relay is private and this would leak relay memberships" default:"false"` Owners []string `env:"ORLY_OWNERS" usage:"list of users whose follow lists designate whitelisted users who can publish events, and who can read if public readable is false (comma separated)"`
Whitelist []string `env:"ORLY_WHITELIST" usage:"only allow connections from this list of IP addresses"` Private bool `env:"ORLY_PRIVATE" usage:"do not spider for user metadata because the relay is private and this would leak relay memberships" default:"false"`
RelaySecret string `env:"ORLY_SECRET_KEY" usage:"secret key for relay cluster replication authentication"` Whitelist []string `env:"ORLY_WHITELIST" usage:"only allow connections from this list of IP addresses"`
PeerRelays []string `env:"ORLY_PEER_RELAYS" usage:"list of peer relays URLs that new events are pushed to in format <pubkey>|<url>"` RelaySecret string `env:"ORLY_SECRET_KEY" usage:"secret key for relay cluster replication authentication"`
PeerRelays []string `env:"ORLY_PEER_RELAYS" usage:"list of peer relays URLs that new events are pushed to in format <pubkey>|<url>"`
} }
// New creates and initializes a new configuration object for the relay // New creates and initializes a new configuration object for the relay

3
pkg/app/relay/server.go
View File

@@ -207,6 +207,7 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
func (s *Server) Start( func (s *Server) Start(
host string, port int, started ...chan bool, host string, port int, started ...chan bool,
) (err error) { ) (err error) {
log.I.F("running spider every %v", s.C.SpiderTime)
if len(s.C.Owners) > 0 { if len(s.C.Owners) > 0 {
// start up spider // start up spider
if err = s.Spider(s.C.Private); chk.E(err) { if err = s.Spider(s.C.Private); chk.E(err) {
@@ -216,7 +217,7 @@ func (s *Server) Start(
} }
} }
// start up a spider run to trigger every 30 minutes // start up a spider run to trigger every 30 minutes
ticker := time.NewTicker(time.Hour) ticker := time.NewTicker(s.C.SpiderTime)
go func() { go func() {
for { for {
select { select {