mleku/indra
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

starting integration of listener, remove holo

Browse Source
This commit is contained in:
херетик
2023-05-11 09:04:10 +01:00
parent cba46e7298
commit 9a568ff720
19 changed files with 36 additions and 304 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
pkg/engine/dispatcher_test.go
View File

@@ -66,7 +66,7 @@ func TestEngine_Dispatcher(t *testing.T) {
var eng *Engine
if eng, e = NewEngine(Params{
Listener: l,
ID: k,
Keys: k,
Node: nod,
}); fails(e) {
t.FailNow()

19
pkg/engine/engine.go
View File

@@ -36,8 +36,8 @@ type Engine struct {
type Params struct {
tpt.Transport
Listener *transport.Listener
ID *crypto.Keys
*transport.Listener
*crypto.Keys
Node *node.Node
Nodes []*node.Node
NReturnSessions int
@@ -45,7 +45,7 @@ type Params struct {
func NewEngine(p Params) (c *Engine, e error) {
p.Node.Transport = p.Transport
p.Node.Identity = p.ID
p.Node.Identity = p.Keys
var ks *crypto.KeySet
if _, ks, e = crypto.NewSigner(); fails(e) {
return
@@ -119,6 +119,14 @@ func (ng *Engine) HandleMessage(s *splice.Splice, pr onions.Onion) {
}
}
}
func (ng *Engine) accept() <-chan *transport.Conn {
// Handle the absence of a listener gracefully.
if ng.Listener != nil {
return ng.Listener.Accept()
}
// This one cannot be sent to so will never select.
return make(chan *transport.Conn)
}
func (ng *Engine) Handler() (out bool) {
log.T.C(func() string {
@@ -130,6 +138,9 @@ func (ng *Engine) Handler() (out bool) {
ng.Shutdown()
out = true
break
case <-ng.accept():
// Add new connection.
case b := <-ng.Manager.ReceiveToLocalNode(0):
s := splice.Load(b, slice.NewCursor())
ng.HandleMessage(s, prev)
@@ -147,7 +158,7 @@ func (ng *Engine) Handler() (out bool) {
})
if !topUp {
ng.Manager.AddPendingPayment(p)
log.T.F("awaiting session keys for preimage %s session ID %s",
log.T.F("awaiting session keys for preimage %s session Keys %s",
p.Preimage, p.ID)
}
// For now if we received this we return true. Later this will wait with

2
pkg/engine/engine_test.go
View File

@@ -95,7 +95,7 @@ out:
t.Error("failed to receive expected message")
}
if id != idd {
t.Error("failed to receive expected message ID")
t.Error("failed to receive expected message Keys")
}
log.I.F("success\n\n")
wg.Done()

2
pkg/engine/mock.go
View File

@@ -39,7 +39,7 @@ func createNMockCircuits(inclSessions bool, nCircuits int,
nodes[i], _ = node.NewNode(addr, id, tpts[i], 50000)
if cl[i], e = NewEngine(Params{
Transport: tpts[i],
ID: id,
Keys: id,
Node: nodes[i],
NReturnSessions: nReturnSessions,
}); fails(e) {

2
pkg/engine/onions/balance_test.go
View File

@@ -35,7 +35,7 @@ func TestOnionSkins_Balance(t *testing.T) {
t.FailNow()
}
if ci.ID != id {
t.Error("ID did not decode correctly")
t.Error("Keys did not decode correctly")
t.FailNow()
}
if ci.MilliSatoshi != sats {

2
pkg/engine/onions/confirmation.go
View File

@@ -29,7 +29,7 @@ func (x *Confirmation) Magic() string { return ConfirmationMagic }
func (x *Confirmation) Encode(s *splice.Splice) (e error) {
// log.T.S("encoding", reflect.TypeOf(x),
// x.ID, x.Load,
// x.Keys, x.Load,
// )
s.Magic(ConfirmationMagic).ID(x.ID).Byte(x.Load)
return

2
pkg/engine/onions/confirmation_test.go
View File

@@ -34,7 +34,7 @@ func TestOnionSkins_Confirmation(t *testing.T) {
t.FailNow()
}
if ci.ID != id {
t.Error("ID did not decode correctly")
t.Error("Keys did not decode correctly")
t.FailNow()
}
if ci.Load != load {

2
pkg/engine/onions/crypt_test.go
View File

@@ -47,7 +47,7 @@ func TestOnionSkins_SimpleCrypt(t *testing.T) {
t.FailNow()
} else {
if cn.ID != n {
t.Error("did not get expected confirmation ID")
t.Error("did not get expected confirmation Keys")
t.FailNow()
}
}

2
pkg/engine/onions/exit_test.go
View File

@@ -55,7 +55,7 @@ func TestOnionSkins_Exit(t *testing.T) {
t.FailNow()
}
if ex.ID != id {
t.Error("ID did not decode correctly")
t.Error("Keys did not decode correctly")
t.FailNow()
}
for i := range ex.Ciphers {

2
pkg/engine/onions/getbalance_test.go
View File

@@ -51,7 +51,7 @@ func TestOnionSkins_GetBalance(t *testing.T) {
t.FailNow()
}
if ex.ID != id {
t.Error("ID did not decode correctly")
t.Error("Keys did not decode correctly")
t.FailNow()
}
for i := range ex.Ciphers {

2
pkg/engine/onions/introquery_test.go
View File

@@ -66,7 +66,7 @@ func TestOnionSkins_IntroQuery(t *testing.T) {
t.FailNow()
}
if ex.ID != id {
t.Error("ID did not decode correctly")
t.Error("Keys did not decode correctly")
t.FailNow()
}
}

4
pkg/engine/onions/response.go
View File

@@ -34,7 +34,7 @@ func (x *Response) GetOnion() interface{} { return x }
func (x *Response) Encode(s *splice.Splice) (e error) {
log.T.S("encoding", reflect.TypeOf(x),
// x.ID, x.Port, x.Load, x.Bytes.ToBytes(),
// x.Keys, x.Port, x.Load, x.Bytes.ToBytes(),
)
s.
Magic(ResponseMagic).
@@ -60,7 +60,7 @@ func (x *Response) Decode(s *splice.Splice) (e error) {
func (x *Response) Handle(s *splice.Splice, p Onion, ng Ngin) (e error) {
pending := ng.Pending().Find(x.ID)
log.T.F("searching for pending ID %s", x.ID)
log.T.F("searching for pending Keys %s", x.ID)
if pending != nil {
for i := range pending.Billable {
se := ng.Mgr().FindSession(pending.Billable[i])

2
pkg/engine/onions/response_test.go
View File

@@ -47,7 +47,7 @@ func TestOnionSkins_Response(t *testing.T) {
t.FailNow()
}
if rs.ID != id {
t.Error("ID did not decode correctly")
t.Error("Keys did not decode correctly")
t.FailNow()
}
if rs.Port != port {

8
pkg/engine/sess/sessionmanager.go
View File

@@ -125,7 +125,7 @@ func (sm *Manager) AddSession(s *sessions.Data) {
// check for dupes
for i := range sm.Sessions {
if sm.Sessions[i].Header.Bytes == s.Header.Bytes {
log.D.F("refusing to add duplicate session ID %x", s.Header.Bytes)
log.D.F("refusing to add duplicate session Keys %x", s.Header.Bytes)
return
}
}
@@ -248,7 +248,7 @@ func (sm *Manager) DeleteNodeAndSessions(id nonce.ID) {
sm.Lock()
defer sm.Unlock()
var exists bool
// If the node exists its ID is in the SessionCache.
// If the node exists its Keys is in the SessionCache.
if _, exists = sm.SessionCache[id]; !exists {
return
}
@@ -372,7 +372,7 @@ func (sm *Manager) FindNodeByIndex(i int) (no *node.Node) {
return sm.nodes[i]
}
// FindNodeByID searches for a Node by ID.
// FindNodeByID searches for a Node by Keys.
func (sm *Manager) FindNodeByID(i nonce.ID) (no *node.Node) {
sm.Lock()
defer sm.Unlock()
@@ -398,7 +398,7 @@ func (sm *Manager) FindNodeByAddrPort(id *netip.AddrPort) (no *node.Node) {
return
}
// DeleteNodeByID deletes a node identified by an ID.
// DeleteNodeByID deletes a node identified by an Keys.
func (sm *Manager) DeleteNodeByID(ii nonce.ID) (e error) {
sm.Lock()
defer sm.Unlock()

2
pkg/engine/sessions/sessions.go
View File

@@ -73,7 +73,7 @@ func NewSessionData(
}
h, p := hdr.Prv.ToBytes(), pld.Prv.ToBytes()
s = &Data{
// ID: id,
// Keys: id,
Node: node,
Remaining: rem,
Header: hdr,

18
pkg/holo/holo.go
View File

@@ -1,18 +0,0 @@
package holo
import (
"git-indra.lan/indra-labs/indra"
log2 "git-indra.lan/indra-labs/indra/pkg/proc/log"
)
var (
log = log2.GetLogger(indra.PathBase)
fails = log.E.Chk
)
type Set struct {
}
func New() *Set {
return &Set{}
}

101
pkg/holo/holo_test.go
View File

@@ -1,101 +0,0 @@
package holo
import (
"encoding/binary"
"fmt"
"math/rand"
"strconv"
"testing"
"time"
"github.com/jbarham/primegen"
log2 "git-indra.lan/indra-labs/indra/pkg/proc/log"
)
func DecodeUint64(b []byte) uint64 { return get64(b) }
var put64 = binary.LittleEndian.PutUint64
var get64 = binary.LittleEndian.Uint64
func Shuffle(l int, fn func(i, j int)) {
rand.Seed(time.Now().Unix())
rand.Shuffle(l, fn)
}
func TestNew(t *testing.T) {
log2.SetLogLevel(log2.Trace)
primer := primegen.New()
counter := 0
var prime uint8 = 1
var primes []uint8
// for i := 2; prime < 72; i++ {
for i := 2; len(primes) < 10; i++ {
prime = uint8(primer.Next())
primes = append(primes, prime)
counter++
}
fmt.Println(primes)
fmt.Printf("\nprime factors addded\n\n")
fmt.Printf(
"|index|prime|cumulative product|binary|product |64 bit prod |64 bit" +
" binary prod | and" +
" | | |\n")
fmt.Printf(
"|-----|-----|------------------|------|--------|------------|----------------------------------|---------|----|----------------------------------|\n")
prod := primes[0]
prod6 := uint64(primes[0])
var prevs []uint8
for i := range primes {
prevs = append(prevs, prod)
prod *= primes[i]
prod6 *= uint64(primes[i])
fmt.Printf("|%5d|%5d|%18d|%6s|%8s|%12d|%34s|%9s|%4d|%34s|\n",
i,
primes[i],
prod,
strconv.FormatInt(int64(primes[i]), 2),
strconv.FormatInt(int64(prod), 2),
prod6,
strconv.FormatInt(int64(prod6), 2),
strconv.FormatInt(int64(uint64(prod)&prod6), 2),
uint64(prod)&prod6,
strconv.FormatInt(int64(uint64(prod)|prod6), 2),
)
}
prevs = append(prevs, prod)
for i := range primes {
if i >= len(primes)/2 {
continue
}
primes[i], primes[len(primes)-1-i] = primes[len(primes)-1-i], primes[i]
prevs[i], prevs[len(prevs)-1-i] = prevs[len(prevs)-1-i],
prevs[i]
}
// fmt.Println(prevs)
// prevs = prevs[1:]
last := prevs[0]
// fmt.Println(prevs)
fmt.Printf("\nprime factors removed\n\n")
fmt.Printf("|index|prime| cumulative product|\n")
fmt.Printf("|-----|-----|--------------------|\n")
var m, n uint8
_ = n
for i := range primes {
if i == 0 {
continue
}
fmt.Printf("|%5d|%5d|%20d|%20d\n",
len(primes)-i-1, primes[i], last, m)
if m != 0 {
// log.D.Ln("remainder not zero!")
// return
}
if m == 0 {
}
}
}

160
pkg/holo/readme.md
View File

@@ -1,160 +0,0 @@
# Holo
Holo is an algorithm for storing large amounts of text using prime numbers
and powers as graph vectors.
Conventional programming for boolean value storage comes in two flavours,
the C style 1 byte 1 value, and the older style assembler bit flag filter.
The efficiency of extracting the value is lower for a bitfield, because you
have to copy the byte and then perform an AND using the true value for the flag.
The byte, like as used in Go, is just a comparison to the true value 1 and
only uses one instruction, where the bitfield must first mask and then
compare to zero.
Except for embedded applications, there usually is not much use for the
trivial memory savings for a relatively small number of values a program can
need that store a true/false value.
## Text Compression
It is well known that text content has a very high level of redundancy in
its structure, and of course most compression algorithms do a pretty good
job with them, but the encoding systems are not built to allow you to
traverse the document one symbol at a time, they just spit the reconstructed
version at whatever file you point them at.
Holo is designed *only* for text, performs a greedy, smaller up to larger
byte length scanning of a chunk of data and generates a dictionary of 256
bit holo vectors associated with each of the dictionary entries, and it
expects text like structure with whitespace, symbols, numbers and letters as
the 4 categories of byte types. Spaces after a symbol are stored also in the
filter, as are paragraph boundaries.
## Prime Numbers as Compact Bitfields
The specific property required for a bit field is that altering one bit in a
field has no impact on any other bit. Thus, the traditional AND masking and
zero comparison for boolean values satisfies this, as flipping a bit does
not alter the other bits. Obviously byte-as-boolean can be operated on with
a single comparison and no bitwise masking, faster, but more memory used.
If your data involves thousands or millions of binary values, the
conventional idea is you are gonna have to use up a few hundred kilobytes
and a big map of symbols to bit positions.
But there is a better way.
The strange and interesting thing about prime numbers is that they can be
composited together in a single number, and all primes except 1 can be
multiplied into your number, and performing a modulo division (the remainder)
with a given prime number yields no remainder if it is present, or a nonzero
remainder if it is not.
There is millions of prime numbers, and way way more than you could ever
need in a 256 bit number. Even a 64 bit number can give you a lot of prime
values all squished together in one little double long word.
## More than Binary
The presence or absence of a particular prime factor can be more than just a
true/false value. It is also possible to exponentiate these values, and they
can then, in addition to the simple boolean present/not present option,
there can be also 1, 2, and more values at each prime number, essentially
enabling also ternary, quaternary, etc values, limited by the precision of
the numbers and the size of the prime number compared to the size of the bit
field.
So, in addition to having a present/missing state for a ridiculously large
number of distinct binary values, they can be multi-state.
It seems incredible but it is possible to thereby store enormous amounts of
data in these fields.
## The Inspiration
Pondering upon the architecture of Transformer style text generators like
OpenAI's GPT, I had the idea that somehow one could store a document as a
dictionary from its distinct and longest repeating sequences, and the lists
of positions of the repeating symbol elements in the text that can be used
to seek forward and backward in the original text from which the holo matrix
is derived, by following the document symbol positions referred to in the
256 bit number fields storing each symbol's set of next symbols.
The prime fields can be done over other sizes than 256 bits, but 256 bits is
necessary to allow enough headroom above the prime numbers to prevent an
overflow and loss of data, for compressing large volumes of text.
Most texts contain less than 10,000 distinct words and about half of them
are repeated several times in a line. So if the average word length is about
5 characters, the symbol part of the dictionary is only going to be a few
hundred kilobytes and up to a few megabytes for a really broad and large
text selection or one containing multiple languages.
Because Holo is aimed at very large text compression, it assumes that all
words are space or symbol separated, and uses some of the primes to store
pre- and post- whitespace, tabs or newlines in the word "symbol" dictionary
prime field.
A document has a first symbol, and from this first symbol one can
reconstitute the original document, with those structuring assumptions based
on text.
The prime field for each symbol, after the first 6 primes 2, 3, 5, 7, 11,
13, which represent space, tab, linefeed before, and after, are a coordinate
space based on the sequence of prime numbers mapped to normal ordinals, each
other symbol in the dictionary represented by the first row, and the
sequence number of these primes corresponds to each symbol in the dictionary.
Then there is more rows at each multiple of the dictionary's entry size,
these subsequent rows representing the sequence of outbound links to
subsequent symbols in the document. As the Holo iterator walks the document,
it must keep a track of the prime sequence number along the path to enable
reverse. The symbol found at a given document symbols position should be
cached every 100th symbol or so, and after the first walkthrough fast random
seeking becomes possible.
An interesting thing about Holo also is that it can be edited, since all
that is required is to change what symbol a given document position contains,
inserting is non-destructive though one must add the branch to the inserted
new symbol, and it must then point back to the previous subsequent symbol.
It is also possible to add or remove symbols, though this requires a full
iteration.
## Planned Uses
Holo, in theory, should be able to compress large bodies of text massively,
while retaining reasonably efficient seek times. So, for example, an entire
program application source code could be stored in a Holo Grid.
The interrelations between symbols, the raw frequency information of the
sequences, alone, provides a minimal statistical probability for one word to
follow another, this is simple to discover and enumerate, although it
requires a full walk of the path.
Grammar information is indirectly stored also, because the majority of
sequences of symbols point to a next symbol that is of a given part of
speech, like for example how the definite article "The", is usually followed
by a noun, or a composite noun with chains of nouns adjectives and nouns.
Alone, by the weightings of the links from one symbol to another in the
document sequence, one can greatly thin out the options for a valid next
word given a current word.
The grammar, the patterns of different parts of speech found in a language,
form repeating sequences of symbols, after tokenising the text, can be
searched in the same way as the sequences of bytes are used to gather the
words, whitespace, symbols and numbers, with multiple symbols repeating
sequence.
The compression of multiple symbols both saves on space and provides a graph
between the words that not only shows the probability of any given
subsequent symbol (word), but can show that a series of longer sequences
that have partial matches with things like "The X of Y", this prime field is
thus storing enough information that it can take an input, generate its own
independent dictionary and path vectors, and then show you a list of the
words that it has observed in a repeating partial match fragment, and by
inference, determine that the symbols that differ between the two otherwise
similar patterns are the same part of speech.

6
pkg/util/splice/eng_splice.go
View File

@@ -124,7 +124,7 @@ func (s *Splice) String() (o string) {
oo = string(tmp)
o += color.LightGreen.Sprint(" ", oo)
}
if prevString == "ID" {
if prevString == "Keys" {
var oo string
var e error
if oo, e = based32.Codec.Encode(v.ToBytes()); fails(e) {
@@ -258,14 +258,14 @@ func (s *Splice) ReadMagic(magic *string) *Splice {
func (s *Splice) ID(id nonce.ID) *Splice {
copy(s.b[*s.c:s.c.Inc(nonce.IDLen)], id[:])
s.Segments = append(s.Segments,
NameOffset{Offset: int(*s.c), Name: "ID"})
NameOffset{Offset: int(*s.c), Name: "Keys"})
return s
}
func (s *Splice) ReadID(id *nonce.ID) *Splice {
copy((*id)[:], s.b[*s.c:s.c.Inc(nonce.IDLen)])
s.Segments = append(s.Segments,
NameOffset{Offset: int(*s.c), Name: "ID"})
NameOffset{Offset: int(*s.c), Name: "Keys"})
return s
}