feat: Add persistent cache for the contract's data #225

src/__tests__/regression/read-state.test.ts

@@ -60,7 +60,10 @@ describe.each(chunked)('v1 compare.suite %#', (contracts: string[]) => {
           },
           'mainnet'
         )
-          .useWarpGateway({ ...defaultWarpGwOptions, source: SourceType.ARWEAVE, confirmationStatus: null })
+          .useWarpGateway({...defaultWarpGwOptions, source: SourceType.ARWEAVE, confirmationStatus: null}, {
+            ...defaultCacheOptions,
+            inMemory: true
+          })
           .build()
           .contract(contractTxId)
           .setEvaluationOptions({
@@ -97,7 +100,10 @@ describe.each(chunkedVm)('v1 compare.suite (VM2) %#', (contracts: string[]) => {
         },
         'mainnet'
       )
-        .useWarpGateway({ ...defaultWarpGwOptions, source: SourceType.ARWEAVE, confirmationStatus: null })
+        .useWarpGateway({...defaultWarpGwOptions, source: SourceType.ARWEAVE, confirmationStatus: null}, {
+          ...defaultCacheOptions,
+          inMemory: true
+        })
         .build()
         .contract(contractTxId)
         .setEvaluationOptions({
@@ -130,7 +136,10 @@ describe.each(chunkedGw)('gateways compare.suite %#', (contracts: string[]) => {
         },
         'mainnet'
       )
-        .useWarpGateway({ ...defaultWarpGwOptions, source: SourceType.ARWEAVE, confirmationStatus: null })
+        .useWarpGateway({...defaultWarpGwOptions, source: SourceType.ARWEAVE, confirmationStatus: null}, {
+          ...defaultCacheOptions,
+          inMemory: true
+        })
         .build();
       const result = await warpR
         .contract(contractTxId)
@@ -182,7 +191,10 @@ describe('readState', () => {
       },
       'mainnet'
     )
-      .useWarpGateway({ ...defaultWarpGwOptions, source: SourceType.ARWEAVE, confirmationStatus: null })
+      .useWarpGateway({...defaultWarpGwOptions, source: SourceType.ARWEAVE, confirmationStatus: null}, {
+        ...defaultCacheOptions,
+        inMemory: true
+      })
       .build()
       .contract(contractTxId)
       .setEvaluationOptions({
@@ -210,7 +222,10 @@ describe('readState', () => {
       },
       'mainnet'
     )
-      .useWarpGateway({ ...defaultWarpGwOptions, source: SourceType.ARWEAVE, confirmationStatus: null })
+      .useWarpGateway({...defaultWarpGwOptions, source: SourceType.ARWEAVE, confirmationStatus: null}, {
+        ...defaultCacheOptions,
+        inMemory: true
+      })
       .build()
       .contract(contractTxId)
       .connect(jwk)

src/cache/impl/LevelDbCache.ts

@@ -22,7 +22,7 @@ export class LevelDbCache<V = any> implements SortKeyCache<V> {
    * and there doesn't seem to be any public interface/abstract type for all Level implementations
    * (the AbstractLevel is not exported from the package...)
    */
-  private db: MemoryLevel;
+  private readonly db: MemoryLevel;
 
   constructor(cacheOptions: CacheOptions) {
     if (cacheOptions.inMemory) {

src/core/WarpBuilder.ts

@@ -13,7 +13,7 @@ import { WarpGatewayInteractionsLoader } from './modules/impl/WarpGatewayInterac
 import { InteractionsLoader } from './modules/InteractionsLoader';
 import { StateEvaluator, EvalStateResult } from './modules/StateEvaluator';
 import { WarpEnvironment, Warp } from './Warp';
-import { GatewayOptions } from './WarpFactory';
+import { CacheOptions, GatewayOptions } from './WarpFactory';
 
 export class WarpBuilder {
   private _definitionLoader?: DefinitionLoader;
@@ -55,7 +55,7 @@ export class WarpBuilder {
     return this.build();
   }
 
-  public useWarpGateway(gatewayOptions: GatewayOptions): WarpBuilder {
+  public useWarpGateway(gatewayOptions: GatewayOptions, cacheOptions: CacheOptions): WarpBuilder {
     this._interactionsLoader = new CacheableInteractionsLoader(
       new WarpGatewayInteractionsLoader(
         gatewayOptions.address,
@@ -66,13 +66,13 @@ export class WarpBuilder {
     this._definitionLoader = new WarpGatewayContractDefinitionLoader(
       gatewayOptions.address,
       this._arweave,
-      new MemCache()
+      cacheOptions
     );
     return this;
   }
 
   public useArweaveGateway(): WarpBuilder {
-    this._definitionLoader = new ContractDefinitionLoader(this._arweave, new MemCache());
+    this._definitionLoader = new ContractDefinitionLoader(this._arweave);
     this._interactionsLoader = new CacheableInteractionsLoader(
       new ArweaveGatewayInteractionsLoader(this._arweave, this._environment)
     );

src/core/WarpFactory.ts

@@ -138,6 +138,6 @@ export class WarpFactory {
     cacheOptions: CacheOptions = defaultCacheOptions,
     environment: WarpEnvironment
   ): Warp {
-    return this.custom(arweave, cacheOptions, environment).useWarpGateway(gatewayOptions).build();
+    return this.custom(arweave, cacheOptions, environment).useWarpGateway(gatewayOptions, cacheOptions).build();
   }
 }

src/core/modules/impl/ContractDefinitionLoader.ts

@@ -19,23 +19,14 @@ export class ContractDefinitionLoader implements DefinitionLoader {
 
   protected arweaveWrapper: ArweaveWrapper;
 
-  constructor(
+  constructor(private readonly arweave: Arweave) {
-    private readonly arweave: Arweave,
-    // TODO: cache should be removed from the core layer and implemented in a wrapper of the core implementation
-    protected readonly cache?: WarpCache<string, ContractDefinition<unknown>>
-  ) {
     this.arweaveWrapper = new ArweaveWrapper(arweave);
   }
 
   async load<State>(contractTxId: string, evolvedSrcTxId?: string): Promise<ContractDefinition<State>> {
-    if (!evolvedSrcTxId && this.cache?.contains(contractTxId)) {
-      this.logger.debug('ContractDefinitionLoader: Hit from cache!');
-      return Promise.resolve(this.cache?.get(contractTxId) as ContractDefinition<State>);
-    }
     const benchmark = Benchmark.measure();
     const contract = await this.doLoad<State>(contractTxId, evolvedSrcTxId);
     this.logger.info(`Contract definition loaded in: ${benchmark.elapsed()}`);
-    this.cache?.put(contractTxId, contract);
 
     return contract;
   }

src/core/modules/impl/WarpGatewayContractDefinitionLoader.ts

@@ -13,6 +13,9 @@ import { stripTrailingSlash } from '../../../utils/utils';
 import { DefinitionLoader } from '../DefinitionLoader';
 import { WarpCache } from '../../../cache/WarpCache';
 import { WasmSrc } from './wasm/WasmSrc';
+import { CacheOptions } from '../../WarpFactory';
+import { MemoryLevel } from 'memory-level';
+import { Level } from 'level';
 
 /**
  * An extension to {@link ContractDefinitionLoader} that makes use of
@@ -26,26 +29,55 @@ export class WarpGatewayContractDefinitionLoader implements DefinitionLoader {
   private readonly rLogger = LoggerFactory.INST.create('WarpGatewayContractDefinitionLoader');
   private contractDefinitionLoader: ContractDefinitionLoader;
   private arweaveWrapper: ArweaveWrapper;
+  private readonly db: MemoryLevel<string, any>;
 
   constructor(
     private readonly baseUrl: string,
     arweave: Arweave,
-    private readonly cache?: WarpCache<string, ContractDefinition<unknown>>
+    cacheOptions: CacheOptions
   ) {
     this.baseUrl = stripTrailingSlash(baseUrl);
-    this.contractDefinitionLoader = new ContractDefinitionLoader(arweave, cache);
+    this.contractDefinitionLoader = new ContractDefinitionLoader(arweave);
     this.arweaveWrapper = new ArweaveWrapper(arweave);
+
+    if (cacheOptions.inMemory) {
+      this.db = new MemoryLevel<string, any>({ valueEncoding: 'json' });
+    } else {
+      if (!cacheOptions.dbLocation) {
+        throw new Error('LevelDb cache configuration error - no db location specified');
+      }
+      const dbLocation = cacheOptions.dbLocation;
+      this.db = new Level<string, any>(`${dbLocation}/contracts`, { valueEncoding: 'json' });
+    }
   }
 
   async load<State>(contractTxId: string, evolvedSrcTxId?: string): Promise<ContractDefinition<State>> {
-    if (!evolvedSrcTxId && this.cache?.contains(contractTxId)) {
+    let cacheKey = contractTxId;
+    if (evolvedSrcTxId) {
+      cacheKey += `_${evolvedSrcTxId}`;
+    }
+
+    let cacheResult = null;
+    try {
+      cacheResult = await this.db.get(cacheKey);
+    } catch (e: any) {
+      if (e.code == 'LEVEL_NOT_FOUND') {
+        cacheResult = null;
+      } else {
+        throw e;
+      }
+    }
+    if (cacheResult) {
       this.rLogger.debug('WarpGatewayContractDefinitionLoader: Hit from cache!');
-      return Promise.resolve(this.cache?.get(contractTxId) as ContractDefinition<State>);
+      if (cacheResult.contractType == 'wasm') {
+        cacheResult.srcBinary = Buffer.from(cacheResult.srcBinary.data);
+      }
+      return cacheResult;
     }
     const benchmark = Benchmark.measure();
     const contract = await this.doLoad<State>(contractTxId, evolvedSrcTxId);
     this.rLogger.info(`Contract definition loaded in: ${benchmark.elapsed()}`);
-    this.cache?.put(contractTxId, contract);
+    await this.db.put(cacheKey, contract);
 
     return contract;
   }

tools/contract.ts

@@ -31,27 +31,9 @@ async function main() {
     logging: false // Enable network request logging
   });
 
-  // Base Contract - e.g. a PST in a separate library
+  const warp = WarpFactory.forMainnet({...defaultCacheOptions, inMemory: false});
-  const BaseContract = {
-    handle: function init(state, action) {
-      //blah blah
-    }
-  };
-
-  // You own contract, that imports the base contract
-  (function() {
-    const baseHandle = BaseContract.handle;
-    BaseContract.handle = handle;
-
-    function handle(state, action) {
-      baseHandle.call(BaseContract, [state, action]);
-      // you contract's additional code.
-    }
-  })();
-
-  const warp = WarpFactory.forMainnet({...defaultCacheOptions, inMemory: true});
   try {
-    const contract = warp.contract("6VHC739Bry-PuwE9xxYuPgj6HK75MQKCWyo8yf2fi-k");
+    const contract = warp.contract("5Yt1IujBmOm1LSux9KDUTjCE7rJqepzP7gZKf_DyzWI");
     const cacheResult = await contract
       .setEvaluationOptions({
         allowBigInt: true

tools/deploytest.ts

@@ -17,7 +17,7 @@ async function main() {
   });
 
   try {
-    const warp = WarpFactory.forMainnet({...defaultCacheOptions, inMemory: true});
+    const warp = WarpFactory.forMainnet({...defaultCacheOptions, inMemory: false});
     /*const warp = WarpFactory
       .custom(arweave, {
         ...defaultCacheOptions,
@@ -91,6 +91,8 @@ async function main() {
     logger.info("Result", cachedValue.state);
     logger.info("Validity", cachedValue.validity);
 
+    const result2 = await contract.readState();
+
   } catch (e) {
     logger.error(e)
 

tools/leveldb.ts

@@ -2,84 +2,31 @@
 import {Level} from "level";
 
 import { MemoryLevel } from 'memory-level';
+import fs from "fs";
+import {WasmSrc} from "../src";
+import {Buffer} from "buffer";
 
 // Create a database
 
 
 async function test() {
   //const db = new Level<string, any>('./leveldb', {valueEncoding: 'json'});
-  const db = new MemoryLevel({ valueEncoding: 'json' })
+  const db = new MemoryLevel<string, any>({ valueEncoding: 'json' });
+  const wasmSrc = fs.readFileSync('./tools/data/rust/rust-pst_bg.wasm');
 
-  const contractA = db.sublevel<string, any>('n05LTiuWcAYjizXAu-ghegaWjL89anZ6VdvuHcU6dno', {valueEncoding: 'json'});
+  const contractData = {
-  const contractB = db.sublevel<string, any>('NwaSMGCdz6Yu5vNjlMtCNBmfEkjYfT-dfYkbQQDGn5s', {valueEncoding: 'json'});
+    src: wasmSrc,
-
+    id: 'n05LTiuWcAYjizXAu-ghegaWjL89anZ6VdvuHcU6dno',
-  await contractA.put("sort_key_01a", {state: "sort_key_01a"});
+    srcId: 'foobar'
-  await contractA.put("sort_key_01b", {state: "sort_key_01b"});
-  await contractA.put("sort_key_02c", {state: "sort_key_02c"});
-  await contractA.put("sort_key_03d", {state: "sort_key_03d"});
-
-  await contractB.put("sort_key_01e", {state: "sort_key_01e"});
-  await contractB.put("sort_key_01f", {state: "sort_key_01f"});
-  await contractB.put("sort_key_02g", {state: "sort_key_02g"});
-  await contractB.put("sort_key_03h", {state: "sort_key_03h"});
-
-  /*for await (const value of contractA.values({lt: '02g'})) {
-    console.log(value)
   }
 
-  console.log("state: " + (await contractB.get('03h')).state);
+  console.log(contractData);
-
-  try {
-    (await contractB.get('06h'));
-  } catch (e: any) {
-    console.log(e.code);
-  }
-
-  const keys = await contractB.keys({reverse: true, limit: 1}).all();
-  console.log(keys.length);
-  console.log(keys);
 
 
-  db.iterator().seek('zzz');
+  await db.put("n05LTiuWcAYjizXAu-ghegaWjL89anZ6VdvuHcU6dno", contractData);
-  const result = await db.iterator().next();
+  const result = await db.get("n05LTiuWcAYjizXAu-ghegaWjL89anZ6VdvuHcU6dno");
-  console.log('result last', result);
+  console.log(result);
-
+  console.log(Buffer.from(result.src.data));
-  const entries = await db.iterator({ limit: 10, reverse: true }).all()
-
-  console.log('last entries');
-  for (const [key, value] of entries) {
-    console.log(key);
-  }*/
-
-  //console.log(contractA.prefix) // '!example!'
-
-
-  const contracts = [];
-
-  let lastSortKey = '';
-
-
-  /*for (const key of await db.keys().all()) {
-    console.log(key);
-    const sortKey = key.substring(45);
-    console.log(key.substring(45));
-    if (sortKey.localeCompare(lastSortKey) > 0) {
-      lastSortKey = sortKey;
-    }
-  }
-
-  console.log({lastSortKey});*/
-
-  const keys = await db.keys().all();
-
-  const result = new Set<string>();
-  keys.forEach((k) => result.add(k.substring(1, 44)));
-
-  console.log(Array.from(result));
-
-  // returns sub-levels - i.e. contracts
-  // console.log(keys.map((k) => k.substring(1, 44)));
-
 }
 
 test();