fix: FileBlockHeightSwCache does not respect maxStoredInMemoryBlockHeights while loading cache files

src/__tests__/regression/read-state.test.ts

@@ -36,7 +36,7 @@ describe.each(chunked)('.suite %#', (contracts: string[]) => {
       const result = await readContract(arweave, contractTxId);
       const resultString = JSON.stringify(result).trim();
       console.log('readState', contractTxId);
-      const result2 = await SmartWeaveNodeFactory.memCached(arweave, 5).contract(contractTxId).readState();
+      const result2 = await SmartWeaveNodeFactory.memCached(arweave, 1).contract(contractTxId).readState();
       const result2String = JSON.stringify(result2.state).trim();
       expect(result2String).toEqual(resultString);
     },
@@ -51,7 +51,7 @@ describe('readState', () => {
     const result = await readContract(arweave, contractTxId, blockHeight);
     const resultString = JSON.stringify(result).trim();
 
-    const result2 = await SmartWeaveNodeFactory.memCached(arweave, 5)
+    const result2 = await SmartWeaveNodeFactory.memCached(arweave, 1)
       .contract(contractTxId)
       .setEvaluationOptions({ updateCacheForEachInteraction: false })
       .readState(blockHeight);
@@ -68,7 +68,7 @@ describe('readState', () => {
       target: '6Z-ifqgVi1jOwMvSNwKWs6ewUEQ0gU9eo4aHYC3rN1M'
     });
 
-    const v2Result = await SmartWeaveNodeFactory.memCached(arweave, 5)
+    const v2Result = await SmartWeaveNodeFactory.memCached(arweave, 1)
       .contract(contractTxId)
       .setEvaluationOptions({ updateCacheForEachInteraction: false })
       .connect(jwk)

src/cache/impl/FileBlockHeightCache.ts

@@ -73,9 +73,11 @@ export class FileBlockHeightSwCache<V = any> extends MemBlockHeightSwCache<V> {
         const height = file.split('.')[0];
         // FIXME: "state" and "validity" should be probably split into separate json files
         const cacheValue = JSON.parse(fs.readFileSync(path.join(cacheFilePath), 'utf-8'));
-        this.storage[contract].set(+height, cacheValue);
+
+        this.putSync({ cacheKey: contract, blockHeight: +height }, cacheValue);
       });
       this.fLogger.info(`loading cache for ${contract}`, benchmark.elapsed());
+      this.fLogger.debug(`Amount of elements loaded for ${contract} to mem: ${this.storage[contract].size}`);
     });
     this.fLogger.debug('Storage keys', this.storage);
 

src/cache/impl/KnexStateCache.ts

@@ -58,8 +58,8 @@ export class KnexStateCache extends MemBlockHeightSwCache<StateCache<any>> {
   ): Promise<KnexStateCache> {
     if (!(await knex.schema.hasTable('states'))) {
       await knex.schema.createTable('states', (table) => {
-        table.string('contract_id', 64).notNullable();
-        table.bigInteger('height').notNullable();
+        table.string('contract_id', 64).notNullable().index();
+        table.bigInteger('height').notNullable().index();
         table.string('hash').notNullable().unique();
         table.json('state').notNullable();
         table.unique(['contract_id', 'height', 'hash'], { indexName: 'states_composite_index' });

src/cache/impl/MemBlockHeightCache.ts

@@ -58,12 +58,14 @@ export class MemBlockHeightSwCache<V = any> implements BlockHeightSwCache<V> {
       this.storage[cacheKey] = new Map();
     }
     const cached = this.storage[cacheKey];
-    if (cached.size == this.maxStoredBlockHeights) {
+    if (cached.size >= this.maxStoredBlockHeights) {
       const toRemove = [...cached.keys()].sort(asc).shift();
       cached.delete(toRemove);
     }
 
-    cached.set(blockHeight, deepCopy(value));
+    // note: "value" should be deep copied here for safety
+    // but it significantly degrades overall performance...
+    cached.set(blockHeight, value);
   }
 
   async contains(key: string): Promise<boolean> {

src/core/SmartWeaveBuilder.ts

@@ -33,7 +33,7 @@ export class SmartWeaveBuilder {
 
   public setCacheableInteractionsLoader(
     value: InteractionsLoader,
-    maxStoredInMemoryBlockHeights: number = Number.MAX_SAFE_INTEGER
+    maxStoredInMemoryBlockHeights = 1
   ): SmartWeaveBuilder {
     this._interactionsLoader = new CacheableContractInteractionsLoader(
       value,

src/core/modules/StateEvaluator.ts

@@ -2,7 +2,7 @@ import { BlockHeightCacheResult, CurrentTx, ExecutionContext, GQLNodeInterface }
 
 /**
  * Implementors of this class are responsible for evaluating contract's state
- * - based on the execution context.
+ * - based on the {@link ExecutionContext}.
  */
 export interface StateEvaluator {
   eval<State>(executionContext: ExecutionContext<State>, currentTx: CurrentTx[]): Promise<EvalStateResult<State>>;
@@ -60,11 +60,6 @@ export interface StateEvaluator {
     contractTxId: string,
     blockHeight: number
   ): Promise<BlockHeightCacheResult<EvalStateResult<State>> | null>;
-
-  transactionState<State>(
-    transaction: GQLNodeInterface,
-    contractTxId: string
-  ): Promise<EvalStateResult<State> | undefined>;
 }
 
 export class EvalStateResult<State> {
@@ -86,8 +81,6 @@ export class DefaultEvaluationOptions implements EvaluationOptions {
 
   waitForConfirmation = false;
 
-  fcpOptimization = false;
-
   updateCacheForEachInteraction = true;
 
   internalWrites = false;
@@ -110,9 +103,6 @@ export interface EvaluationOptions {
   // you will know, when the new interaction is effectively available on the network
   waitForConfirmation: boolean;
 
-  // experimental optimization for contracts that utilize the Foreign Call Protocol
-  fcpOptimization: boolean;
-
   // whether cache should be updated after evaluating each interaction transaction.
   // this can be switched off to speed up cache writes (ie. for some contracts (with flat structure)
   // and caches it maybe more suitable to cache only after state has been fully evaluated)

src/core/modules/impl/CacheableStateEvaluator.ts

@@ -150,10 +150,7 @@ export class CacheableStateEvaluator extends DefaultStateEvaluator {
       return null;
     }
 
-    return new BlockHeightCacheResult<EvalStateResult<State>>(
-      stateCache.cachedHeight,
-      [...stateCache.cachedValue].pop()
-    );
+    return new BlockHeightCacheResult<EvalStateResult<State>>(stateCache.cachedHeight, stateCache.cachedValue);
   }
 
   async onInternalWriteStateUpdate<State>(
@@ -178,23 +175,6 @@ export class CacheableStateEvaluator extends DefaultStateEvaluator {
     //await this.putInCache(executionContext.contractDefinition.txId, transaction, state);
   }
 
-  async transactionState<State>(
-    transaction: GQLNodeInterface,
-    contractTxId: string
-  ): Promise<EvalStateResult<State> | undefined> {
-    const stateCache = (await this.cache.get(contractTxId, transaction.block.height)) as BlockHeightCacheResult<
-      StateCache<State>
-    >;
-
-    if (stateCache == null) {
-      return undefined;
-    }
-
-    return stateCache.cachedValue.find((sc) => {
-      return sc.transactionId === transaction.id;
-    });
-  }
-
   protected async putInCache<State>(
     contractTxId: string,
     transaction: GQLNodeInterface,
@@ -205,19 +185,8 @@ export class CacheableStateEvaluator extends DefaultStateEvaluator {
     }
     const transactionId = transaction.id;
     const blockHeight = transaction.block.height;
-
     const stateToCache = new EvalStateResult(state.state, state.validity, transactionId, transaction.block.id);
 
-    // we do not return a deepCopy here - as this operation significantly (2-3x) degrades performance
-    // for contracts with multiple interactions on single block height
-    const stateCache = await this.cache.get(contractTxId, blockHeight, false);
-    if (stateCache != null) {
-      // note: since we're not returning deepCopy of the cached array in this case
-      // - there is no need to put the updated array in the cache manually (ie. calling this.cache.put())
-      // - as we're operating on the reference.
-      stateCache.cachedValue.push(stateToCache);
-    } else {
-      await this.cache.put(new BlockHeightKey(contractTxId, blockHeight), [stateToCache]);
-    }
+    await this.cache.put(new BlockHeightKey(contractTxId, blockHeight), stateToCache);
   }
 }

src/core/modules/impl/DefaultStateEvaluator.ts

@@ -59,7 +59,7 @@ export abstract class DefaultStateEvaluator implements StateEvaluator {
     const { contract, contractDefinition, sortedInteractions } = executionContext;
 
     let currentState = baseState.state;
-    let validity = deepCopy(baseState.validity);
+    const validity = baseState.validity;
 
     this.logger.info(
       `Evaluating state for ${contractDefinition.txId} [${missingInteractions.length} non-cached of ${sortedInteractions.length} all]`
@@ -82,7 +82,6 @@ export abstract class DefaultStateEvaluator implements StateEvaluator {
       );
 
       // verifying whether state isn't already available for this exact interaction.
-      const state = await this.transactionState<State>(interactionTx, contractDefinition.txId);
       const isInteractWrite = this.tagsParser.isInteractWrite(missingInteraction, contractDefinition.txId);
 
       this.logger.debug('interactWrite?:', isInteractWrite);
@@ -107,10 +106,10 @@ export abstract class DefaultStateEvaluator implements StateEvaluator {
         this.logger.debug('Reading state of the calling contract', interactionTx.block.height);
 
         /**
-        Reading the state of the writing contract.
-        This in turn will cause the state of THIS contract to be
-        updated in cache - see {@link ContractHandlerApi.assignWrite}
-        */
+         Reading the state of the writing contract.
+         This in turn will cause the state of THIS contract to be
+         updated in cache - see {@link ContractHandlerApi.assignWrite}
+         */
         await writingContract.readState(interactionTx.block.height, [
           ...(currentTx || []),
           {
@@ -164,7 +163,7 @@ export abstract class DefaultStateEvaluator implements StateEvaluator {
           caller: interactionTx.owner.address
         };
 
-        let intermediaryCacheHit = false;
+        const intermediaryCacheHit = false;
 
         const interactionData = {
           interaction,
@@ -176,35 +175,28 @@ export abstract class DefaultStateEvaluator implements StateEvaluator {
 
         const interactionCall: InteractionCall = contract.getCallStack().addInteractionData(interactionData);
 
-        if (state) {
-          this.logger.debug('Found in cache');
-          intermediaryCacheHit = true;
-          currentState = state.state;
-          validity = state.validity;
-        } else {
-          const result = await executionContext.handler.handle(
-            executionContext,
-            new EvalStateResult(currentState, validity),
-            interactionData
-          );
-          errorMessage = result.errorMessage;
+        const result = await executionContext.handler.handle(
+          executionContext,
+          new EvalStateResult(currentState, validity),
+          interactionData
+        );
+        errorMessage = result.errorMessage;
 
-          this.logResult<State>(result, interactionTx, executionContext);
+        this.logResult<State>(result, interactionTx, executionContext);
 
-          if (result.type === 'exception' && ignoreExceptions !== true) {
-            throw new Error(`Exception while processing ${JSON.stringify(interaction)}:\n${result.errorMessage}`);
-          }
-
-          validity[interactionTx.id] = result.type === 'ok';
-          currentState = result.state;
-
-          // cannot simply take last element of the missingInteractions
-          // as there is no certainty that it has been evaluated (e.g. issues with input tag).
-          lastEvaluatedInteraction = interactionTx;
-
-          this.logger.debug('Interaction evaluation', singleInteractionBenchmark.elapsed());
+        if (result.type === 'exception' && ignoreExceptions !== true) {
+          throw new Error(`Exception while processing ${JSON.stringify(interaction)}:\n${result.errorMessage}`);
         }
 
+        validity[interactionTx.id] = result.type === 'ok';
+        currentState = result.state;
+
+        // cannot simply take last element of the missingInteractions
+        // as there is no certainty that it has been evaluated (e.g. issues with input tag).
+        lastEvaluatedInteraction = interactionTx;
+
+        this.logger.debug('Interaction evaluation', singleInteractionBenchmark.elapsed());
+
         interactionCall.update({
           cacheHit: false,
           intermediaryCacheHit,
@@ -223,7 +215,7 @@ export abstract class DefaultStateEvaluator implements StateEvaluator {
         executionContext = await modify<State>(currentState, executionContext);
       }
     }
-    this.logger.debug('State evaluation total:', stateEvaluationBenchmark.elapsed());
+    this.logger.info('State evaluation total:', stateEvaluationBenchmark.elapsed());
     const evalStateResult = new EvalStateResult<State>(currentState, validity);
 
     // state could have been full retrieved from cache
@@ -291,9 +283,4 @@ export abstract class DefaultStateEvaluator implements StateEvaluator {
     executionContext: ExecutionContext<State>,
     state: EvalStateResult<State>
   ): Promise<void>;
-
-  abstract transactionState<State>(
-    transaction: GQLNodeInterface,
-    contractTxId: string
-  ): Promise<EvalStateResult<State> | undefined>;
 }

src/core/modules/impl/StateCache.ts

@@ -1,4 +1,4 @@
 import { EvalStateResult } from '@smartweave';
 
-export type StateCache<State> = Array<EvalStateResult<State>>;
-//export type StateCache<State> = EvalStateResult<State>;
+//export type StateCache<State> = Array<EvalStateResult<State>>;
+export type StateCache<State> = EvalStateResult<State>;

src/core/web/SmartWeaveWebFactory.ts

@@ -60,7 +60,7 @@ export class SmartWeaveWebFactory {
    * Returns a fully configured {@link SmartWeave} that is using mem cache for all layers.
    */
   static memCached(arweave: Arweave, maxStoredBlockHeights: number = Number.MAX_SAFE_INTEGER): SmartWeave {
-    return this.memCachedBased(arweave).build();
+    return this.memCachedBased(arweave, maxStoredBlockHeights).build();
   }
 
   /**
@@ -88,7 +88,7 @@ export class SmartWeaveWebFactory {
 
     return SmartWeave.builder(arweave)
       .setDefinitionLoader(definitionLoader)
-      .setCacheableInteractionsLoader(interactionsLoader, maxStoredBlockHeights)
+      .setCacheableInteractionsLoader(interactionsLoader)
       .setInteractionsSorter(interactionsSorter)
       .setExecutorFactory(executorFactory)
       .setStateEvaluator(stateEvaluator);

tools/contract-filecache.js

@@ -0,0 +1,56 @@
+/* eslint-disable */
+const Arweave = require('arweave');
+const { LoggerFactory } = require('../lib/cjs/logging/LoggerFactory');
+const { RedstoneGatewayInteractionsLoader } = require('../lib/cjs/core/modules/impl/RedstoneGatewayInteractionsLoader');
+const { SmartWeaveWebFactory } = require('../lib/cjs/core/web/SmartWeaveWebFactory');
+const {TsLogFactory} = require('../lib/cjs/logging/node/TsLogFactory');
+const fs = require('fs');
+const path =require('path');
+
+const logger = LoggerFactory.INST.create('Contract');
+
+LoggerFactory.use(new TsLogFactory());
+LoggerFactory.INST.logLevel('info');
+
+
+async function main() {
+  const arweave = Arweave.init({
+    host: 'arweave.net', // Hostname or IP address for a Arweave host
+    port: 443, // Port
+    protocol: 'https', // Network protocol http or https
+    timeout: 60000, // Network request timeouts in milliseconds
+    logging: false // Enable network request logging
+  });
+
+  const contractTxId = '-8A6RexFkpfWwuyVO98wzSFZh0d6VJuI-buTJvlwOJQ';
+
+  //const interactionsLoader = new FromFileInteractionsLoader(path.join(__dirname, 'data', 'interactions.json'));
+
+  // const smartweave = SmartWeaveWebFactory.memCachedBased(arweave).setInteractionsLoader(interactionsLoader).build();
+  const smartweave = SmartWeaveWebFactory
+    .memCachedBased(arweave, 1)
+    .setInteractionsLoader(new RedstoneGatewayInteractionsLoader(
+      'https://gateway.redstone.finance')
+    ).build();
+
+  const usedBefore = Math.round((process.memoryUsage().heapUsed / 1024 / 1024) * 100) / 100
+  const lootContract = smartweave.contract(contractTxId)
+    .setEvaluationOptions({updateCacheForEachInteraction: true});
+  const {state, validity} = await lootContract.readState();
+  const usedAfter = Math.round((process.memoryUsage().heapUsed / 1024 / 1024) * 100) / 100
+  logger.warn("Heap used in MB", {
+    usedBefore,
+    usedAfter
+  });
+
+
+  //fs.writeFileSync(path.join(__dirname, 'data', 'validity.json'), JSON.stringify(validity));
+
+  //fs.writeFileSync(path.join(__dirname, 'data', 'validity_old.json'), JSON.stringify(result.validity));
+  fs.writeFileSync(path.join(__dirname, 'data', 'state.json'), JSON.stringify(state));
+
+  // console.log('second read');
+  // await lootContract.readState();
+}
+
+main().catch((e) => console.error(e));

tools/contract.ts

@@ -1,14 +1,17 @@
 /* eslint-disable */
 import Arweave from 'arweave';
-import { LoggerFactory } from '../src';
-import { TsLogFactory } from '../src/logging/node/TsLogFactory';
+import {LoggerFactory, RedstoneGatewayInteractionsLoader, SmartWeaveWebFactory} from '../src';
+import {TsLogFactory} from '../src/logging/node/TsLogFactory';
 import fs from 'fs';
 import path from 'path';
-import { FromFileInteractionsLoader } from './FromFileInteractionsLoader';
-import { SmartWeaveNodeFactory } from '../src/core/node/SmartWeaveNodeFactory';
+import {FromFileInteractionsLoader} from './FromFileInteractionsLoader';
+import {SmartWeaveNodeFactory} from '../src/core/node/SmartWeaveNodeFactory';
+
+const logger = LoggerFactory.INST.create('Contract');
 
 LoggerFactory.use(new TsLogFactory());
-LoggerFactory.INST.logLevel('debug');
+LoggerFactory.INST.logLevel('info');
+
 
 async function main() {
   const arweave = Arweave.init({
@@ -24,12 +27,22 @@ async function main() {
   //const interactionsLoader = new FromFileInteractionsLoader(path.join(__dirname, 'data', 'interactions.json'));
 
   // const smartweave = SmartWeaveWebFactory.memCachedBased(arweave).setInteractionsLoader(interactionsLoader).build();
-  const smartweave = SmartWeaveNodeFactory.fileCached(arweave, 'cache');
+  const smartweave = SmartWeaveWebFactory
+    .memCachedBased(arweave, 1)
+    .setInteractionsLoader(new RedstoneGatewayInteractionsLoader(
+      'https://gateway.redstone.finance')
+    ).build();
 
+  const usedBefore = Math.round((process.memoryUsage().heapUsed / 1024 / 1024) * 100) / 100
   const lootContract = smartweave.contract(contractTxId)
-    .setEvaluationOptions({updateCacheForEachInteraction: false});
+    .setEvaluationOptions({updateCacheForEachInteraction: true});
+  const {state, validity} = await lootContract.readState();
+  const usedAfter = Math.round((process.memoryUsage().heapUsed / 1024 / 1024) * 100) / 100
+  logger.warn("Heap used in MB", {
+    usedBefore,
+    usedAfter
+  });
 
-  const { state, validity } = await lootContract.readState();
 
   //fs.writeFileSync(path.join(__dirname, 'data', 'validity.json'), JSON.stringify(validity));