diff --git a/README.md b/README.md index b529f5d..f977057 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,7 @@ The base motivation behind rewriting the original SDK (and roadmap proposal) has To further improve contract state evaluation time, one can additionally use AWS CloudFront based Arweave cache described [here](https://github.com/redstone-finance/redstone-smartweave-contracts/blob/main/docs/CACHE.md). - [Architecture](#architecture) +- [State evaluation diagram](#state-evaluation-diagram) - [Development](#development) - [Installation and import](#installation-and-import) - [Examples](#examples) @@ -46,6 +47,17 @@ This modular architecture has several advantages: 2. The SmartWeave client can be customized depending on user needs (e.g. different type of caches for web and node environment) 3. It makes it easier to add new features on top of the core protocol - without the risk of breaking the functionality of the core layer. +## State evaluation diagram +![readState](docs/img/readstate.png) + +In order to perform contract state evaluation (at given block height), SDK performs certain operations. +The diagram above and description assume the most basic “mem-cached” SDK client. +1. Users who are interacting with the contract, call the “readState” method. +2. Interactions Loader and Contract Definition Loader modules are then called in parallel - to load all the data required for state evaluation. Both Interactions Loader and Contract Definition Loader first check its corresponding cache whether data is already loaded - and load from Arweave only the missing part. +3. With interactions and contract definition loaded - Executor Factory creates a handle to the SmartWeave contract main function (or loads it from its own cache) +4. With all the interactions and a contract handle - the State Evaluator evaluates the state from the lastly cached value - and returns the result to User. + + ## Development PRs are welcome! :-) Also, feel free to submit [issues](https://github.com/redstone-finance/redstone-smartcontracts/issues) - with both bugs and feature proposals. In case of creating a PR - please use [semantic commit messages](https://gist.github.com/joshbuchea/6f47e86d2510bce28f8e7f42ae84c716). diff --git a/docs/SMARTWEAVE_PROTOCOL.md b/docs/SMARTWEAVE_PROTOCOL.md index 800ae72..5904078 100644 --- a/docs/SMARTWEAVE_PROTOCOL.md +++ b/docs/SMARTWEAVE_PROTOCOL.md @@ -101,6 +101,8 @@ In functional programming terms it acts as a `fold` function - the state of the 2. A contract function 3. An ordered list of actions +![protocol](img/protocol.png) + In order to evaluate contract state, SmartWeave Protocol client: 1. Loads all the contract's interaction transactions up to the requested block height. 2. Sorts the interaction transactions. The order of the interactions is determined firstly by interaction diff --git a/docs/img/protocol.png b/docs/img/protocol.png new file mode 100644 index 0000000..7d90fc2 Binary files /dev/null and b/docs/img/protocol.png differ diff --git a/docs/img/readstate.png b/docs/img/readstate.png new file mode 100644 index 0000000..ac598fc Binary files /dev/null and b/docs/img/readstate.png differ diff --git a/package.json b/package.json index 4b66fce..e8c91de 100644 --- a/package.json +++ b/package.json @@ -65,7 +65,7 @@ "@types/node": "^16.7.1", "@typescript-eslint/eslint-plugin": "^4.29.2", "@typescript-eslint/parser": "^4.29.2", - "arlocal": "^1.0.43", + "arlocal": "^1.0.44", "eslint": "^7.32.0", "eslint-config-prettier": "^8.3.0", "eslint-plugin-prettier": "^3.4.1", @@ -78,6 +78,7 @@ "ts-node": "^10.2.1", "tsc-alias": "^1.3.9", "tsconfig-paths": "^3.10.1", - "typescript": "^4.3.5" + "typescript": "^4.3.5", + "cors": "^2.8.5" } } diff --git a/src/__tests__/regression/readState.test.ts b/src/__tests__/regression/readState.test.ts index c263f71..b9558be 100644 --- a/src/__tests__/regression/readState.test.ts +++ b/src/__tests__/regression/readState.test.ts @@ -21,7 +21,7 @@ const arweave = Arweave.init({ }); LoggerFactory.INST.logLevel('fatal'); -const smartWeave = SmartWeaveNodeFactory.memCached(arweave); +const smartWeave = SmartWeaveNodeFactory.memCached(arweave, 10); const testCases: string[] = JSON.parse(fs.readFileSync(path.join(__dirname, 'test-cases.json'), 'utf-8')); @@ -37,6 +37,7 @@ describe.each(chunked)('.suite %#', (contracts: string[]) => { const result = await readContract(arweave, contractTxId); const resultString = JSON.stringify(result).trim(); + console.log('readState', contractTxId); const result2 = await smartWeave .contract(contractTxId) .setEvaluationOptions({ diff --git a/src/cache/impl/RemoteBlockHeightCache.ts b/src/cache/impl/RemoteBlockHeightCache.ts index 6aff84c..c5fbf19 100644 --- a/src/cache/impl/RemoteBlockHeightCache.ts +++ b/src/cache/impl/RemoteBlockHeightCache.ts @@ -4,6 +4,9 @@ import axios, { AxiosInstance } from 'axios'; /** * A {@link BlockHeightSwCache} implementation that delegates all its methods * to remote endpoints. + * + * TODO: this could be further optimised - i.e. with the help of "level 1" memory cache + * that would store max X elements - and would be backed up by the "level 2" remote cache. */ export class RemoteBlockHeightCache implements BlockHeightSwCache { private axios: AxiosInstance; @@ -39,9 +42,13 @@ export class RemoteBlockHeightCache implements BlockHeightSwCache { } /** + * TODO: data should "flushed" in batches... * PUT '/:type/:key/:blockHeight' {data: value} */ async put({ cacheKey, blockHeight }: BlockHeightKey, value: V): Promise { + if (!value) { + return; + } await this.axios.put(`/${this.type}/${cacheKey}/${blockHeight}`, value); } diff --git a/src/contract/HandlerBasedContract.ts b/src/contract/HandlerBasedContract.ts index 07d30c2..dbc27de 100644 --- a/src/contract/HandlerBasedContract.ts +++ b/src/contract/HandlerBasedContract.ts @@ -32,7 +32,6 @@ export class HandlerBasedContract implements Contract { /** * wallet connected to this contract - * @protected */ protected wallet?: ArWallet; private evaluationOptions: EvaluationOptions = new DefaultEvaluationOptions(); @@ -87,7 +86,8 @@ export class HandlerBasedContract implements Contract { const executionContext = await this.createExecutionContext(this.contractTxId, blockHeight); this.logger.info('Execution Context', { blockHeight: executionContext.blockHeight, - srcTxId: executionContext.contractDefinition.srcTxId + srcTxId: executionContext.contractDefinition.srcTxId, + missingInteractions: executionContext.sortedInteractions.length }); this.logger.debug('context', benchmark.elapsed()); benchmark.reset(); @@ -149,7 +149,7 @@ export class HandlerBasedContract implements Contract { // call one of the contract's view method const handleResult = await executionContext.handler.handle( executionContext, - evalStateResult.state, + evalStateResult, interaction, { id: null, @@ -192,7 +192,7 @@ export class HandlerBasedContract implements Contract { return await executionContext.handler.handle( executionContext, - evalStateResult.state, + evalStateResult, interaction, transaction, [] diff --git a/src/core/modules/StateEvaluator.ts b/src/core/modules/StateEvaluator.ts index 4f10e68..7ff272b 100644 --- a/src/core/modules/StateEvaluator.ts +++ b/src/core/modules/StateEvaluator.ts @@ -1,4 +1,4 @@ -import { ExecutionContext, GQLEdgeInterface, GQLNodeInterface } from '@smartweave'; +import { ExecutionContext, GQLNodeInterface, InteractionTx } from '@smartweave'; /** * Implementors of this class are responsible for evaluating contract's state @@ -10,11 +10,32 @@ export interface StateEvaluator { currentTx: { interactionTxId: string; contractTxId: string }[] ): Promise>; + /** + * a hook that is called on each state update (i.e. after evaluating state for each interaction) + */ onStateUpdate( currentInteraction: GQLNodeInterface, executionContext: ExecutionContext, state: EvalStateResult ): Promise; + + /** + * a hook that is called after state has been fully evaluated + */ + onStateEvaluated( + lastInteraction: GQLNodeInterface, + executionContext: ExecutionContext, + state: EvalStateResult + ): Promise; + + /** + * a hook that is called before communicating with other contract + */ + onContractCall( + currentInteraction: InteractionTx, + executionContext: ExecutionContext, + state: EvalStateResult + ): Promise; } export class EvalStateResult { @@ -32,6 +53,8 @@ export class DefaultEvaluationOptions implements EvaluationOptions { waitForConfirmation = false; fcpOptimization = false; + + updateCacheForEachInteraction = true; } // an interface for the contract EvaluationOptions - can be used to change the behaviour of some of the features. @@ -45,4 +68,9 @@ export interface EvaluationOptions { // experimental optimization for contracts that utilize the Foreign Call Protocol fcpOptimization: boolean; + + // whether cache should be updated after evaluating each interaction transaction. + // this can be switched off to speed up cache writes (ie. for some contracts (with flat structure) + // and caches it maybe more suitable to cache only after state has been fully evaluated) + updateCacheForEachInteraction: boolean; } diff --git a/src/core/modules/impl/DefaultStateEvaluator.ts b/src/core/modules/impl/DefaultStateEvaluator.ts index 7c33f77..a11705f 100644 --- a/src/core/modules/impl/DefaultStateEvaluator.ts +++ b/src/core/modules/impl/DefaultStateEvaluator.ts @@ -10,6 +10,7 @@ import { GQLTagInterface, HandlerApi, InteractionResult, + InteractionTx, LoggerFactory, MemCache, StateEvaluator, @@ -50,30 +51,25 @@ export class DefaultStateEvaluator implements StateEvaluator { ): Promise> { const stateEvaluationBenchmark = Benchmark.measure(); const { ignoreExceptions } = executionContext.evaluationOptions; + const { contractDefinition, sortedInteractions } = executionContext; let currentState = baseState.state; let validity = deepCopy(baseState.validity); this.logger.info( - `Evaluating state for ${executionContext.contractDefinition.txId} [${missingInteractions.length} non-cached of ${executionContext.sortedInteractions.length} all]` + `Evaluating state for ${executionContext.contractDefinition.txId} + [${missingInteractions.length} non-cached of ${executionContext.sortedInteractions.length} all]` ); - this.logger.trace( - 'missingInteractions', - missingInteractions.map((int) => { - return int.node.id; - }) - ); - - this.logger.trace('Init state', JSON.stringify(baseState.state)); + let lastEvaluatedInteraction = null; for (const missingInteraction of missingInteractions) { const currentInteraction: GQLNodeInterface = missingInteraction.node; this.logger.debug( - `[${executionContext.contractDefinition.txId}][${missingInteraction.node.id}]: ${ + `[${contractDefinition.txId}][${missingInteraction.node.id}][${missingInteraction.node.block.height}]: ${ missingInteractions.indexOf(missingInteraction) + 1 - }/${missingInteractions.length} [of all:${executionContext.sortedInteractions.length}]` + }/${missingInteractions.length} [of all:${sortedInteractions.length}]` ); const state = await this.onNextIteration(currentInteraction, executionContext); @@ -84,7 +80,7 @@ export class DefaultStateEvaluator implements StateEvaluator { } else { const singleInteractionBenchmark = Benchmark.measure(); - const inputTag = this.tagsParser.getInputTag(missingInteraction, executionContext.contractDefinition.txId); + const inputTag = this.tagsParser.getInputTag(missingInteraction, contractDefinition.txId); if (!inputTag) { this.logger.error(`Skipping tx with missing or invalid Input tag - ${currentInteraction.id}`); continue; @@ -103,7 +99,7 @@ export class DefaultStateEvaluator implements StateEvaluator { const result = await executionContext.handler.handle( executionContext, - currentState, + new EvalStateResult(currentState, validity), interaction, currentInteraction, currentTx @@ -115,10 +111,6 @@ export class DefaultStateEvaluator implements StateEvaluator { throw new Error(`Exception while processing ${JSON.stringify(interaction)}:\n${result.result}`); } - if (result.type === 'exception') { - this.logger.error('Credit:', (currentState as any).credit); - } - validity[currentInteraction.id] = result.type === 'ok'; // strangely - state is for some reason modified for some contracts (eg. YLVpmhSq5JmLltfg6R-5fL04rIRPrlSU22f6RQ6VyYE) // when calling any async (even simple timeout) function here... @@ -126,6 +118,10 @@ export class DefaultStateEvaluator implements StateEvaluator { // see https://github.com/ArweaveTeam/SmartWeave/pull/92 for more details currentState = deepCopy(result.state); + // cannot simply take last element of the missingInteractions + // as there is no certainty that it has been evaluated (e.g. issues with input tag). + lastEvaluatedInteraction = currentInteraction; + this.logger.debug('Interaction evaluation', singleInteractionBenchmark.elapsed()); } @@ -142,7 +138,15 @@ export class DefaultStateEvaluator implements StateEvaluator { } } this.logger.debug('State evaluation total:', stateEvaluationBenchmark.elapsed()); - return new EvalStateResult(currentState, validity); + const result = new EvalStateResult(currentState, validity); + + // state could have been full retrieved from cache + // or there were no interactions below requested block height + if (lastEvaluatedInteraction !== null) { + await this.onStateEvaluated(lastEvaluatedInteraction, executionContext, result); + } + + return result; } private logResult( @@ -199,4 +203,20 @@ export class DefaultStateEvaluator implements StateEvaluator { return deepCopy(cachedState as EvalStateResult); } } + + onContractCall( + currentInteraction: InteractionTx, + executionContext: ExecutionContext, + state: EvalStateResult + ): Promise { + return Promise.resolve(undefined); + } + + onStateEvaluated( + lastInteraction: GQLNodeInterface, + executionContext: ExecutionContext, + state: EvalStateResult + ): Promise { + return Promise.resolve(undefined); + } } diff --git a/src/core/modules/impl/HandlerExecutorFactory.ts b/src/core/modules/impl/HandlerExecutorFactory.ts index 3d9354e..c166dd5 100644 --- a/src/core/modules/impl/HandlerExecutorFactory.ts +++ b/src/core/modules/impl/HandlerExecutorFactory.ts @@ -4,6 +4,7 @@ import * as clarity from '@weavery/clarity'; import { ContractDefinition, deepCopy, + EvalStateResult, ExecutionContext, ExecutorFactory, InteractionTx, @@ -17,7 +18,7 @@ import { export interface HandlerApi { handle( executionContext: ExecutionContext, - state: State, + currentResult: EvalStateResult, interaction: ContractInteraction, interactionTx: InteractionTx, currentTx: { interactionTxId: string; contractTxId: string }[] @@ -54,7 +55,7 @@ export class HandlerExecutorFactory implements ExecutorFactory( executionContext: ExecutionContext, - state: State, + currentResult: EvalStateResult, interaction: ContractInteraction, interactionTx: InteractionTx, currentTx: { interactionTxId: string; contractTxId: string }[] @@ -65,11 +66,19 @@ export class HandlerExecutorFactory implements ExecutorFactory; - const stateCopy = JSON.parse(JSON.stringify(state)); + const stateCopy = JSON.parse(JSON.stringify(currentResult.state)); swGlobal._activeTx = interactionTx; self.logger.trace(`SmartWeave.contract.id:`, swGlobal.contract.id); - self.assignReadContractState(swGlobal, contractDefinition, executionContext, currentTx); + // TODO: refactor - too many arguments + self.assignReadContractState( + swGlobal, + contractDefinition, + executionContext, + currentTx, + currentResult, + interactionTx + ); self.assignViewContractState(swGlobal, contractDefinition, executionContext); const handlerResult = await handler(stateCopy, interaction); @@ -78,7 +87,7 @@ export class HandlerExecutorFactory implements ExecutorFactory, executionContext: ExecutionContext, - currentTx: { interactionTxId: string; contractTxId: string }[] + currentTx: { interactionTxId: string; contractTxId: string }[], + currentResult: EvalStateResult, + interactionTx: InteractionTx ) { swGlobal.contracts.readContractState = async (contractTxId: string, height?: number, returnValidity?: boolean) => { const requestedHeight = height || swGlobal.block.height; @@ -143,10 +154,14 @@ export class HandlerExecutorFactory implements ExecutorFactory>(), [ - new Evolve(definitionLoader, executorFactory) - ]); + const stateEvaluator = new CacheableStateEvaluator( + arweave, + new MemBlockHeightSwCache>(maxStoredBlockHeights), + [new Evolve(definitionLoader, executorFactory)] + ); const interactionsSorter = new LexicographicalInteractionsSorter(arweave); diff --git a/src/plugins/CacheableStateEvaluator.ts b/src/plugins/CacheableStateEvaluator.ts index 9f96054..bc7392f 100644 --- a/src/plugins/CacheableStateEvaluator.ts +++ b/src/plugins/CacheableStateEvaluator.ts @@ -1,4 +1,4 @@ -import { BlockHeightCacheResult, BlockHeightKey, BlockHeightSwCache, MemCache } from '@smartweave/cache'; +import { BlockHeightCacheResult, BlockHeightKey, BlockHeightSwCache } from '@smartweave/cache'; import { DefaultStateEvaluator, EvalStateResult, @@ -7,9 +7,8 @@ import { HandlerApi } from '@smartweave/core'; import Arweave from 'arweave'; -import { GQLEdgeInterface, GQLNodeInterface } from '@smartweave/legacy'; +import { GQLNodeInterface, InteractionTx } from '@smartweave/legacy'; import { Benchmark, LoggerFactory } from '@smartweave/logging'; -import { deepCopy } from '@smartweave/utils'; /** * An implementation of DefaultStateEvaluator that adds caching capabilities @@ -34,12 +33,16 @@ export class CacheableStateEvaluator extends DefaultStateEvaluator { let cachedState: BlockHeightCacheResult> | null = null; + this.cLogger.debug('executionContext.sortedInteractions', executionContext.sortedInteractions.length); + const sortedInteractionsUpToBlock = executionContext.sortedInteractions.filter((tx) => { return tx.node.block.height <= executionContext.blockHeight; }); let missingInteractions = sortedInteractionsUpToBlock.slice(); + this.cLogger.debug('missingInteractions', missingInteractions.length); + // if there was anything to cache... if (sortedInteractionsUpToBlock.length > 0) { // get latest available cache for the requested block height @@ -52,7 +55,7 @@ export class CacheableStateEvaluator extends DefaultStateEvaluator { if (cachedState != null) { this.cLogger.debug(`Cached state for ${executionContext.contractDefinition.txId}`, { - block: cachedState.cachedHeight, + cachedHeight: cachedState.cachedHeight, requestedBlockHeight }); @@ -107,16 +110,30 @@ export class CacheableStateEvaluator extends DefaultStateEvaluator { ); } - async onStateUpdate( - currentInteraction: GQLNodeInterface, + async onStateEvaluated( + lastInteraction: GQLNodeInterface, executionContext: ExecutionContext, state: EvalStateResult - ) { - await super.onStateUpdate(currentInteraction, executionContext, state); - + ): Promise { + this.cLogger.debug( + `onStateEvaluated: cache update for contract ${executionContext.contractDefinition.txId} [${lastInteraction.block.height}]` + ); await this.cache.put( - new BlockHeightKey(executionContext.contractDefinition.txId, currentInteraction.block.height), + new BlockHeightKey(executionContext.contractDefinition.txId, lastInteraction.block.height), state ); } + + async onStateUpdate( + currentInteraction: GQLNodeInterface, + executionContext: ExecutionContext, + state: EvalStateResult + ): Promise { + if (executionContext.evaluationOptions.updateCacheForEachInteraction) { + await this.cache.put( + new BlockHeightKey(executionContext.contractDefinition.txId, currentInteraction.block.height), + state + ); + } + } } diff --git a/tools/server.js b/tools/server.js index 1bfa1cf..9544de1 100644 --- a/tools/server.js +++ b/tools/server.js @@ -1,11 +1,12 @@ const express = require('express'); +const cors = require('cors'); const { MemBlockHeightSwCache } = require('../lib/cjs/cache/impl/MemBlockHeightCache'); const app = express(); const port = 3000; -console.log(MemBlockHeightSwCache); +app.use(cors()); +app.use(express.json({ limit: "50mb", extended: true })); -app.use(express.json()); const caches = { STATE: new MemBlockHeightSwCache(1), diff --git a/yarn.lock b/yarn.lock index 7eec6a8..90e3005 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1475,10 +1475,10 @@ argparse@^1.0.7: dependencies: sprintf-js "~1.0.2" -arlocal@^1.0.43: - version "1.0.43" - resolved "https://registry.yarnpkg.com/arlocal/-/arlocal-1.0.43.tgz#ce45ed2882a75293990aadd57dea1c085b7f345f" - integrity sha512-9G5QAtuDrD/8DvkrLk7O8uCIIVwo2tW4+1XCx90OrDfOh3QH0/7peKLcpE2dWYkl3684lvyJ/ku/kRJcQhQvNA== +arlocal@^1.0.44: + version "1.0.44" + resolved "https://registry.yarnpkg.com/arlocal/-/arlocal-1.0.44.tgz#21f0d206b6d539aff04a24c2002f4c42ad467419" + integrity sha512-NviY1QzOb4v66rh0zTwNV8eoYu4my1hJbLE+RvuPgqrg32nBS3Snv53CyZ36czs4ffpDkpHv01p4dpFIrox1wA== dependencies: "@koa/cors" "^3.1.0" apollo-server-koa "^2.25.1" @@ -2163,6 +2163,14 @@ core-util-is@^1.0.2, core-util-is@~1.0.0: resolved "https://registry.yarnpkg.com/core-util-is/-/core-util-is-1.0.3.tgz#a6042d3634c2b27e9328f837b965fac83808db85" integrity sha512-ZQBvi1DcpJ4GDqanjucZ2Hj3wEO5pZDS89BWbkcrvdxksJorwUDDZamX9ldFkp9aw2lmBDLgkObEA4DWNJ9FYQ== +cors@^2.8.5: + version "2.8.5" + resolved "https://registry.yarnpkg.com/cors/-/cors-2.8.5.tgz#eac11da51592dd86b9f06f6e7ac293b3df875d29" + integrity sha512-KIHbLJqu73RGr/hnbrO9uBeixNGuvSQjul/jdFvS/KFSIH1hWVd1ng7zOHx+YrEfInLG7q4n6GHQ9cDtxv/P6g== + dependencies: + object-assign "^4" + vary "^1" + create-require@^1.1.0: version "1.1.1" resolved "https://registry.yarnpkg.com/create-require/-/create-require-1.1.1.tgz#c1d7e8f1e5f6cfc9ff65f9cd352d37348756c333" @@ -4794,7 +4802,7 @@ oauth-sign@~0.9.0: resolved "https://registry.yarnpkg.com/oauth-sign/-/oauth-sign-0.9.0.tgz#47a7b016baa68b5fa0ecf3dee08a85c679ac6455" integrity sha512-fexhUFFPTGV8ybAtSIGbV6gOkSv8UtRbDBnAyLQw4QPKkgNlsH2ByPGtMUqdWkos6YCRmAqViwgZrJc/mRDzZQ== -object-assign@^4.1.0: +object-assign@^4, object-assign@^4.1.0: version "4.1.1" resolved "https://registry.yarnpkg.com/object-assign/-/object-assign-4.1.1.tgz#2109adc7965887cfc05cbbd442cac8bfbb360863" integrity sha1-IQmtx5ZYh8/AXLvUQsrIv7s2CGM= @@ -6230,7 +6238,7 @@ v8-to-istanbul@^8.0.0: convert-source-map "^1.6.0" source-map "^0.7.3" -vary@^1.1.2, vary@~1.1.2: +vary@^1, vary@^1.1.2, vary@~1.1.2: version "1.1.2" resolved "https://registry.yarnpkg.com/vary/-/vary-1.1.2.tgz#2299f02c6ded30d4a5961b0b9f74524a18f634fc" integrity sha1-IpnwLG3tMNSllhsLn3RSShj2NPw=