chore: state cache interface

This commit is contained in:
Tadeuchi
2023-05-08 11:45:53 +02:00
parent e47809d185
commit 20e8d38ef7
11 changed files with 124 additions and 98 deletions

71
src/cache/BasicSortKeyCache.ts vendored Normal file
View File

@@ -0,0 +1,71 @@
import { CacheKey, PruneStats, SortKeyCacheResult } from './SortKeyCache';
/**
* A cache that stores its values per dedicated key and sort key.
* A sort key is a value that the SmartWeave protocol is using
* to sort contract transactions ({@link LexicographicalInteractionsSorter}.
*
* All values should be stored in a lexicographical order (per key) -
* sorted by the sort key.
*/
export interface BasicSortKeyCache<V> {
getLessOrEqual(key: string, sortKey: string): Promise<SortKeyCacheResult<V> | null>;
/**
* returns value stored for a given key and last sortKey
*/
getLast(key: string): Promise<SortKeyCacheResult<V> | null>;
/**
* returns last cached sort key - takes all keys into account
*/
getLastSortKey(): Promise<string | null>;
/**
* returns value for the key and exact sortKey
*/
get(cacheKey: CacheKey): Promise<SortKeyCacheResult<V> | null>;
/**
* puts new value in cache under given {@link CacheKey.key} and {@link CacheKey.sortKey}.
*/
put(cacheKey: CacheKey, value: V): Promise<void>;
/**
* removes all data stored under a specified key
*/
delete(key: string): Promise<void>;
open(): Promise<void>;
close(): Promise<void>;
begin(): Promise<void>;
rollback(): Promise<void>;
commit(): Promise<void>;
/**
* used mostly for debugging, allows to dump the current content cache
* It's slow.
*/
dump(): Promise<any>; // eslint-disable-line @typescript-eslint/no-explicit-any
/**
* returns underlying storage (LevelDB, LMDB, sqlite...)
* - useful for performing low-level operations
*/
storage<S>(): S;
/**
* leaves n-latest (i.e. with latest (in lexicographic order) sort keys)
* entries for each cached key
*
* @param entriesStored - how many latest entries should be left
* for each cached key
*
* @retun PruneStats if getting them doesn't introduce a delay, null otherwise
*/
prune(entriesStored: number): Promise<PruneStats | null>;
}

View File

@@ -1,68 +1,26 @@
import { SortKeyCacheRangeOptions } from './SortKeyCacheRangeOptions'; import { SortKeyCacheRangeOptions } from './SortKeyCacheRangeOptions';
import { BasicSortKeyCache } from './BasicSortKeyCache';
/** /**
* A cache that stores its values per dedicated key and sort key. * Key-value cache storage.
* A sort key is a value that the SmartWeave protocol is using * Just as {@link BasicSortKeyCache}, items are stored
* to sort contract transactions ({@link LexicographicalInteractionsSorter}. * in lexicographical order using by sort key.
* *
* All values should be stored in a lexicographical order (per key) - * In addition, this interface provide functionality related to
* sorted by the sort key. * fetching keys and values using range options. {@link SortKeyCacheRangeOptions}
*/ */
export interface SortKeyCache<V> { export interface SortKeyCache<V> extends BasicSortKeyCache<V> {
getLessOrEqual(key: string, sortKey: string): Promise<SortKeyCacheResult<V> | null>;
/**
* returns value stored for a given key and last sortKey
*/
getLast(key: string): Promise<SortKeyCacheResult<V> | null>;
/**
* returns last cached sort key - takes all keys into account
*/
getLastSortKey(): Promise<string | null>;
/**
* returns value for the key and exact sortKey
*/
get(cacheKey: CacheKey): Promise<SortKeyCacheResult<V> | null>;
/**
* puts new value in cache under given {@link CacheKey.key} and {@link CacheKey.sortKey}.
*/
put(cacheKey: CacheKey, value: V): Promise<void>;
/** /**
* deletes value in cache under given {@link CacheKey.key} from {@link CacheKey.sortKey}. * deletes value in cache under given {@link CacheKey.key} from {@link CacheKey.sortKey}.
* the value will be still available if fetched using a lower sortKey * the value will be still available if fetched using a lower sortKey
*/ */
del(cacheKey: CacheKey): Promise<void>; del(cacheKey: CacheKey): Promise<void>;
/**
* removes all data stored under a specified key
*/
delete(key: string): Promise<void>;
/** /**
* executes a list of stacked operations * executes a list of stacked operations
*/ */
batch(opStack: BatchDBOp<V>[]); batch(opStack: BatchDBOp<V>[]);
open(): Promise<void>;
close(): Promise<void>;
begin(): Promise<void>;
rollback(): Promise<void>;
commit(): Promise<void>;
/**
* used mostly for debugging, allows to dump the current content cache
* It's slow.
*/
dump(): Promise<any>; // eslint-disable-line @typescript-eslint/no-explicit-any
/** /**
* Returns keys for a specified range * Returns keys for a specified range
*/ */
@@ -72,23 +30,6 @@ export interface SortKeyCache<V> {
* Returns a key value map for a specified range * Returns a key value map for a specified range
*/ */
kvMap(sortKey: string, options?: SortKeyCacheRangeOptions): Promise<Map<string, V>>; kvMap(sortKey: string, options?: SortKeyCacheRangeOptions): Promise<Map<string, V>>;
/**
* returns underlying storage (LevelDB, LMDB, sqlite...)
* - useful for performing low-level operations
*/
storage<S>(): S;
/**
* leaves n-latest (i.e. with latest (in lexicographic order) sort keys)
* entries for each cached key
*
* @param entriesStored - how many latest entries should be left
* for each cached key
*
* @retun PruneStats if getting them doesn't introduce a delay, null otherwise
*/
prune(entriesStored: number): Promise<PruneStats | null>;
} }
export interface PruneStats { export interface PruneStats {
@@ -107,11 +48,13 @@ export class SortKeyCacheResult<V> {
} }
export declare type BatchDBOp<V> = PutBatch<V> | DelBatch; export declare type BatchDBOp<V> = PutBatch<V> | DelBatch;
export interface PutBatch<V> { export interface PutBatch<V> {
type: 'put'; type: 'put';
key: CacheKey; key: CacheKey;
value: V; value: V;
} }
export interface DelBatch { export interface DelBatch {
type: 'del'; type: 'del';
key: string; key: string;

View File

@@ -1,3 +1,10 @@
/**
* Range option for fetching items from kv storage {@link SortKeyCache}
* @param gte - greater than equals
* @param lt - less than
* @param reverse - reverses the order
* @param limit - limits output elements
*/
export interface SortKeyCacheRangeOptions { export interface SortKeyCacheRangeOptions {
gte?: string; gte?: string;
lt?: string; lt?: string;

View File

@@ -33,6 +33,7 @@ import { DEFAULT_LEVEL_DB_LOCATION, WARP_GW_URL } from './WarpFactory';
import { LevelDbCache } from '../cache/impl/LevelDbCache'; import { LevelDbCache } from '../cache/impl/LevelDbCache';
import { SourceData } from '../contract/deploy/Source'; import { SourceData } from '../contract/deploy/Source';
import { BundlerSigner, DataItem } from '../contract/deploy/DataItem'; import { BundlerSigner, DataItem } from '../contract/deploy/DataItem';
import { BasicSortKeyCache } from '../cache/BasicSortKeyCache';
export type WarpEnvironment = 'local' | 'testnet' | 'mainnet' | 'custom'; export type WarpEnvironment = 'local' | 'testnet' | 'mainnet' | 'custom';
export type KVStorageFactory = (contractTxId: string) => SortKeyCache<unknown>; export type KVStorageFactory = (contractTxId: string) => SortKeyCache<unknown>;
@@ -87,7 +88,7 @@ export class Warp {
static builder( static builder(
arweave: Arweave, arweave: Arweave,
stateCache: SortKeyCache<EvalStateResult<unknown>>, stateCache: BasicSortKeyCache<EvalStateResult<unknown>>,
environment: WarpEnvironment environment: WarpEnvironment
): WarpBuilder { ): WarpBuilder {
return new WarpBuilder(arweave, stateCache, environment); return new WarpBuilder(arweave, stateCache, environment);
@@ -138,12 +139,12 @@ export class Warp {
return new PstContractImpl(contractTxId, this); return new PstContractImpl(contractTxId, this);
} }
useStateCache(stateCache: SortKeyCache<EvalStateResult<unknown>>): Warp { useStateCache(stateCache: BasicSortKeyCache<EvalStateResult<unknown>>): Warp {
this.stateEvaluator.setCache(stateCache); this.stateEvaluator.setCache(stateCache);
return this; return this;
} }
useContractCache(definition: SortKeyCache<ContractDefinition<unknown>>, src: SortKeyCache<SrcCache>): Warp { useContractCache(definition: BasicSortKeyCache<ContractDefinition<unknown>>, src: SortKeyCache<SrcCache>): Warp {
this.definitionLoader.setSrcCache(src); this.definitionLoader.setSrcCache(src);
this.definitionLoader.setCache(definition); this.definitionLoader.setCache(definition);
return this; return this;

View File

@@ -12,9 +12,9 @@ import { InteractionsLoader } from './modules/InteractionsLoader';
import { StateEvaluator, EvalStateResult } from './modules/StateEvaluator'; import { StateEvaluator, EvalStateResult } from './modules/StateEvaluator';
import { WarpEnvironment, Warp } from './Warp'; import { WarpEnvironment, Warp } from './Warp';
import { CacheOptions, GatewayOptions } from './WarpFactory'; import { CacheOptions, GatewayOptions } from './WarpFactory';
import { SortKeyCache } from '../cache/SortKeyCache';
import { LevelDbCache } from '../cache/impl/LevelDbCache'; import { LevelDbCache } from '../cache/impl/LevelDbCache';
import { ContractCache, SrcCache } from './ContractDefinition'; import { ContractCache, SrcCache } from './ContractDefinition';
import { BasicSortKeyCache } from '../cache/BasicSortKeyCache';
export class WarpBuilder { export class WarpBuilder {
private _definitionLoader?: DefinitionLoader; private _definitionLoader?: DefinitionLoader;
@@ -24,7 +24,7 @@ export class WarpBuilder {
constructor( constructor(
private readonly _arweave: Arweave, private readonly _arweave: Arweave,
private readonly _stateCache: SortKeyCache<EvalStateResult<unknown>>, private readonly _stateCache: BasicSortKeyCache<EvalStateResult<unknown>>,
private readonly _environment: WarpEnvironment = 'custom' private readonly _environment: WarpEnvironment = 'custom'
) {} ) {}

View File

@@ -1,7 +1,7 @@
import { ContractCache, ContractDefinition, ContractSource, SrcCache } from '../../core/ContractDefinition'; import { ContractCache, ContractDefinition, ContractSource, SrcCache } from '../../core/ContractDefinition';
import { GwTypeAware } from './InteractionsLoader'; import { GwTypeAware } from './InteractionsLoader';
import { SortKeyCache } from '../../cache/SortKeyCache';
import { WarpAware } from '../Warp'; import { WarpAware } from '../Warp';
import { BasicSortKeyCache } from '../../cache/BasicSortKeyCache';
/** /**
* Implementors of this interface are responsible for loading contract's definitions - * Implementors of this interface are responsible for loading contract's definitions -
@@ -13,12 +13,12 @@ export interface DefinitionLoader extends GwTypeAware, WarpAware {
loadContractSource(srcTxId: string): Promise<ContractSource>; loadContractSource(srcTxId: string): Promise<ContractSource>;
setCache(cache: SortKeyCache<ContractCache<unknown>>): void; setCache(cache: BasicSortKeyCache<ContractCache<unknown>>): void;
// Cache for storing common source code or binaries // Cache for storing common source code or binaries
setSrcCache(cacheSrc?: SortKeyCache<SrcCache>): void; setSrcCache(cacheSrc?: BasicSortKeyCache<SrcCache>): void;
getCache(): SortKeyCache<ContractCache<unknown>>; getCache(): BasicSortKeyCache<ContractCache<unknown>>;
getSrcCache(): SortKeyCache<SrcCache>; getSrcCache(): BasicSortKeyCache<SrcCache>;
} }

View File

@@ -1,7 +1,8 @@
import { SortKeyCache, SortKeyCacheResult } from '../../cache/SortKeyCache'; import { SortKeyCacheResult } from '../../cache/SortKeyCache';
import { ExecutionContext } from '../ExecutionContext'; import { ExecutionContext } from '../ExecutionContext';
import { GQLNodeInterface } from '../../legacy/gqlResult'; import { GQLNodeInterface } from '../../legacy/gqlResult';
import { SourceType } from './impl/WarpGatewayInteractionsLoader'; import { SourceType } from './impl/WarpGatewayInteractionsLoader';
import { BasicSortKeyCache } from '../../cache/BasicSortKeyCache';
/** /**
* Implementors of this class are responsible for evaluating contract's state * Implementors of this class are responsible for evaluating contract's state
@@ -87,9 +88,9 @@ export interface StateEvaluator {
lastCachedSortKey(): Promise<string | null>; lastCachedSortKey(): Promise<string | null>;
setCache(cache: SortKeyCache<EvalStateResult<unknown>>): void; setCache(cache: BasicSortKeyCache<EvalStateResult<unknown>>): void;
getCache(): SortKeyCache<EvalStateResult<unknown>>; getCache(): BasicSortKeyCache<EvalStateResult<unknown>>;
} }
export class EvalStateResult<State> { export class EvalStateResult<State> {

View File

@@ -1,5 +1,5 @@
import Arweave from 'arweave'; import Arweave from 'arweave';
import { CacheKey, SortKeyCache, SortKeyCacheResult } from '../../../cache/SortKeyCache'; import { CacheKey, SortKeyCacheResult } from '../../../cache/SortKeyCache';
import { ExecutionContext } from '../../../core/ExecutionContext'; import { ExecutionContext } from '../../../core/ExecutionContext';
import { ExecutionContextModifier } from '../../../core/ExecutionContextModifier'; import { ExecutionContextModifier } from '../../../core/ExecutionContextModifier';
import { GQLNodeInterface } from '../../../legacy/gqlResult'; import { GQLNodeInterface } from '../../../legacy/gqlResult';
@@ -9,6 +9,7 @@ import { EvalStateResult } from '../StateEvaluator';
import { DefaultStateEvaluator } from './DefaultStateEvaluator'; import { DefaultStateEvaluator } from './DefaultStateEvaluator';
import { HandlerApi } from './HandlerExecutorFactory'; import { HandlerApi } from './HandlerExecutorFactory';
import { genesisSortKey } from './LexicographicalInteractionsSorter'; import { genesisSortKey } from './LexicographicalInteractionsSorter';
import { BasicSortKeyCache } from '../../../cache/BasicSortKeyCache';
/** /**
* An implementation of DefaultStateEvaluator that adds caching capabilities. * An implementation of DefaultStateEvaluator that adds caching capabilities.
@@ -23,7 +24,7 @@ export class CacheableStateEvaluator extends DefaultStateEvaluator {
constructor( constructor(
arweave: Arweave, arweave: Arweave,
private cache: SortKeyCache<EvalStateResult<unknown>>, private cache: BasicSortKeyCache<EvalStateResult<unknown>>,
executionContextModifiers: ExecutionContextModifier[] = [] executionContextModifiers: ExecutionContextModifier[] = []
) { ) {
super(arweave, executionContextModifiers); super(arweave, executionContextModifiers);
@@ -224,11 +225,11 @@ export class CacheableStateEvaluator extends DefaultStateEvaluator {
return await this.cache.getLastSortKey(); return await this.cache.getLastSortKey();
} }
setCache(cache: SortKeyCache<EvalStateResult<unknown>>): void { setCache(cache: BasicSortKeyCache<EvalStateResult<unknown>>): void {
this.cache = cache; this.cache = cache;
} }
getCache(): SortKeyCache<EvalStateResult<unknown>> { getCache(): BasicSortKeyCache<EvalStateResult<unknown>> {
return this.cache; return this.cache;
} }
} }

View File

@@ -16,8 +16,8 @@ import { GW_TYPE } from '../InteractionsLoader';
import { TagsParser } from './TagsParser'; import { TagsParser } from './TagsParser';
import { WasmSrc } from './wasm/WasmSrc'; import { WasmSrc } from './wasm/WasmSrc';
import { Warp, WarpEnvironment } from '../../Warp'; import { Warp, WarpEnvironment } from '../../Warp';
import { SortKeyCache } from '../../../cache/SortKeyCache';
import { Transaction } from '../../../utils/types/arweave-types'; import { Transaction } from '../../../utils/types/arweave-types';
import { BasicSortKeyCache } from '../../../cache/BasicSortKeyCache';
export class ContractDefinitionLoader implements DefinitionLoader { export class ContractDefinitionLoader implements DefinitionLoader {
private readonly logger = LoggerFactory.INST.create('ContractDefinitionLoader'); private readonly logger = LoggerFactory.INST.create('ContractDefinitionLoader');
@@ -147,20 +147,20 @@ export class ContractDefinitionLoader implements DefinitionLoader {
} }
// eslint-disable-next-line @typescript-eslint/no-unused-vars // eslint-disable-next-line @typescript-eslint/no-unused-vars
setCache(cache: SortKeyCache<ContractDefinition<unknown>>): void { setCache(cache: BasicSortKeyCache<ContractDefinition<unknown>>): void {
throw new Error('No cache implemented for this loader'); throw new Error('No cache implemented for this loader');
} }
// eslint-disable-next-line @typescript-eslint/no-unused-vars // eslint-disable-next-line @typescript-eslint/no-unused-vars
setSrcCache(cache: SortKeyCache<SrcCache>): void { setSrcCache(cache: BasicSortKeyCache<SrcCache>): void {
throw new Error('No cache implemented for this loader'); throw new Error('No cache implemented for this loader');
} }
getCache(): SortKeyCache<ContractCache<unknown>> { getCache(): BasicSortKeyCache<ContractCache<unknown>> {
throw new Error('No cache implemented for this loader'); throw new Error('No cache implemented for this loader');
} }
getSrcCache(): SortKeyCache<SrcCache> { getSrcCache(): BasicSortKeyCache<SrcCache> {
throw new Error('No cache implemented for this loader'); throw new Error('No cache implemented for this loader');
} }

View File

@@ -1,6 +1,6 @@
import Arweave from 'arweave'; import Arweave from 'arweave';
import { SortKeyCache, SortKeyCacheResult } from '../../../cache/SortKeyCache'; import { SortKeyCacheResult } from '../../../cache/SortKeyCache';
import { InteractionCall } from '../../ContractCallRecord'; import { InteractionCall } from '../../ContractCallRecord';
import { ExecutionContext } from '../../../core/ExecutionContext'; import { ExecutionContext } from '../../../core/ExecutionContext';
import { ExecutionContextModifier } from '../../../core/ExecutionContextModifier'; import { ExecutionContextModifier } from '../../../core/ExecutionContextModifier';
@@ -12,6 +12,7 @@ import { EvalStateResult, StateEvaluator } from '../StateEvaluator';
import { ContractInteraction, HandlerApi, InteractionResult } from './HandlerExecutorFactory'; import { ContractInteraction, HandlerApi, InteractionResult } from './HandlerExecutorFactory';
import { TagsParser } from './TagsParser'; import { TagsParser } from './TagsParser';
import { VrfPluginFunctions } from '../../WarpPlugin'; import { VrfPluginFunctions } from '../../WarpPlugin';
import { BasicSortKeyCache } from '../../../cache/BasicSortKeyCache';
type EvaluationProgressInput = { type EvaluationProgressInput = {
contractTxId: string; contractTxId: string;
@@ -396,9 +397,9 @@ export abstract class DefaultStateEvaluator implements StateEvaluator {
abstract lastCachedSortKey(): Promise<string | null>; abstract lastCachedSortKey(): Promise<string | null>;
abstract setCache(cache: SortKeyCache<EvalStateResult<unknown>>): void; abstract setCache(cache: BasicSortKeyCache<EvalStateResult<unknown>>): void;
abstract getCache(): SortKeyCache<EvalStateResult<unknown>>; abstract getCache(): BasicSortKeyCache<EvalStateResult<unknown>>;
} }
function canBeCached(tx: GQLNodeInterface): boolean { function canBeCached(tx: GQLNodeInterface): boolean {

View File

@@ -11,9 +11,10 @@ import { DefinitionLoader } from '../DefinitionLoader';
import { WasmSrc } from './wasm/WasmSrc'; import { WasmSrc } from './wasm/WasmSrc';
import { Warp, WarpEnvironment } from '../../Warp'; import { Warp, WarpEnvironment } from '../../Warp';
import { TagsParser } from './TagsParser'; import { TagsParser } from './TagsParser';
import { CacheKey, SortKeyCache, SortKeyCacheResult } from '../../../cache/SortKeyCache'; import { CacheKey, SortKeyCacheResult } from '../../../cache/SortKeyCache';
import { Transaction } from '../../../utils/types/arweave-types'; import { Transaction } from '../../../utils/types/arweave-types';
import { getJsonResponse, stripTrailingSlash } from '../../../utils/utils'; import { getJsonResponse, stripTrailingSlash } from '../../../utils/utils';
import { BasicSortKeyCache } from '../../../cache/BasicSortKeyCache';
/** /**
* An extension to {@link ContractDefinitionLoader} that makes use of * An extension to {@link ContractDefinitionLoader} that makes use of
@@ -32,8 +33,8 @@ export class WarpGatewayContractDefinitionLoader implements DefinitionLoader {
constructor( constructor(
arweave: Arweave, arweave: Arweave,
private definitionCache: SortKeyCache<ContractCache<unknown>>, private definitionCache: BasicSortKeyCache<ContractCache<unknown>>,
private srcCache: SortKeyCache<SrcCache>, private srcCache: BasicSortKeyCache<SrcCache>,
private readonly env: WarpEnvironment private readonly env: WarpEnvironment
) { ) {
this.contractDefinitionLoader = new ContractDefinitionLoader(arweave, env); this.contractDefinitionLoader = new ContractDefinitionLoader(arweave, env);
@@ -102,19 +103,19 @@ export class WarpGatewayContractDefinitionLoader implements DefinitionLoader {
return 'warp'; return 'warp';
} }
setCache(cache: SortKeyCache<ContractCache<unknown>>): void { setCache(cache: BasicSortKeyCache<ContractCache<unknown>>): void {
this.definitionCache = cache; this.definitionCache = cache;
} }
setSrcCache(cacheSrc: SortKeyCache<SrcCache>): void { setSrcCache(cacheSrc: BasicSortKeyCache<SrcCache>): void {
this.srcCache = cacheSrc; this.srcCache = cacheSrc;
} }
getCache(): SortKeyCache<ContractCache<unknown>> { getCache(): BasicSortKeyCache<ContractCache<unknown>> {
return this.definitionCache; return this.definitionCache;
} }
getSrcCache(): SortKeyCache<SrcCache> { getSrcCache(): BasicSortKeyCache<SrcCache> {
return this.srcCache; return this.srcCache;
} }