feat: using Uint8Array as backing store for all box values

- also:
  - expand generic type info being captured to include type info for all
    levels
  - capture generic type info for variables initialised in a function to
    make test code simpler to write for boxes
  - update box and box map implementations and tests to work with encoders

Author: boblat

src/encoders.ts

@@ -0,0 +1,66 @@
+import { biguint, BigUint, bytes, Bytes, internal, TransactionType, uint64, Uint64 } from '@algorandfoundation/algorand-typescript'
+import { OnCompleteAction } from '@algorandfoundation/algorand-typescript/arc4'
+import { AccountCls } from './impl/account'
+import { ApplicationCls } from './impl/application'
+import { AssetCls } from './impl/asset'
+
+export interface GenericTypeInfo {
+  name: string
+  wtypeName?: string
+  genericArgs?: GenericTypeInfo[]
+}
+
+type fromBytes<T> = (val: Uint8Array, typeInfo: GenericTypeInfo) => T
+
+const booleanFromBytes: fromBytes<boolean> = (val) => {
+  return internal.encodingUtil.uint8ArrayToBigInt(val) > 0n
+}
+
+const bigUintFromBytes: fromBytes<biguint> = (val) => {
+  return BigUint(internal.encodingUtil.uint8ArrayToBigInt(val))
+}
+
+const bytesFromBytes: fromBytes<bytes> = (val) => {
+  return Bytes(val)
+}
+
+const stringFromBytes: fromBytes<string> = (val) => {
+  return Bytes(val).toString()
+}
+
+const uint64FromBytes: fromBytes<uint64> = (val) => {
+  return Uint64(internal.encodingUtil.uint8ArrayToBigInt(val))
+}
+
+const onCompletionFromBytes: fromBytes<OnCompleteAction> = (val) => {
+  return Uint64(internal.encodingUtil.uint8ArrayToBigInt(val)) as OnCompleteAction
+}
+
+const transactionTypeFromBytes: fromBytes<TransactionType> = (val) => {
+  return Uint64(internal.encodingUtil.uint8ArrayToBigInt(val)) as TransactionType
+}
+
+export const encoders = {
+  account: AccountCls.fromBytes,
+  Account: AccountCls.fromBytes,
+  application: ApplicationCls.fromBytes,
+  Application: ApplicationCls.fromBytes,
+  asset: AssetCls.fromBytes,
+  Asset: AssetCls.fromBytes,
+  bool: booleanFromBytes,
+  boolean: booleanFromBytes,
+  biguint: bigUintFromBytes,
+  bytes: bytesFromBytes,
+  string: stringFromBytes,
+  uint64: uint64FromBytes,
+  OnCompleteAction: onCompletionFromBytes,
+  TransactionType: transactionTypeFromBytes,
+}
+
+export const getEncoder = <T>(typeInfo: GenericTypeInfo): fromBytes<T> => {
+  const encoder = encoders[typeInfo.name as keyof typeof encoders]
+  if (!encoder) {
+    throw new Error(`No encoder found for type ${typeInfo.name}`)
+  }
+  return encoder as fromBytes<T>
+}

src/impl/application.ts

@@ -13,7 +13,7 @@ export class ApplicationData {
     appLogs: bytes[]
     globalStates: BytesMap<GlobalStateCls<unknown>>
     localStates: BytesMap<LocalState<unknown>>
-    boxes: BytesMap<unknown>
+    boxes: BytesMap<Uint8Array>
   }
   isCreating: boolean = false
 

src/impl/base.ts

@@ -1,13 +1,25 @@
-import { bytes, uint64 } from '@algorandfoundation/algorand-typescript'
+import { Bytes, bytes, Uint64, uint64 } from '@algorandfoundation/algorand-typescript'
+import { encodingUtil } from '@algorandfoundation/puya-ts'
+import type { GenericTypeInfo } from '../encoders'
 
 export abstract class BytesBackedCls {
   #value: bytes
+  // #typeInfo: GenericTypeInfo | undefined
 
   get bytes() {
     return this.#value
   }
-  constructor(value: bytes) {
+  constructor(value: bytes, _typeInfo?: GenericTypeInfo) {
     this.#value = value
+    // this.#typeInfo = typeInfo
+  }
+
+  static fromBytes<T extends BytesBackedCls>(
+    this: { new (v: bytes, typeInfo?: GenericTypeInfo): T },
+    value: Uint8Array,
+    typeInfo?: GenericTypeInfo,
+  ) {
+    return new this(Bytes(value), typeInfo)
   }
 }
 
@@ -21,4 +33,9 @@ export abstract class Uint64BackedCls {
   constructor(value: uint64) {
     this.#value = value
   }
+
+  static fromBytes<T extends Uint64BackedCls>(this: { new (v: uint64): T }, value: Uint8Array) {
+    const uint64Value = Uint64(encodingUtil.uint8ArrayToBigInt(value))
+    return new this(uint64Value)
+  }
 }

src/impl/box.ts

@@ -1,7 +1,7 @@
 import { bytes, internal, uint64 } from '@algorandfoundation/algorand-typescript'
 import { MAX_BOX_SIZE } from '../constants'
 import { lazyContext } from '../context-helpers/internal-context'
-import { asBytes, asBytesCls, asNumber, conactUint8Arrays, toBytes } from '../util'
+import { asBytes, asBytesCls, asNumber, asUint8Array, conactUint8Arrays, toBytes } from '../util'
 
 export const Box: internal.opTypes.BoxType = {
   create(a: internal.primitives.StubBytesCompat, b: internal.primitives.StubUint64Compat): boolean {
@@ -35,11 +35,7 @@ export const Box: internal.opTypes.BoxType = {
       throw new internal.errors.InternalError('Box does not exist')
     }
     const boxContent = lazyContext.ledger.getBox(app, name)
-    if (boxContent instanceof Uint8Array) {
-      return toBytes(boxContent.slice(start, start + length))
-    }
-    const result = toBytes(boxContent).slice(start, start + length)
-    return result
+    return toBytes(boxContent.slice(start, start + length))
   },
   get(a: internal.primitives.StubBytesCompat): readonly [bytes, boolean] {
     const name = asBytes(a)
@@ -52,43 +48,34 @@ export const Box: internal.opTypes.BoxType = {
     const app = lazyContext.activeApplication
     const boxContent = lazyContext.ledger.getBox(app, name)
     const exists = lazyContext.ledger.boxExists(app, name)
-    if (boxContent instanceof Uint8Array) {
-      return [boxContent.length, exists]
-    }
-    const bytesContent = toBytes(boxContent)
-    return [bytesContent.length, exists]
+    return [boxContent.length, exists]
   },
   put(a: internal.primitives.StubBytesCompat, b: internal.primitives.StubBytesCompat): void {
     const name = asBytes(a)
     const app = lazyContext.activeApplication
-    const newContent = asBytes(b)
+    const newContent = asBytesCls(b)
     if (lazyContext.ledger.boxExists(app, name)) {
       const boxContent = lazyContext.ledger.getBox(app, name)
-      const length = boxContent instanceof Uint8Array ? boxContent.length : toBytes(boxContent).length
+      const length = boxContent.length
       if (asNumber(length) !== asNumber(newContent.length)) {
         throw new internal.errors.InternalError('New content length does not match existing box length')
       }
     }
-    lazyContext.ledger.setBox(app, name, newContent)
+    lazyContext.ledger.setBox(app, name, newContent.asUint8Array())
   },
   replace(a: internal.primitives.StubBytesCompat, b: internal.primitives.StubUint64Compat, c: internal.primitives.StubBytesCompat): void {
     const name = asBytes(a)
     const start = asNumber(b)
-    const newContent = asBytesCls(c).asUint8Array()
+    const newContent = asUint8Array(c)
     const app = lazyContext.activeApplication
     if (!lazyContext.ledger.boxExists(app, name)) {
       throw new internal.errors.InternalError('Box does not exist')
     }
     const boxContent = lazyContext.ledger.getBox(app, name)
-    const uint8ArrayContent = boxContent instanceof Uint8Array ? boxContent : asBytesCls(toBytes(boxContent)).asUint8Array()
-    if (start + newContent.length > uint8ArrayContent.length) {
+    if (start + newContent.length > boxContent.length) {
       throw new internal.errors.InternalError('Replacement content exceeds box size')
     }
-    const updatedContent = conactUint8Arrays(
-      uint8ArrayContent.slice(0, start),
-      newContent,
-      uint8ArrayContent.slice(start + newContent.length),
-    )
+    const updatedContent = conactUint8Arrays(boxContent.slice(0, start), newContent, boxContent.slice(start + newContent.length))
     lazyContext.ledger.setBox(app, name, updatedContent)
   },
   resize(a: internal.primitives.StubBytesCompat, b: internal.primitives.StubUint64Compat): void {
@@ -99,13 +86,12 @@ export const Box: internal.opTypes.BoxType = {
       throw new internal.errors.InternalError('Box does not exist')
     }
     const boxContent = lazyContext.ledger.getBox(app, name)
-    const uint8ArrayContent = boxContent instanceof Uint8Array ? boxContent : asBytesCls(toBytes(boxContent)).asUint8Array()
-    const size = uint8ArrayContent.length
+    const size = boxContent.length
     let updatedContent
     if (newSize > size) {
-      updatedContent = conactUint8Arrays(uint8ArrayContent, new Uint8Array(Array(newSize - size).fill(0)))
+      updatedContent = conactUint8Arrays(boxContent, new Uint8Array(Array(newSize - size).fill(0)))
     } else {
-      updatedContent = uint8ArrayContent.slice(0, newSize)
+      updatedContent = boxContent.slice(0, newSize)
     }
     lazyContext.ledger.setBox(app, name, updatedContent)
   },
@@ -118,19 +104,18 @@ export const Box: internal.opTypes.BoxType = {
     const name = asBytes(a)
     const start = asNumber(b)
     const length = asNumber(c)
-    const newContent = asBytesCls(d).asUint8Array()
+    const newContent = asUint8Array(d)
     const app = lazyContext.activeApplication
     if (!lazyContext.ledger.boxExists(app, name)) {
       throw new internal.errors.InternalError('Box does not exist')
     }
     const boxContent = lazyContext.ledger.getBox(app, name)
-    const uint8ArrayContent = boxContent instanceof Uint8Array ? boxContent : asBytesCls(toBytes(boxContent)).asUint8Array()
-    const size = uint8ArrayContent.length
+    const size = boxContent.length
     if (start > size) {
       throw new internal.errors.InternalError('Start index exceeds box size')
     }
     const end = Math.min(start + length, size)
-    let updatedContent = conactUint8Arrays(uint8ArrayContent.slice(0, start), newContent, uint8ArrayContent.slice(end))
+    let updatedContent = conactUint8Arrays(boxContent.slice(0, start), newContent, boxContent.slice(end))
     //  Adjust the size if necessary
     if (updatedContent.length > size) {
       updatedContent = updatedContent.slice(0, size)

src/impl/state.ts

@@ -4,6 +4,7 @@ import {
   BoxMap as BoxMapType,
   BoxRef as BoxRefType,
   Box as BoxType,
+  Bytes,
   bytes,
   GlobalStateOptions,
   GlobalState as GlobalStateType,
@@ -16,7 +17,9 @@ import {
 import { AccountMap } from '../collections/custom-key-map'
 import { MAX_BOX_SIZE } from '../constants'
 import { lazyContext } from '../context-helpers/internal-context'
-import { asBytes, asBytesCls, asNumber, conactUint8Arrays, toBytes } from '../util'
+import { getEncoder } from '../encoders'
+import { getGenericTypeInfo } from '../runtime-helpers'
+import { asBytes, asBytesCls, asNumber, asUint8Array, conactUint8Arrays, toBytes } from '../util'
 
 export class GlobalStateCls<ValueType> {
   private readonly _type: string = GlobalStateCls.name
@@ -126,14 +129,21 @@ export class BoxCls<TValue> {
     this.#app = lazyContext.activeApplication
   }
 
+  private get fromBytes() {
+    const typeInfo = getGenericTypeInfo(this)
+    const valueType = typeInfo!.genericArgs![0]
+    return (val: Uint8Array) => getEncoder<TValue>(valueType)(val, valueType)
+  }
+
   get value(): TValue {
     if (!this.exists) {
       throw new internal.errors.InternalError('Box has not been created')
     }
-    return lazyContext.ledger.getBox(this.#app, this.key)
+
+    return this.fromBytes(lazyContext.ledger.getBox(this.#app, this.key))
   }
   set value(v: TValue) {
-    lazyContext.ledger.setBox(this.#app, this.key, v)
+    lazyContext.ledger.setBox(this.#app, this.key, asUint8Array(toBytes(v)))
   }
 
   get hasKey(): boolean {
@@ -172,7 +182,8 @@ export class BoxCls<TValue> {
   }
 
   maybe(): readonly [TValue, boolean] {
-    return [lazyContext.ledger.getBox(this.#app, this.key), lazyContext.ledger.boxExists(this.#app, this.key)]
+    const value = this.fromBytes(lazyContext.ledger.getBox(this.#app, this.key))
+    return [value, lazyContext.ledger.boxExists(this.#app, this.key)]
   }
 }
 
@@ -186,6 +197,12 @@ export class BoxMapCls<TKey, TValue> {
     return x instanceof Object && '_type' in x && (x as { _type: string })['_type'] === BoxMapCls.name
   }
 
+  private get fromBytes() {
+    const typeInfo = getGenericTypeInfo(this)
+    const valueType = typeInfo!.genericArgs![1]
+    return (val: Uint8Array) => getEncoder<TValue>(valueType)(val, valueType)
+  }
+
   constructor(keyPrefix?: internal.primitives.StubBytesCompat) {
     this.#keyPrefix = keyPrefix ? asBytes(keyPrefix) : undefined
     this.#app = lazyContext.activeApplication
@@ -215,7 +232,7 @@ export class BoxMapCls<TKey, TValue> {
   }
 
   set(key: TKey, value: TValue): void {
-    lazyContext.ledger.setBox(this.#app, this.getFullKey(key), value)
+    lazyContext.ledger.setBox(this.#app, this.getFullKey(key), asUint8Array(toBytes(value)))
   }
 
   delete(key: TKey): boolean {
@@ -228,7 +245,8 @@ export class BoxMapCls<TKey, TValue> {
 
   maybe(key: TKey): readonly [TValue, boolean] {
     const fullKey = this.getFullKey(key)
-    return [lazyContext.ledger.getBox(this.#app, fullKey), lazyContext.ledger.boxExists(this.#app, fullKey)]
+    const value = this.fromBytes(lazyContext.ledger.getBox(this.#app, fullKey))
+    return [value, lazyContext.ledger.boxExists(this.#app, fullKey)]
   }
 
   length(key: TKey): uint64 {
@@ -394,7 +412,7 @@ export class BoxRefCls {
   }
 
   maybe(): readonly [bytes, boolean] {
-    return [asBytes(lazyContext.ledger.getBox(this.#app, this.key)), lazyContext.ledger.boxExists(this.#app, this.key)]
+    return [Bytes(lazyContext.ledger.getBox(this.#app, this.key)), lazyContext.ledger.boxExists(this.#app, this.key)]
   }
 
   get length(): uint64 {

src/runtime-helpers.ts

@@ -1,5 +1,6 @@
 import { internal } from '@algorandfoundation/algorand-typescript'
 import { MAX_UINT64 } from './constants'
+import type { GenericTypeInfo } from './encoders'
 import { DeliberateAny } from './typescript-helpers'
 import { nameOfType } from './util'
 
@@ -273,12 +274,12 @@ function defaultUnaryOp(_operand: DeliberateAny, op: UnaryOps): DeliberateAny {
   internal.errors.internalError(`Unsupported operator ${op}`)
 }
 
-const genericTypeMap = new Map<DeliberateAny, string>()
+const genericTypeMap = new Map<DeliberateAny, GenericTypeInfo>()
 export function captureGenericTypeInfo(target: DeliberateAny, t: string) {
-  genericTypeMap.set(target, t)
+  genericTypeMap.set(target, JSON.parse(t))
   return target
 }
 
-export function getGenericTypeInfo(target: DeliberateAny): string | undefined {
+export function getGenericTypeInfo(target: DeliberateAny): GenericTypeInfo | undefined {
   return genericTypeMap.get(target)
 }

src/subcontexts/contract-context.ts

@@ -2,6 +2,7 @@ import { Account, Application, Asset, BaseContract, Bytes, bytes, Contract, Loca
 import { getAbiMetadata } from '../abi-metadata'
 import { BytesMap } from '../collections/custom-key-map'
 import { lazyContext } from '../context-helpers/internal-context'
+import type { GenericTypeInfo } from '../encoders'
 import { AccountCls } from '../impl/account'
 import { ApplicationCls } from '../impl/application'
 import { AssetCls } from '../impl/asset'
@@ -17,7 +18,7 @@ import {
 } from '../impl/transactions'
 import { getGenericTypeInfo } from '../runtime-helpers'
 import { DeliberateAny } from '../typescript-helpers'
-import { asUint64Cls, extractGenericTypeArgs } from '../util'
+import { asUint64Cls } from '../util'
 
 interface IConstructor<T> {
   new (...args: DeliberateAny[]): T
@@ -31,10 +32,9 @@ interface States {
   totals: StateTotals
 }
 
-const isUint64GenericType = (typeName: string | undefined) => {
-  if (typeName === undefined) return false
-  const genericTypes: string[] = extractGenericTypeArgs(typeName)
-  return genericTypes.some((t) => t.toLocaleLowerCase() === 'uint64')
+const isUint64GenericType = (typeInfo: GenericTypeInfo | undefined) => {
+  if (!typeInfo?.genericArgs?.length) return false
+  return typeInfo.genericArgs.some((t) => t.name.toLocaleLowerCase() === 'uint64')
 }
 
 const extractStates = (contract: BaseContract): States => {

src/subcontexts/ledger-context.ts

@@ -153,12 +153,12 @@ export class LedgerContext {
     }
   }
 
-  getBox<TValue>(app: Application, key: internal.primitives.StubBytesCompat): TValue {
+  getBox(app: Application, key: internal.primitives.StubBytesCompat): Uint8Array {
     const appData = this.applicationDataMap.getOrFail(app.id)
-    return appData.application.boxes.get(key) as TValue
+    return appData.application.boxes.get(key) ?? new Uint8Array()
   }
 
-  setBox<TValue>(app: Application, key: internal.primitives.StubBytesCompat, value: TValue): void {
+  setBox(app: Application, key: internal.primitives.StubBytesCompat, value: Uint8Array): void {
     const appData = this.applicationDataMap.getOrFail(app.id)
     appData.application.boxes.set(key, value)
   }