jsonutils.nim

##[
This module implements a hookable (de)serialization for arbitrary types using JSON.
Design goal: avoid importing modules where a custom serialization is needed;
see strtabs.fromJsonHook,toJsonHook for an example.
]##

runnableExamples:
import std/[strtabs,json]
typeFoo=refobject
 t: bool
 z1: int8
let a = (1.5'f32, (b: "b2", a: "a2"), 'x', @[Foo(t: true, z1: -3), nil], [{"name": "John"}.newStringTable])
let j = a.toJson
assert j.jsonTo(typeof(a)).toJson == j
assert$[NaN, Inf, -Inf, 0.0, -0.0, 1.0, 1e-2].toJson =="""["nan","inf","-inf",0.0,-0.0,1.0,0.01]"""
assert0.0.toJson.kind ==JFloat
assertInf.toJson.kind ==JString

import std/[json, strutils, tables, sets, strtabs, options]

#[
Future directions:
add a way to customize serialization, for e.g.:
* field renaming
* allow serializing `enum` and `char` as `string` instead of `int`
 (enum is more compact/efficient, and robust to enum renamings, but string
 is more human readable)
* handle cyclic references, using a cache of already visited addresses
* implement support for serialization and de-serialization of nested variant
 objects.
]#

import std/macros
from std/enumutils import symbolName
from std/typetraits importOrdinalEnum, tupleLen

whendefined(nimPreviewSlimSystem):
import std/assertions


procisNamedTuple(T: typedesc): bool {.magic: "TypeTrait".}

type
Joptions*=object# xxx rename FromJsonOptions
## Options controlling the behavior of `fromJson`.
 allowExtraKeys*: bool
## If `true` Nim's object to which the JSON is parsed is not required to
## have a field for every JSON key.
 allowMissingKeys*: bool
## If `true` Nim's object to which JSON is parsed is allowed to have
## fields without corresponding JSON keys.
# in future work: a key rename could be added
EnumMode*=enum
 joptEnumOrd
 joptEnumSymbol
 joptEnumString
JsonNodeMode*=enum## controls `toJson` for JsonNode types
 joptJsonNodeAsRef ## returns the ref as is
 joptJsonNodeAsCopy ## returns a deep copy of the JsonNode
 joptJsonNodeAsObject ## treats JsonNode as a regular ref object
ToJsonOptions*=object
 enumMode*: EnumMode
 jsonNodeMode*: JsonNodeMode
# xxx charMode, etc

procinitToJsonOptions*(): ToJsonOptions=
## initializes `ToJsonOptions` with sane options.
ToJsonOptions(enumMode: joptEnumOrd, jsonNodeMode: joptJsonNodeAsRef)

procdistinctBase(T: typedesc, recursive: staticbool=true): typedesc {.magic: "TypeTrait".}
templatedistinctBase[T](a: T, recursive: staticbool=true): untyped=distinctBase(typeof(a), recursive)(a)

macrogetDiscriminants(a: typedesc): seq[string] =
## return the discriminant keys
# candidate for std/typetraits
var a = a.getTypeImpl
doAssert a.kind == nnkBracketExpr
let sym = a[1]
let t = sym.getTypeImpl
let t2 = t[2]
case t2.kind
 of nnkEmpty: # allow empty objects
result=quotedo:
seq[string].default
 of nnkRecList:
result=newTree(nnkBracket)
for ti in t2:
if ti.kind == nnkRecCase:
let key = ti[0][0]
result.addnewLit key.strVal
ifresult.len >0:
result=quotedo:
@`result`
else:
result=quotedo:
seq[string].default
else:
raiseAssert"unexpected kind: "&$t2.kind

macroinitCaseObject(T: typedesc, fun: untyped): untyped=
## does the minimum to construct a valid case object, only initializing
## the discriminant fields; see also `getDiscriminants`
# maybe candidate for std/typetraits
var a = T.getTypeImpl
doAssert a.kind == nnkBracketExpr
let sym = a[1]
let t = sym.getTypeImpl
var t2: NimNode
case t.kind
 of nnkObjectTy: t2 = t[2]
 of nnkRefTy: t2 = t[0].getTypeImpl[2]
else: raiseAssert$t.kind # xxx `nnkPtrTy` could be handled too
doAssert t2.kind == nnkRecList
result=newTree(nnkObjConstr)
result.add T
for ti in t2:
if ti.kind == nnkRecCase:
let key = ti[0][0]
let typ = ti[0][1]
let key2 = key.strVal
let val =quotedo:
`fun`(`key2`, typedesc[`typ`])
result.addnewTree(nnkExprColonExpr, key, val)

procraiseJsonException(condStr: string, msg: string) {.noinline.} =
# just pick 1 exception type for simplicity; other choices would be:
# JsonError, JsonParser, JsonKindError
raisenewException(ValueError, condStr &" failed: "& msg)

templatecheckJson(cond: untyped, msg ="") =
ifnot cond:
raiseJsonException(astToStr(cond), msg)

prochasField[T](obj: T, field: string): bool=
for k, _ infieldPairs(obj):
if k == field:
returntrue
returnfalse

macroaccessField(obj: typed, name: staticstring): untyped=
newDotExpr(obj, ident(name))

templatefromJsonFields(newObj, oldObj, json, discKeys, opt) =
type T =typeof(newObj)
# we could customize whether to allow JNull
checkJson json.kind ==JObject, $json.kind
var num, numMatched =0
for key, val infieldPairs(newObj):
 num.inc
when key notin discKeys:
if json.hasKey key:
 numMatched.inc
fromJson(val, json[key], opt)
elif opt.allowMissingKeys:
# if there are no discriminant keys the `oldObj` must always have the
# same keys as the new one. Otherwise we must check, because they could
# be set to different branches.
whentypeof(oldObj) isnottypeof(nil):
if discKeys.len ==0orhasField(oldObj, key):
 val =accessField(oldObj, key)
else:
checkJsonfalse, "key '$1' for $2 not in $3"% [key, $T, json.pretty()]
else:
if json.hasKey key:
 numMatched.inc

let ok =
if opt.allowExtraKeys and opt.allowMissingKeys:
true
elif opt.allowExtraKeys:
# This check is redundant because if here missing keys are not allowed,
# and if `num != numMatched` it will fail in the loop above but it is left
# for clarity.
assert num == numMatched
 num == numMatched
elif opt.allowMissingKeys:
 json.len == numMatched
else:
 json.len == num and num == numMatched

checkJson ok, "There were $1 keys (expecting $2) for $3 with $4"% [$json.len, $num, $T, json.pretty()]

procfromJson*[T](a: var T, b: JsonNode, opt =Joptions())

procdiscKeyMatch[T](obj: T, json: JsonNode, key: staticstring): bool=
ifnot json.hasKey key:
returntrue
let field =accessField(obj, key)
var jsonVal: typeof(field)
fromJson(jsonVal, json[key])
if jsonVal != field:
returnfalse
returntrue

macrodiscKeysMatchBodyGen(obj: typed, json: JsonNode,
 keys: staticseq[string]): untyped=
result=newStmtList()
let r =ident("result")
for key in keys:
let keyLit =newLit key
result.addquotedo:
 `r` = `r` anddiscKeyMatch(`obj`, `json`, `keyLit`)

procdiscKeysMatch[T](obj: T, json: JsonNode, keys: staticseq[string]): bool=
result=true
discKeysMatchBodyGen(obj, json, keys)

procjsonTo*(b: JsonNode, T: typedesc, opt =Joptions()): T

procfromJson*[T](a: var T, b: JsonNode, opt =Joptions()) =
## inplace version of `jsonTo`
#[
 adding "json path" leading to `b` can be added in future work.
]#
checkJson b !=nil, $($T, b)
whencompiles(fromJsonHook(a, b, opt)): fromJsonHook(a, b, opt)
elifcompiles(fromJsonHook(a, b)): fromJsonHook(a, b)
elif T isbool: a =to(b,T)
elif T isenum:
case b.kind
 ofJInt: a =T(b.getBiggestInt())
 ofJString: a =parseEnum[T](b.getStr())
else: checkJsonfalse, "Expecting int/string for "&$T &" got "& b.pretty()
elif T isuint|uint64: a =T(to(b, uint64))
elif T isOrdinal: a =cast[T](to(b, int))
elif T ispointer: a =cast[pointer](to(b, int))
elif T isdistinct: a.distinctBase.fromJson(b)
elif T isstring|SomeNumber: a =to(b,T)
elif T iscstring:
case b.kind
 ofJNull: a =nil
 ofJString: a = b.str
else: checkJsonfalse, "Expecting null/string for "&$T &" got "& b.pretty()
elif T isJsonNode: a = b
elif T isref|ptr:
if b.kind ==JNull: a =nil
else:
 a =T()
fromJson(a[], b, opt)
elif T isarray:
checkJson a.len == b.len, "Json array size doesn't match for "&$T
var i =0
for ai inmitems(a):
fromJson(ai, b[i], opt)
 i.inc
elif T isset:
type E =typeof(for ai in a: ai)
for val in b.getElems:
incl a, jsonTo(val, E)
elif T isseq:
 a.setLen b.len
for i, val in b.getElems:
fromJson(a[i], val, opt)
elif T isobject:
templatefun(key, typ): untyped {.used.} =
if b.hasKey key:
jsonTo(b[key], typ)
elifhasField(a, key):
accessField(a, key)
else:
default(typ)
const keys =getDiscriminants(T)
when keys.len ==0:
fromJsonFields(a, nil, b, keys, opt)
else:
ifdiscKeysMatch(a, b, keys):
fromJsonFields(a, nil, b, keys, opt)
else:
var newObj =initCaseObject(T, fun)
fromJsonFields(newObj, a, b, keys, opt)
 a = newObj
elif T istuple:
whenisNamedTuple(T):
fromJsonFields(a, nil, b, seq[string].default, opt)
else:
checkJson b.kind ==JArray, $(b.kind) # we could customize whether to allow JNull

whencompiles(tupleLen(T)):
let tupleSize =tupleLen(T)
else:
# Tuple len isn't in csources_v1 so using tupleLen would fail.
# Else branch basically never runs (tupleLen added in 1.1 and jsonutils in 1.4), but here for consistency
var tupleSize =0
for val infields(a):
 tupleSize.inc

checkJson b.len == tupleSize, "Json doesn't match expected length of "&$tupleSize &", got "& b.pretty()
var i =0
for val infields(a):
fromJson(val, b[i], opt)
 i.inc
else:
# checkJson not appropriate here
static: raiseAssert"not yet implemented: "&$T

procjsonTo*(b: JsonNode, T: typedesc, opt =Joptions()): T =
## reverse of `toJson`
fromJson(result, b, opt)

proctoJson*[T](a: T, opt =initToJsonOptions()): JsonNode=
## serializes `a` to json; uses `toJsonHook(a: T)` if it's in scope to
## customize serialization, see strtabs.toJsonHook for an example.
##
## .. note:: With `-d:nimPreviewJsonutilsHoleyEnum`, `toJson` now can 
## serialize/deserialize holey enums as regular enums (via `ord`) instead of as strings.
## It is expected that this behavior becomes the new default in upcoming versions.
whencompiles(toJsonHook(a, opt)): result=toJsonHook(a, opt)
elifcompiles(toJsonHook(a)): result=toJsonHook(a)
elif T isobject|tuple:
when T isobjectorisNamedTuple(T):
result=newJObject()
for k, v in a.fieldPairs: result[k] =toJson(v, opt)
else:
result=newJArray()
for v in a.fields: result.addtoJson(v, opt)
elif T isref|ptr:
templateimpl=
if system.`==`(a, nil): result=newJNull()
else: result=toJson(a[], opt)
when T isJsonNode:
case opt.jsonNodeMode
 of joptJsonNodeAsRef: result= a
 of joptJsonNodeAsCopy: result=copy(a)
 of joptJsonNodeAsObject: impl()
else: impl()
elif T isarray|seq|set:
result=newJArray()
for ai in a: result.addtoJson(ai, opt)
elif T ispointer: result=toJson(cast[int](a), opt)
# edge case: `a == nil` could've also led to `newJNull()`, but this results
# in simpler code for `toJson` and `fromJson`.
elif T isdistinct: result=toJson(a.distinctBase, opt)
elif T isbool: result=%(a)
elif T isSomeInteger: result=%a
elif T isenum:
case opt.enumMode
 of joptEnumOrd:
when T isOrdinalordefined(nimPreviewJsonutilsHoleyEnum): %(a.ord)
else: toJson($a, opt)
 of joptEnumSymbol:
when T isOrdinalEnum:
toJson(symbolName(a), opt)
else:
toJson($a, opt)
 of joptEnumString: toJson($a, opt)
elif T isOrdinal: result=%(a.ord)
elif T iscstring: (if a ==nil: result=newJNull() else: result=%$a)
else: result=%a

procfromJsonHook*[K: string|cstring, V](t: var (Table[K, V] |OrderedTable[K, V]),
 jsonNode: JsonNode, opt =Joptions()) =
## Enables `fromJson` for `Table` and `OrderedTable` types.
##
## See also:
## * `toJsonHook proc<#toJsonHook>`_
 runnableExamples:
import std/[tables, json]
var foo: tuple[t: Table[string, int], ot: OrderedTable[string, int]]
fromJson(foo, parseJson("""
 {"t":{"two":2,"one":1},"ot":{"one":1,"three":3}}"""))
assert foo.t == [("one", 1), ("two", 2)].toTable
assert foo.ot == [("one", 1), ("three", 3)].toOrderedTable

assert jsonNode.kind ==JObject,
"The kind of the `jsonNode` must be `JObject`, but its actual "&
"type is `"&$jsonNode.kind &"`."
clear(t)
for k, v in jsonNode:
 t[k] =jsonTo(v, V, opt)

proctoJsonHook*[K: string|cstring, V](t: (Table[K, V] |OrderedTable[K, V]), opt =initToJsonOptions()): JsonNode=
## Enables `toJson` for `Table` and `OrderedTable` types.
##
## See also:
## * `fromJsonHook proc<#fromJsonHook,,JsonNode>`_
 runnableExamples:
import std/[tables, json, sugar]
let foo = (
 t: [("two", 2)].toTable,
 ot: [("one", 1), ("three", 3)].toOrderedTable)
assert$toJson(foo) =="""{"t":{"two":2},"ot":{"one":1,"three":3}}"""
# if keys are not string|cstring, you can use this:
let a = {10: "foo", 11: "bar"}.newOrderedTable
let a2 = collect: (for k,v in a: (k,v))
assert$toJson(a2) =="""[[10,"foo"],[11,"bar"]]"""

result=newJObject()
for k, v inpairs(t):
# not sure if $k has overhead for string
result[(when K isstring: k else: $k)] =toJson(v, opt)

procfromJsonHook*[A](s: varSomeSet[A], jsonNode: JsonNode, opt =Joptions()) =
## Enables `fromJson` for `HashSet` and `OrderedSet` types.
##
## See also:
## * `toJsonHook proc<#toJsonHook,SomeSet[A]>`_
 runnableExamples:
import std/[sets, json]
var foo: tuple[hs: HashSet[string], os: OrderedSet[string]]
fromJson(foo, parseJson("""
 {"hs": ["hash", "set"], "os": ["ordered", "set"]}"""))
assert foo.hs == ["hash", "set"].toHashSet
assert foo.os == ["ordered", "set"].toOrderedSet

assert jsonNode.kind ==JArray,
"The kind of the `jsonNode` must be `JArray`, but its actual "&
"type is `"&$jsonNode.kind &"`."
clear(s)
for v in jsonNode:
incl(s, jsonTo(v, A, opt))

proctoJsonHook*[A](s: SomeSet[A], opt =initToJsonOptions()): JsonNode=
## Enables `toJson` for `HashSet` and `OrderedSet` types.
##
## See also:
## * `fromJsonHook proc<#fromJsonHook,SomeSet[A],JsonNode>`_
 runnableExamples:
import std/[sets, json]
let foo = (hs: ["hash"].toHashSet, os: ["ordered", "set"].toOrderedSet)
assert$toJson(foo) =="""{"hs":["hash"],"os":["ordered","set"]}"""

result=newJArray()
for k in s:
add(result, toJson(k, opt))

procfromJsonHook*[T](self: varOption[T], jsonNode: JsonNode, opt =Joptions()) =
## Enables `fromJson` for `Option` types.
##
## See also:
## * `toJsonHook proc<#toJsonHook,Option[T]>`_
 runnableExamples:
import std/[options, json]
var opt: Option[string]
fromJsonHook(opt, parseJson("\"test\""))
assertget(opt) =="test"
fromJson(opt, parseJson("null"))
assertisNone(opt)

if jsonNode.kind !=JNull:
 self =some(jsonTo(jsonNode, T, opt))
else:
 self =none[T]()

proctoJsonHook*[T](self: Option[T], opt =initToJsonOptions()): JsonNode=
## Enables `toJson` for `Option` types.
##
## See also:
## * `fromJsonHook proc<#fromJsonHook,Option[T],JsonNode>`_
 runnableExamples:
import std/[options, json]
let optSome =some("test")
assert$toJson(optSome) =="\"test\""
let optNone =none[string]()
assert$toJson(optNone) =="null"

ifisSome(self):
toJson(get(self), opt)
else:
newJNull()

procfromJsonHook*(a: varStringTableRef, b: JsonNode) =
## Enables `fromJson` for `StringTableRef` type.
##
## See also:
## * `toJsonHook proc<#toJsonHook,StringTableRef>`_
 runnableExamples:
import std/[strtabs, json]
var t =newStringTable(modeCaseSensitive)
let jsonStr ="""{"mode": 0, "table": {"name": "John", "surname": "Doe"}}"""
fromJsonHook(t, parseJson(jsonStr))
assert t[] ==newStringTable("name", "John", "surname", "Doe",
 modeCaseSensitive)[]

var mode =jsonTo(b["mode"], StringTableMode)
 a =newStringTable(mode)
let b2 = b["table"]
for k,v in b2: a[k] =jsonTo(v, string)

proctoJsonHook*(a: StringTableRef): JsonNode=
## Enables `toJson` for `StringTableRef` type.
##
## See also:
## * `fromJsonHook proc<#fromJsonHook,StringTableRef,JsonNode>`_
 runnableExamples:
import std/[strtabs, json]
let t =newStringTable("name", "John", "surname", "Doe", modeCaseSensitive)
let jsonStr ="""{"mode": "modeCaseSensitive",
 "table": {"name": "John", "surname": "Doe"}}"""
asserttoJson(t) ==parseJson(jsonStr)

result=newJObject()
result["mode"] =toJson($a.mode)
let t =newJObject()
for k,v in a: t[k] =toJson(v)
result["table"] = t