Allow to read and write more dtypes

This commit is contained in:
Dimitri Lozeve 2023-10-04 17:15:13 +02:00
parent 9b2e9b184f
commit 2610d9e8b2
2 changed files with 29 additions and 12 deletions

29
npy.bqn
View file

@ -1,40 +1,45 @@
SaveNpy,LoadNpy SaveNpy,LoadNpy
lf@+10 lf@+10
magicString(@+147)"NUMPY" # "\x93NUMPY" magicString(@+147)"NUMPY"
# Type conversions
dtypesformats"<f8","<i4","<u4"64'f',32'i',32'u'
DtypeToFormat{(dtypes<𝕩)formats}
ArrayToBytes{DtypeToFormat 𝕨,8'c'•bit._cast 𝕩}
ArrayFromBytes{8'c',DtypeToFormat 𝕨•bit._cast 𝕩}
# Saving
To_i16le{@+256(|÷˜)𝕩} To_i16le{@+256(|÷˜)𝕩}
# To_i16le←⟨16,8‿'c'⟩•bit._cast∘≍ # assumes the platform is little-endian
FloatToBytes64,8'c'•bit._cast
FormatShape{'('(1','˘•Fmt¨𝕩)')'} FormatShape{'('(1','˘•Fmt¨𝕩)')'}
BuildHeader{𝕊𝕩: BuildHeader{dtype 𝕊 data:
version10 version10
# Hypothesis: BQN arrays are C-contiguous # Hypothesis: BQN arrays are C-contiguous
headerData"{'descr': '<f8', 'fortran_order': False, 'shape': "(FormatShape 𝕩)", }" headerData"{'descr':'"dtype"','fortran_order':False,'shape':"(FormatShape data)",}"
padding' '˜64-64|(magicString)+2+2+(headerData)+1 padding' '˜64-64|(magicString)+2+2+(headerData)+1
! 0=64|(magicString)+2+2+(padding)+(headerData)+1 ! 0=64|(magicString)+2+2+(padding)+(headerData)+1
headerDataPaddedheaderDatapaddinglf headerDataPaddedheaderDatapaddinglf
magicString(@+version)(To_i16le headerDataPadded)headerDataPadded magicString(@+version)(To_i16le headerDataPadded)headerDataPadded
} }
DetectDtype{(´=¨𝕩)"<f8",(´0𝕩)"<i4""<u4"}
EncodeNpyBuildHeaderFloatToBytes EncodeNpyDetectDtype(BuildHeaderArrayToBytes)
SaveNpy•file.BytesEncodeNpy SaveNpy•file.BytesEncodeNpy
# Loading
ParseHeader{𝕊𝕩: ParseHeader{𝕊𝕩:
"Not a valid NPY file"! magicString(magicString)𝕩 "Not a valid NPY file"! magicString(magicString)𝕩
version@-˜(01+magicString)𝕩 version@-˜(01+magicString)𝕩
headerlen+´1256×@-˜(23+magicString)𝕩 headerlen+´1256×@-˜(23+magicString)𝕩
header(¬(' 'lf))/headerlen(4+magicString)𝕩 header(¬(' 'lf))/headerlen(4+magicString)𝕩
dtype3('<') (("descr"header)1)header
shapestr(')') (1+'(') (("shape"header)1)header shapestr(')') (1+'(') (("shape"header)1)header
shape•BQN¨','((-˜+`׬)=)shapestr shape•BQN¨','((-˜+`׬)=)shapestr
version,shape,headerlen+2+2+magicString version,dtype,shape,headerlen+2+2+magicString
} }
LoadNpy{𝕊𝕩: LoadNpy{𝕊𝕩:
bytes•file.Bytes 𝕩 bytes•file.Bytes 𝕩
versionshapedatastartParseHeader bytes versiondtypeshapedatastartParseHeader bytes
shape8'c',64'f'•bit._cast datastartbytes shapedtype ArrayFromBytes datastartbytes
} }

View file

@ -5,3 +5,15 @@ a←3‿4‿5•rand.Range 0
bLoadNpy"testbqn.npy" bLoadNpy"testbqn.npy"
! ab ! ab
! ab ! ab
floatArr23•rand.Range 0
intArr50-˜23•rand.Range 100
uintArr23•rand.Range 100
"test_float.npy"SaveNpy floatArr
"test_int.npy"SaveNpy intArr
"test_uint.npy"SaveNpy uintArr
floatArrLoadNpy"test_float.npy"
intArrLoadNpy"test_int.npy"
uintArrLoadNpy"test_uint.npy"