Allow to read and write more dtypes
This commit is contained in:
parent
9b2e9b184f
commit
2610d9e8b2
2 changed files with 29 additions and 12 deletions
29
npy.bqn
29
npy.bqn
|
@ -1,40 +1,45 @@
|
||||||
⟨SaveNpy,LoadNpy⟩⇐
|
⟨SaveNpy,LoadNpy⟩⇐
|
||||||
|
|
||||||
lf←@+10
|
lf←@+10
|
||||||
magicString←(@+147)∾"NUMPY" # "\x93NUMPY"
|
magicString←(@+147)∾"NUMPY"
|
||||||
|
|
||||||
|
# Type conversions
|
||||||
|
dtypes‿formats←⟨"<f8","<i4","<u4"⟩‿⟨64‿'f',32‿'i',32‿'u'⟩
|
||||||
|
DtypeToFormat←{(⊑dtypes⊐<𝕩)⊑formats}
|
||||||
|
ArrayToBytes←{⟨DtypeToFormat 𝕨,8‿'c'⟩•bit._cast 𝕩}
|
||||||
|
ArrayFromBytes←{⟨8‿'c',DtypeToFormat 𝕨⟩•bit._cast 𝕩}
|
||||||
|
|
||||||
|
# Saving
|
||||||
To_i16le←{@+256(|∾⌊∘÷˜)𝕩}
|
To_i16le←{@+256(|∾⌊∘÷˜)𝕩}
|
||||||
# To_i16le←⟨16,8‿'c'⟩•bit._cast∘≍ # assumes the platform is little-endian
|
|
||||||
FloatToBytes←⟨64,8‿'c'⟩•bit._cast
|
|
||||||
|
|
||||||
FormatShape←{'('∾(1↓∾⥊','∾˘•Fmt¨≢𝕩)∾')'}
|
FormatShape←{'('∾(1↓∾⥊','∾˘•Fmt¨≢𝕩)∾')'}
|
||||||
|
|
||||||
BuildHeader←{𝕊𝕩:
|
BuildHeader←{dtype 𝕊 data:
|
||||||
version←1‿0
|
version←1‿0
|
||||||
# Hypothesis: BQN arrays are C-contiguous
|
# Hypothesis: BQN arrays are C-contiguous
|
||||||
headerData←"{'descr': '<f8', 'fortran_order': False, 'shape': "∾(FormatShape 𝕩)∾", }"
|
headerData←"{'descr':'"∾dtype∾"','fortran_order':False,'shape':"∾(FormatShape data)∾",}"
|
||||||
padding←' '↑˜64-64|(≠magicString)+2+2+(≠headerData)+1
|
padding←' '↑˜64-64|(≠magicString)+2+2+(≠headerData)+1
|
||||||
! 0=64|(≠magicString)+2+2+(≠padding)+(≠headerData)+1
|
! 0=64|(≠magicString)+2+2+(≠padding)+(≠headerData)+1
|
||||||
headerDataPadded←headerData∾padding∾lf
|
headerDataPadded←headerData∾padding∾lf
|
||||||
magicString∾(@+version)∾(To_i16le ≠headerDataPadded)∾headerDataPadded
|
magicString∾(@+version)∾(To_i16le ≠headerDataPadded)∾headerDataPadded
|
||||||
}
|
}
|
||||||
|
DetectDtype←{(∧´⥊⌊⊸=¨𝕩)⊑⟨"<f8",(∧´⥊0≤𝕩)⊑"<i4"‿"<u4"⟩}
|
||||||
EncodeNpy←BuildHeader∾FloatToBytes∘⥊
|
EncodeNpy←DetectDtype⊸(BuildHeader∾ArrayToBytes⟜⥊)
|
||||||
|
|
||||||
SaveNpy←•file.Bytes⟜EncodeNpy
|
SaveNpy←•file.Bytes⟜EncodeNpy
|
||||||
|
|
||||||
|
# Loading
|
||||||
ParseHeader←{𝕊𝕩:
|
ParseHeader←{𝕊𝕩:
|
||||||
"Not a valid NPY file"! magicString≡(≠magicString)↑𝕩
|
"Not a valid NPY file"! magicString≡(≠magicString)↑𝕩
|
||||||
version←@-˜(0‿1+≠magicString)⊏𝕩
|
version←@-˜(0‿1+≠magicString)⊏𝕩
|
||||||
headerlen←+´1‿256×@-˜(2‿3+≠magicString)⊏𝕩
|
headerlen←+´1‿256×@-˜(2‿3+≠magicString)⊏𝕩
|
||||||
header←(¬∊⟜(' '∾lf))⊸/headerlen↑(4+≠magicString)↓𝕩
|
header←(¬∊⟜(' '∾lf))⊸/headerlen↑(4+≠magicString)↓𝕩
|
||||||
|
dtype←3↑(⊐⟜'<'↓⊢) (("descr"⍷header)⊐1)↓header
|
||||||
shapestr←(⊐⟜')'↑⊢) (1⊸+∘⊐⟜'('↓⊢) (("shape"⍷header)⊐1)↓header
|
shapestr←(⊐⟜')'↑⊢) (1⊸+∘⊐⟜'('↓⊢) (("shape"⍷header)⊐1)↓header
|
||||||
shape←•BQN¨','((⊢-˜+`׬)∘=⊔⊢)shapestr
|
shape←•BQN¨','((⊢-˜+`׬)∘=⊔⊢)shapestr
|
||||||
⟨version,shape,headerlen+2+2+≠magicString⟩
|
⟨version,dtype,shape,headerlen+2+2+≠magicString⟩
|
||||||
}
|
}
|
||||||
|
|
||||||
LoadNpy←{𝕊𝕩:
|
LoadNpy←{𝕊𝕩:
|
||||||
bytes←•file.Bytes 𝕩
|
bytes←•file.Bytes 𝕩
|
||||||
version‿shape‿datastart←ParseHeader bytes
|
version‿dtype‿shape‿datastart←ParseHeader bytes
|
||||||
shape⥊⟨8‿'c',64‿'f'⟩•bit._cast datastart↓bytes
|
shape⥊dtype ArrayFromBytes datastart↓bytes
|
||||||
}
|
}
|
||||||
|
|
12
tests.bqn
12
tests.bqn
|
@ -5,3 +5,15 @@ a←3‿4‿5•rand.Range 0
|
||||||
b←LoadNpy"testbqn.npy"
|
b←LoadNpy"testbqn.npy"
|
||||||
! a≡○≢b
|
! a≡○≢b
|
||||||
! a≡b
|
! a≡b
|
||||||
|
|
||||||
|
floatArr←2‿3•rand.Range 0
|
||||||
|
intArr←50-˜2‿3•rand.Range 100
|
||||||
|
uintArr←2‿3•rand.Range 100
|
||||||
|
|
||||||
|
"test_float.npy"SaveNpy floatArr
|
||||||
|
"test_int.npy"SaveNpy intArr
|
||||||
|
"test_uint.npy"SaveNpy uintArr
|
||||||
|
|
||||||
|
floatArr≡LoadNpy"test_float.npy"
|
||||||
|
intArr≡LoadNpy"test_int.npy"
|
||||||
|
uintArr≡LoadNpy"test_uint.npy"
|
||||||
|
|
Loading…
Add table
Add a link
Reference in a new issue