Initial commit
This commit is contained in:
commit
6c1d57f9cd
2 changed files with 330 additions and 0 deletions
227
json.bqn
Normal file
227
json.bqn
Normal file
|
|
@ -0,0 +1,227 @@
|
|||
# Part of bqn-libs: https://github.com/mlochbaum/bqn-libs
|
||||
# 0-BSD License
|
||||
|
||||
# JSON: JavaScript Object Notation
|
||||
⟨
|
||||
Parse # JSON string to BQN
|
||||
Export # BQN value to JSON (also Parse⁼)
|
||||
⟩⇐
|
||||
|
||||
# JSON numbers, strings, and lists correspond directly to BQN
|
||||
# Objects are represented as keys≍values
|
||||
# true, false, null are represented as <"true", <"false", <"null"
|
||||
|
||||
# An empty list exports as "" if its fill is a space and [] otherwise
|
||||
|
||||
⟨Consts, ExportConst⟩ ← {
|
||||
val ← <¨ name ← "true"‿"false"‿"null"
|
||||
Consts ⇐ {
|
||||
i ← name ⊐ 𝕩
|
||||
"Unknown constant" 𝕎 i = ≠name # 𝕎 formats errors
|
||||
i ⊏ val
|
||||
}
|
||||
cm ← "Enclosed value must be JSON constant"∾∾' '⊸∾¨"or "⊸∾⌾(¯1⊸⊑)name
|
||||
ExportConst ⇐ {
|
||||
i ← val⊸⊐⌾< 𝕩
|
||||
cm ! i<≠val
|
||||
i ⊑ name
|
||||
}
|
||||
}
|
||||
⟨UnEscape, Escape⟩ ← {
|
||||
in ← """\/bfrnt"
|
||||
out ← (3↑in)∾@+8‿12‿13‿10‿9
|
||||
diff ← (out-in) ∾ 0
|
||||
Basic ← {
|
||||
i ← in ⊐ 𝕩
|
||||
"Unknown escape" ! ∧´𝕨≤i<≠in
|
||||
𝕩 + 𝕨 × i ⊏ diff
|
||||
}
|
||||
|
||||
hc ← "0Aa"
|
||||
hb ← ⥊hc+0≍˘10‿6‿6 # Hex boundaries, start and after-end
|
||||
ho ← 2/hc-0‿10‿10 # Corresponding offsets
|
||||
Hex ← { u 𝕊 𝕩:
|
||||
d ← 𝕩 /˜ m ← ≠` (4⥊0)⊸»⊸≠ »u
|
||||
t ← hb ⍋ d
|
||||
"String \u must be followed by 4 hex characters" ! ∧´1=2|t
|
||||
# Now m can't run past the end or self-intersect,
|
||||
# or it would have hit a closing quote or backslash
|
||||
v ← 16⊸×⊸+˜˝⌽ ⍉∘‿4⥊ d-t⊏ho
|
||||
w‿e ← Surrogate v
|
||||
⟨(w+@-'u')⊸+⌾(u⊸/)𝕩, e⌾(u⊸/)m⟩
|
||||
}
|
||||
sr ← 2⋆10 # Surrogate base/radix
|
||||
sb ← sr×52+2+↕3 # Surrogate character boundaries
|
||||
Surrogate ← {
|
||||
c ← (≠sb)|sb⍋𝕩 # 0 for non-surrogate, 1 then 2 for surrogate
|
||||
h ← 1=c # First half
|
||||
"Unmatched surrogate pair" ! (0∾h) ≡ (2=c)∾0
|
||||
r ← 𝕩 - c⊏0∾sb # Numeric value of surrogates
|
||||
v ← r + »h×sr×(2⋆6)+r
|
||||
⟨v, h⟩
|
||||
}
|
||||
|
||||
UnEscape ⇐ { e 𝕊 𝕩:
|
||||
u ← e ∧ 𝕩='u'
|
||||
(u<e)⊸Basic⌾⊑ u Hex⟜⊑⍟(∨´u) 𝕩‿0
|
||||
}
|
||||
|
||||
Hex32 ← { # Convert 𝕩<32 to two hex digits
|
||||
u‿l ← 16(⌊∘÷˜⋈|)𝕩 # Upper and lower digits; ∧´u<2
|
||||
'0'+u≍˘l-(10+-´"0A")×10≤l
|
||||
}
|
||||
Escape ⇐ {
|
||||
e ← (𝕩∊2↑in) ∨ 𝕩<@+32 # Quote, backslash, and control characters
|
||||
j ← /e ⋄ k ← ¬e # Their indices; characters to keep
|
||||
c ← j ⊏ 𝕩 # Characters to be escaped
|
||||
m ← c - (out⊐c)⊏diff # Escape the basic ones
|
||||
g ← 2 ∾˜ m<@+32 # Group 𝕨 to separate...
|
||||
n‿u ← g ⊔ m # Characters requiring basic, hex escapes
|
||||
i ← ∾⟨/k⟩∾2‿6/¨g⊔j # Target indices
|
||||
i ⍋⊸⊏ ∾⟨ # Use them to reorder:
|
||||
k/𝕩 # Kept characters
|
||||
⥊'\'≍˘n # Basic escapes
|
||||
⥊"\u00"⊸∾˘ Hex32 u-@ # Hex escapes
|
||||
⟩
|
||||
}
|
||||
}
|
||||
|
||||
# Format error with message 𝕨 at locations 𝕩 in 𝕗
|
||||
_fmtErr ← { msg src _𝕣 pos:
|
||||
lf ← @+10
|
||||
s ← 0∾1+ b←/src=lf # Line start and break positions
|
||||
l ← (1↓s) ⍋ pos # Error line numbers
|
||||
c ← pos - l⊏s # Error column numbers
|
||||
d ← ⊑l # First line only
|
||||
1↓∾lf⊸∾¨ ⟨
|
||||
msg
|
||||
(d⊑s)↓(d⊑b∾≠src)↑src # Display first line
|
||||
" ^" ⊏˜ /⁼ (d=l)/c # And carets under errors
|
||||
⟩∾{
|
||||
n ← ≠ a ← d⊸≠⊸/l # 1-indexed numbers of other lines
|
||||
0<n ? ⋈((-1=n)↓"Also lines")∾1↓∾⥊(<", ")≍˘•Repr¨1+a
|
||||
; ⟨⟩
|
||||
}
|
||||
}
|
||||
|
||||
# JSON to ⟨tokens, constants, numbers, strings⟩
|
||||
# Tokens are characters {}[],: and a constant, 0 number, " string
|
||||
# Values correspond to a, 0, and " in order
|
||||
Tokenize ← {
|
||||
FE ← 𝕩_fmtErr ⋄ _err ← {(! 𝕗 FE /)⍟(∨´)}
|
||||
# Strings
|
||||
e ← »eo ← <`'\'=𝕩
|
||||
s ← ≠`q ← e<'"'=𝕩
|
||||
"Unclosed quote" ! ¬¯1⊑s
|
||||
"Backslash outside string"_err s<eo
|
||||
"Un-escaped control character"_err s>𝕩≥@+32
|
||||
xe‿dr ← e UnEscape 𝕩 # Escaped 𝕩; characters to drop
|
||||
sg ← ((1-˜(s>q∨eo∨dr)×+`)∾+´)s∧q # Start at s∧q; exclude q, eo, dr
|
||||
str ← sg ⊔ xe # Strings
|
||||
|
||||
# Numbers and constants
|
||||
b ← s ∨ 𝕩∊@+9‿10‿13‿32 # Whitespace (blank)
|
||||
l ← ¬ b ∨ 𝕩∊"""{}[],:" # Word characters
|
||||
w ← »⊸< l # Word starts
|
||||
neg ← '-' = 𝕩
|
||||
dig ← ('0'≤𝕩) ∧ 𝕩≤'9'
|
||||
m ← l ∧ (+`w)⊏0∾w/dig∨neg # Numbers
|
||||
n ← m∧w ⋄ k ← m<w # Number and constant starts
|
||||
"Leading zero"_err (n≠»⊸∨n∧neg)∧('0'=𝕩)∧«dig
|
||||
"Digits required on both sides of decimal"_err (m∧'.'=𝕩)>(»∧«)dig
|
||||
CE ← {(! 𝕨 FE /⟜(/k))⍟(∨´) 𝕩}
|
||||
cns ← ce Consts (1-˜(m<l)×+`k)⊔𝕩 # Constants
|
||||
num ← •ParseFloat¨ (1-˜m×+`n)⊔𝕩 # Numbers
|
||||
|
||||
# Tokenize
|
||||
f ← ¬b∨w<l # First characters of tokens
|
||||
tok ← '0'¨⌾((f/n)⊸/) 'a'¨⌾((f/k)⊸/) f/𝕩
|
||||
⟨tok, f, cns, num, str⟩
|
||||
}
|
||||
|
||||
Parse ← {
|
||||
𝕊⁼𝕩: Export 𝕩 ;
|
||||
"Empty input" ! 0<≠𝕩
|
||||
t‿tf‿cns‿num‿strk ← Tokenize 𝕩
|
||||
"Empty input" ! 0<≠t
|
||||
|
||||
# Validate
|
||||
# _tr records persistent transformations
|
||||
# Temporary transformations are passed to _err_ as 𝔽
|
||||
trs ← ⟨tf,/⟩ ⋄ _tr ← {trs∾↩⟨𝕨⊸𝔽⟩⋄𝕨𝔽𝕩}
|
||||
FE ← 𝕩_fmtErr ⋄ _err_ ← {𝕗{! 𝕗 FE 𝕩/𝔾{𝕎𝕩}´⌽trs}𝔾⍟(∨´)}
|
||||
q ← '"'=t
|
||||
c ← ':'=t
|
||||
"Object keys must be strings"_err_⊢ (»q)<c
|
||||
g ← ⍋+`(co←t∊"[{")-cc←t∊"]}" # Bracket depth ordering indices
|
||||
u ← g ⊏_tr t
|
||||
r ← +` s ← u∊"[{"
|
||||
o ← s/'{'=u # Container is object
|
||||
uc← (u∊"]}")/'}'=u
|
||||
"Unmatched brackets" ! o=○≠uc
|
||||
"Mismatched brackets"_err_{s/𝕩} o ≠ uc
|
||||
v ← («⊸∨¬s) /_tr u # Remove empty lists/objects
|
||||
vs← v ∊ "[{,:" # Must alternate 0101...10
|
||||
"Improper , or : usage"_err_(⊢∾⊢´) 1(∾=∾˜) vs
|
||||
v /_tr˜↩ vs
|
||||
# v should be composed of lists [,,, and objects {:,:,:
|
||||
# Convert {: to { and ,: to : to get [,,, and {::
|
||||
"Multiple keys for one value"_err_⊢ »⊸∧v=':'
|
||||
v /_tr˜↩ ¬((»v='{')⊸∧ ∨ (v=',')∧«)v=':'
|
||||
# Or, every , follows list-like [, and every : follows object-like {:
|
||||
d ← v∊",:"
|
||||
"Top-level , or :"_err_{1↑𝕩} 1↑d
|
||||
"Bad object structure"_err_⊢ d > »⊸=v∊"{:"
|
||||
|
||||
# Keys
|
||||
l ← (⍋g) ⊏ r # Container index
|
||||
j ← +`⊸× o # Object index (start at 1; 0 if list)
|
||||
keys ← ((q/(«c)×l⊏0∾j)∾1+´o) ⊔ strk
|
||||
str ← ⊑keys
|
||||
|
||||
# Purely numeric lists
|
||||
l ⊏↩ ⍋⍋nm←0⌾⊑1(∾/∾˜)s(∨/⊣)u∊"]}""a"
|
||||
nn ← +´¬nm
|
||||
nl ← num⊔˜(1+´nm)∾˜0⊸<⊸×(1-nn)+(t='0')/l
|
||||
jj‿jn ← 2↑(1↓nm)⊔j
|
||||
Ob ← ⊑⟜keys⊸≍⍟(0<⊣)
|
||||
n ← jn Ob¨ 1↓nl
|
||||
|
||||
# Build collections
|
||||
nv← n -˜○≠ vals←∾⟨cns,⊑nl,str,⌽n⟩ # Initial set of values
|
||||
f ← (l<nn)∧¬(«⊸∨c)∨co∨','=t # Filter for just values a0"]}
|
||||
vi← ⍋⍋(f/'0'=t)+(2×f/q)+3×f/cc # Value indices
|
||||
i ← vi ⊏ (↕nv) ∾ nv+≠⊸-cc/»l # Adjust for collection ordering
|
||||
jj {vals∾↩⟨𝕨 Ob 𝕩⊏vals⟩⋄@}¨○⌽ ((≠jj)∾˜1-˜f/l)⊔i
|
||||
¯1⊑vals
|
||||
}
|
||||
|
||||
ExportNumber ← {
|
||||
"NaN can't be represented in JSON" ! =˜𝕩
|
||||
"Infinities can't be represented in JSON" ! ∞≠|𝕩
|
||||
r ← •Repr 𝕩
|
||||
r + (-´"-¯")×r='¯'
|
||||
}
|
||||
IsString ← {∧´2=•Type¨ 1⊸↑⍟(0=≠) 𝕩}
|
||||
ExportList ← {
|
||||
IsString 𝕩 ? ExportString 𝕩 ;
|
||||
∾ ⟨"["⟩∾⟨"]"⟩∾˜ 1↓⥊(<",")≍˘Export¨ 𝕩
|
||||
}
|
||||
ExportString ← '"' (∾∾⊣) Escape
|
||||
ExportObject ← {
|
||||
"Object must consist of keys≍values" ! 2=≠𝕩
|
||||
[k,v] ← 𝕩
|
||||
"Object keys must be strings" ! ∧´ IsString¨ k
|
||||
kv ← ⥊ (ExportString¨ k) ≍˘ Export¨ v
|
||||
∾ ⟨"{"⟩∾⟨"}"⟩∾˜ ¯1↓⥊ kv ≍˘ (≠kv) ⥊ ⟨":",","⟩
|
||||
}
|
||||
Export ← (2⌊•Type)◶⟨ # Check type
|
||||
(3⌊=)◶⟨ # Type 0, array: Check rank
|
||||
ExportConst # 0 constant (enclosed string)
|
||||
ExportList # 1 List or string
|
||||
ExportObject # 2 Object
|
||||
!∘"Rank >2 cannot be exported to JSON"
|
||||
⟩
|
||||
ExportNumber # Type 1: number
|
||||
!∘"Only numbers and arrays can be exported to JSON"
|
||||
⟩
|
||||
103
safetensors.bqn
Normal file
103
safetensors.bqn
Normal file
|
|
@ -0,0 +1,103 @@
|
|||
⟨ExtractMetadata,GetArrayNames,GetArray,SerializeArrays⟩⇐
|
||||
|
||||
⟨Parse,Export⟩←•Import"json.bqn"
|
||||
|
||||
ExtractHeader←{𝕊bytes:
|
||||
n←2⊸×⊸+˜´⟨8‿'c',1‿'u'⟩•bit._cast 8↑bytes
|
||||
⟨Parse n↑8↓bytes,n+8⟩
|
||||
}
|
||||
|
||||
JsonGet←{(⊑(⊏𝕨)⊐<𝕩)⊑1⊏𝕨}
|
||||
|
||||
ExtractMetadata←{𝕊bytes:
|
||||
header‿·←ExtractHeader bytes
|
||||
header JsonGet⎊⟨⟩ "__metadata__"
|
||||
}
|
||||
|
||||
GetArrayNames←{𝕊bytes:
|
||||
header‿·←ExtractHeader bytes
|
||||
"__metadata__"⊸≢¨⊸/⊏header
|
||||
}
|
||||
|
||||
# Valid for sizes 8, 16, and 32 (passed as 𝕨)
|
||||
ParseUint←{2⊸×⊸+˜´˘⟨8‿'c',1‿'u'⟩•bit._cast˘ ∘‿(𝕨÷8)⥊𝕩}
|
||||
ParseInt←{(-2⋆𝕨-1)+(2⋆𝕨)|(2⋆𝕨-1)+𝕨ParseUint𝕩}
|
||||
|
||||
ParseUint64←{(2⋆32)⊸×⊸+˜´˘ ∘‿2⥊32 ParseUInt 𝕩}
|
||||
ParseInt64←{(2⋆32)⊸×⊸+˜´˘ ∘‿2⥊32 ParseInt 𝕩}
|
||||
|
||||
# Parse a floating point number
|
||||
# e is the size of the exponent part
|
||||
ParseFloat←{e𝕊bytes:
|
||||
n←⌽⟨8‿'c',1‿'u'⟩•bit._cast bytes
|
||||
s←(≠n)-e+1
|
||||
sign←1+2×-⊑n
|
||||
exponent←2⊸×⊸+˜´⌽e↑1↓n
|
||||
significand←2⊸×⊸+˜´⌽1∾e↓1↓n
|
||||
sign×(2⋆exponent-((2⋆e-1)-1))×significand÷2⋆s
|
||||
}
|
||||
|
||||
dtypes←⟨
|
||||
"BOOL", # Boolean type
|
||||
"U8", # Unsigned byte
|
||||
"I8", # Signed byte
|
||||
"F8_E5M2", # FP8 <https://arxiv.org/pdf/2209.05433.pdf>
|
||||
"F8_E4M3", # FP8 <https://arxiv.org/pdf/2209.05433.pdf>
|
||||
"I16", # Signed integer (16-bit)
|
||||
"U16", # Unsigned integer (16-bit)
|
||||
"F16", # Half-precision floating point
|
||||
"BF16", # Brain floating point
|
||||
"I32", # Signed integer (32-bit)
|
||||
"U32", # Unsigned integer (32-bit)
|
||||
"F32", # Floating point (32-bit)
|
||||
"F64", # Floating point (64-bit)
|
||||
"I64", # Signed integer (64-bit)
|
||||
"U64", # Unsigned integer (64-bit)
|
||||
⟩
|
||||
typeConversions←⟨
|
||||
⟨8‿'c', 1‿'u'⟩•bit._cast, # BOOL
|
||||
8⊸ParseUint, # U8
|
||||
⟨8‿'c', 8‿'i'⟩•bit._cast, # I8
|
||||
5⊸ParseFloat˘∘‿1⊸⥊, # F8_E5M2
|
||||
4⊸ParseFloat˘∘‿1⊸⥊, # F8_E4M4
|
||||
⟨8‿'c',16‿'i'⟩•bit._cast, # I16
|
||||
16⊸ParseUint, # U16
|
||||
5⊸ParseFloat˘∘‿2⊸⥊, # F16
|
||||
8⊸ParseFloat˘∘‿2⊸⥊, # BF16
|
||||
⟨8‿'c',32‿'i'⟩•bit._cast, # I32
|
||||
32⊸ParseUint, # U32
|
||||
8⊸ParseFloat˘∘‿4⊸⥊, # F32
|
||||
⟨8‿'c',64‿'f'⟩•bit._cast, # F64
|
||||
ParseInt64, # I64
|
||||
ParseUint64, # U64
|
||||
⟩
|
||||
|
||||
GetArray←{bytes𝕊name:
|
||||
header‿n←ExtractHeader bytes
|
||||
byteBuf←n↓bytes
|
||||
info←header JsonGet name
|
||||
s‿e←info JsonGet "data_offsets"
|
||||
shape←info JsonGet "shape"
|
||||
dtypeIdx←⊑dtypes⊐<info JsonGet "dtype"
|
||||
conv←dtypeIdx⊑typeConversions
|
||||
shape⥊Conv s↓e↑byteBuf
|
||||
}
|
||||
|
||||
SerializeArray←{
|
||||
dtype←(∧´⌊⊸=⥊𝕩)⊑"F64"‿"I32"
|
||||
shape←≢𝕩
|
||||
data←(∧´⌊⊸=)◶⟨⟨64‿'f',8‿'c'⟩•bit._cast,⟨32‿'i',8‿'c'⟩•bit._cast⟩⥊𝕩
|
||||
dtype‿shape‿data
|
||||
}
|
||||
|
||||
SerializeArrays←{names𝕊arrs:
|
||||
dtypes‿shapes‿datas←<˘⍉>SerializeArray¨arrs
|
||||
dataOffsets←<˘2↕0∾+`≠¨datas
|
||||
blocks←{𝕊name‿dtype‿shape‿dataOffset:
|
||||
["dtype"‿"shape"‿"data_offsets",dtype‿shape‿dataOffset]
|
||||
}¨<˘⍉>names‿dtypes‿shapes‿dataOffsets
|
||||
header←[names,blocks]
|
||||
n←≠headerJson←Export header
|
||||
nEncoded←⟨32‿'i',8‿'c'⟩•bit._cast⟨n,0⟩
|
||||
nEncoded∾headerJson∾∾datas
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue