commit 6c1d57f9cd8a1cf4c1d99ee25f7b4d7a97f50c6a
Author: Dimitri Lozeve <dimitri@lozeve.com>
Date:   Fri Nov 15 20:58:49 2024 +0100

    Initial commit

diff --git a/json.bqn b/json.bqn
new file mode 100644
index 0000000..74ef0f3
--- /dev/null
+++ b/json.bqn
@@ -0,0 +1,227 @@
+# Part of bqn-libs: https://github.com/mlochbaum/bqn-libs
+# 0-BSD License
+
+# JSON: JavaScript Object Notation
+⟨
+  Parse   # JSON string to BQN
+  Export  # BQN value to JSON (also Parse⁼)
+⟩⇐
+
+# JSON numbers, strings, and lists correspond directly to BQN
+# Objects are represented as keys≍values
+# true, false, null are represented as <"true", <"false", <"null"
+
+# An empty list exports as "" if its fill is a space and [] otherwise
+
+⟨Consts, ExportConst⟩ ← {
+  val ← <¨ name ← "true"‿"false"‿"null"
+  Consts ⇐ {
+    i ← name ⊐ 𝕩
+    "Unknown constant" 𝕎 i = ≠name  # 𝕎 formats errors
+    i ⊏ val
+  }
+  cm ← "Enclosed value must be JSON constant"∾∾' '⊸∾¨"or "⊸∾⌾(¯1⊸⊑)name
+  ExportConst ⇐ {
+    i ← val⊸⊐⌾< 𝕩
+    cm ! i<≠val
+    i ⊑ name
+  }
+}
+⟨UnEscape, Escape⟩ ← {
+  in ← """\/bfrnt"
+  out ← (3↑in)∾@+8‿12‿13‿10‿9
+  diff ← (out-in) ∾ 0
+  Basic ← {
+    i ← in ⊐ 𝕩
+    "Unknown escape" ! ∧´𝕨≤i<≠in
+    𝕩 + 𝕨 × i ⊏ diff
+  }
+
+  hc ← "0Aa"
+  hb ← ⥊hc+0≍˘10‿6‿6   # Hex boundaries, start and after-end
+  ho ← 2/hc-0‿10‿10    # Corresponding offsets
+  Hex ← { u 𝕊 𝕩:
+    d ← 𝕩 /˜ m ← ≠` (4⥊0)⊸»⊸≠ »u
+    t ← hb ⍋ d
+    "String \u must be followed by 4 hex characters" ! ∧´1=2|t
+    # Now m can't run past the end or self-intersect,
+    # or it would have hit a closing quote or backslash
+    v ← 16⊸×⊸+˜˝⌽ ⍉∘‿4⥊ d-t⊏ho
+    w‿e ← Surrogate v
+    ⟨(w+@-'u')⊸+⌾(u⊸/)𝕩, e⌾(u⊸/)m⟩
+  }
+  sr ← 2⋆10            # Surrogate base/radix
+  sb ← sr×52+2+↕3      # Surrogate character boundaries
+  Surrogate ← {
+    c ← (≠sb)|sb⍋𝕩     # 0 for non-surrogate, 1 then 2 for surrogate
+    h ← 1=c            # First half
+    "Unmatched surrogate pair" ! (0∾h) ≡ (2=c)∾0
+    r ← 𝕩 - c⊏0∾sb     # Numeric value of surrogates
+    v ← r + »h×sr×(2⋆6)+r
+    ⟨v, h⟩
+  }
+
+  UnEscape ⇐ { e 𝕊 𝕩:
+    u ← e ∧ 𝕩='u'
+    (u<e)⊸Basic⌾⊑ u Hex⟜⊑⍟(∨´u) 𝕩‿0
+  }
+
+  Hex32 ← {                # Convert 𝕩<32 to two hex digits
+    u‿l ← 16(⌊∘÷˜⋈|)𝕩      # Upper and lower digits; ∧´u<2
+    '0'+u≍˘l-(10+-´"0A")×10≤l
+  }
+  Escape ⇐ {
+    e ← (𝕩∊2↑in) ∨ 𝕩<@+32  # Quote, backslash, and control characters
+    j ← /e ⋄ k ← ¬e        # Their indices; characters to keep
+    c ← j ⊏ 𝕩              # Characters to be escaped
+    m ← c - (out⊐c)⊏diff   # Escape the basic ones
+    g ← 2 ∾˜ m<@+32        # Group 𝕨 to separate...
+    n‿u ← g ⊔ m            # Characters requiring basic, hex escapes
+    i ← ∾⟨/k⟩∾2‿6/¨g⊔j     # Target indices
+    i ⍋⊸⊏ ∾⟨               # Use them to reorder:
+      k/𝕩                  # Kept characters
+      ⥊'\'≍˘n              # Basic escapes
+      ⥊"\u00"⊸∾˘ Hex32 u-@ # Hex escapes
+    ⟩
+  }
+}
+
+# Format error with message 𝕨 at locations 𝕩 in 𝕗
+_fmtErr ← { msg src _𝕣 pos:
+  lf ← @+10
+  s ← 0∾1+ b←/src=lf                # Line start and break positions
+  l ← (1↓s) ⍋ pos                   # Error line numbers
+  c ← pos - l⊏s                     # Error column numbers
+  d ← ⊑l                            # First line only
+  1↓∾lf⊸∾¨ ⟨
+    msg
+    (d⊑s)↓(d⊑b∾≠src)↑src            # Display first line
+    " ^" ⊏˜ /⁼ (d=l)/c              # And carets under errors
+  ⟩∾{
+    n ← ≠ a ← d⊸≠⊸/l                # 1-indexed numbers of other lines
+    0<n ? ⋈((-1=n)↓"Also lines")∾1↓∾⥊(<", ")≍˘•Repr¨1+a
+    ; ⟨⟩
+  }
+}
+
+# JSON to ⟨tokens, constants, numbers, strings⟩
+# Tokens are characters {}[],: and a constant, 0 number, " string
+# Values correspond to a, 0, and " in order
+Tokenize ← {
+  FE ← 𝕩_fmtErr ⋄ _err ← {(! 𝕗 FE /)⍟(∨´)}
+  # Strings
+  e ← »eo ← <`'\'=𝕩
+  s ← ≠`q ← e<'"'=𝕩
+  "Unclosed quote" ! ¬¯1⊑s
+  "Backslash outside string"_err s<eo
+  "Un-escaped control character"_err s>𝕩≥@+32
+  xe‿dr ← e UnEscape 𝕩              # Escaped 𝕩; characters to drop
+  sg ← ((1-˜(s>q∨eo∨dr)×+`)∾+´)s∧q  # Start at s∧q; exclude q, eo, dr
+  str ← sg ⊔ xe                     # Strings
+
+  # Numbers and constants
+  b ← s ∨ 𝕩∊@+9‿10‿13‿32            # Whitespace (blank)
+  l ← ¬ b ∨ 𝕩∊"""{}[],:"            # Word characters
+  w ← »⊸< l                         # Word starts
+  neg ← '-' = 𝕩
+  dig ← ('0'≤𝕩) ∧ 𝕩≤'9'
+  m ← l ∧ (+`w)⊏0∾w/dig∨neg         # Numbers
+  n ← m∧w ⋄ k ← m<w                 # Number and constant starts
+  "Leading zero"_err (n≠»⊸∨n∧neg)∧('0'=𝕩)∧«dig
+  "Digits required on both sides of decimal"_err (m∧'.'=𝕩)>(»∧«)dig
+  CE ← {(! 𝕨 FE /⟜(/k))⍟(∨´) 𝕩}
+  cns ← ce Consts (1-˜(m<l)×+`k)⊔𝕩  # Constants
+  num ← •ParseFloat¨ (1-˜m×+`n)⊔𝕩   # Numbers
+
+  # Tokenize
+  f ← ¬b∨w<l                        # First characters of tokens
+  tok ← '0'¨⌾((f/n)⊸/) 'a'¨⌾((f/k)⊸/) f/𝕩
+  ⟨tok, f, cns, num, str⟩
+}
+
+Parse ← {
+  𝕊⁼𝕩: Export 𝕩 ;
+  "Empty input" ! 0<≠𝕩
+  t‿tf‿cns‿num‿strk ← Tokenize 𝕩
+  "Empty input" ! 0<≠t
+
+  # Validate
+  # _tr records persistent transformations
+  # Temporary transformations are passed to _err_ as 𝔽
+  trs ← ⟨tf,/⟩ ⋄ _tr ← {trs∾↩⟨𝕨⊸𝔽⟩⋄𝕨𝔽𝕩}
+  FE ← 𝕩_fmtErr ⋄ _err_ ← {𝕗{! 𝕗 FE 𝕩/𝔾{𝕎𝕩}´⌽trs}𝔾⍟(∨´)}
+  q ← '"'=t
+  c ← ':'=t
+  "Object keys must be strings"_err_⊢ (»q)<c
+  g ← ⍋+`(co←t∊"[{")-cc←t∊"]}"      # Bracket depth ordering indices
+  u ← g ⊏_tr t
+  r ← +` s ← u∊"[{"
+  o ← s/'{'=u                       # Container is object
+  uc← (u∊"]}")/'}'=u
+  "Unmatched brackets" ! o=○≠uc
+  "Mismatched brackets"_err_{s/𝕩} o ≠ uc
+  v ← («⊸∨¬s) /_tr u                # Remove empty lists/objects
+  vs← v ∊ "[{,:"                    # Must alternate 0101...10
+  "Improper , or : usage"_err_(⊢∾⊢´) 1(∾=∾˜) vs
+  v /_tr˜↩ vs
+  # v should be composed of lists [,,, and objects {:,:,:
+  # Convert {: to { and ,: to : to get [,,, and {::
+  "Multiple keys for one value"_err_⊢ »⊸∧v=':'
+  v /_tr˜↩ ¬((»v='{')⊸∧ ∨ (v=',')∧«)v=':'
+  # Or, every , follows list-like [, and every : follows object-like {:
+  d ← v∊",:"
+  "Top-level , or :"_err_{1↑𝕩} 1↑d
+  "Bad object structure"_err_⊢ d > »⊸=v∊"{:"
+
+  # Keys
+  l ← (⍋g) ⊏ r                      # Container index
+  j ← +`⊸× o                        # Object index (start at 1; 0 if list)
+  keys ← ((q/(«c)×l⊏0∾j)∾1+´o) ⊔ strk
+  str ← ⊑keys
+
+  # Purely numeric lists
+  l ⊏↩ ⍋⍋nm←0⌾⊑1(∾/∾˜)s(∨/⊣)u∊"]}""a"
+  nn ← +´¬nm
+  nl ← num⊔˜(1+´nm)∾˜0⊸<⊸×(1-nn)+(t='0')/l
+  jj‿jn ← 2↑(1↓nm)⊔j
+  Ob ← ⊑⟜keys⊸≍⍟(0<⊣)
+  n ← jn Ob¨ 1↓nl
+
+  # Build collections
+  nv← n -˜○≠ vals←∾⟨cns,⊑nl,str,⌽n⟩ # Initial set of values
+  f ← (l<nn)∧¬(«⊸∨c)∨co∨','=t       # Filter for just values a0"]}
+  vi← ⍋⍋(f/'0'=t)+(2×f/q)+3×f/cc    # Value indices
+  i ← vi ⊏ (↕nv) ∾ nv+≠⊸-cc/»l      # Adjust for collection ordering
+  jj {vals∾↩⟨𝕨 Ob 𝕩⊏vals⟩⋄@}¨○⌽ ((≠jj)∾˜1-˜f/l)⊔i
+  ¯1⊑vals
+}
+
+ExportNumber ← {
+  "NaN can't be represented in JSON" ! =˜𝕩
+  "Infinities can't be represented in JSON" ! ∞≠|𝕩
+  r ← •Repr 𝕩
+  r + (-´"-¯")×r='¯'
+}
+IsString ← {∧´2=•Type¨ 1⊸↑⍟(0=≠) 𝕩}
+ExportList ← {
+  IsString 𝕩 ? ExportString 𝕩 ;
+  ∾ ⟨"["⟩∾⟨"]"⟩∾˜ 1↓⥊(<",")≍˘Export¨ 𝕩
+}
+ExportString ← '"' (∾∾⊣) Escape
+ExportObject ← {
+  "Object must consist of keys≍values" ! 2=≠𝕩
+  [k,v] ← 𝕩
+  "Object keys must be strings" ! ∧´ IsString¨ k
+  kv ← ⥊ (ExportString¨ k) ≍˘ Export¨ v
+  ∾ ⟨"{"⟩∾⟨"}"⟩∾˜ ¯1↓⥊ kv ≍˘ (≠kv) ⥊ ⟨":",","⟩
+}
+Export ← (2⌊•Type)◶⟨ # Check type
+  (3⌊=)◶⟨            # Type 0, array: Check rank
+    ExportConst      #   0 constant (enclosed string)
+    ExportList       #   1 List or string
+    ExportObject     #   2 Object
+    !∘"Rank >2 cannot be exported to JSON"
+  ⟩
+  ExportNumber       # Type 1: number
+  !∘"Only numbers and arrays can be exported to JSON"
+⟩
diff --git a/safetensors.bqn b/safetensors.bqn
new file mode 100644
index 0000000..e453789
--- /dev/null
+++ b/safetensors.bqn
@@ -0,0 +1,103 @@
+⟨ExtractMetadata,GetArrayNames,GetArray,SerializeArrays⟩⇐
+
+⟨Parse,Export⟩←•Import"json.bqn"
+
+ExtractHeader←{𝕊bytes:
+  n←2⊸×⊸+˜´⟨8‿'c',1‿'u'⟩•bit._cast 8↑bytes
+  ⟨Parse n↑8↓bytes,n+8⟩
+}
+
+JsonGet←{(⊑(⊏𝕨)⊐<𝕩)⊑1⊏𝕨}
+
+ExtractMetadata←{𝕊bytes:
+  header‿·←ExtractHeader bytes
+  header JsonGet⎊⟨⟩ "__metadata__"
+}
+
+GetArrayNames←{𝕊bytes:
+  header‿·←ExtractHeader bytes
+  "__metadata__"⊸≢¨⊸/⊏header
+}
+
+# Valid for sizes 8, 16, and 32 (passed as 𝕨)
+ParseUint←{2⊸×⊸+˜´˘⟨8‿'c',1‿'u'⟩•bit._cast˘ ∘‿(𝕨÷8)⥊𝕩}
+ParseInt←{(-2⋆𝕨-1)+(2⋆𝕨)|(2⋆𝕨-1)+𝕨ParseUint𝕩}
+
+ParseUint64←{(2⋆32)⊸×⊸+˜´˘ ∘‿2⥊32 ParseUInt 𝕩}
+ParseInt64←{(2⋆32)⊸×⊸+˜´˘ ∘‿2⥊32 ParseInt 𝕩}
+
+# Parse a floating point number
+# e is the size of the exponent part
+ParseFloat←{e𝕊bytes:
+  n←⌽⟨8‿'c',1‿'u'⟩•bit._cast bytes
+  s←(≠n)-e+1
+  sign←1+2×-⊑n
+  exponent←2⊸×⊸+˜´⌽e↑1↓n
+  significand←2⊸×⊸+˜´⌽1∾e↓1↓n
+  sign×(2⋆exponent-((2⋆e-1)-1))×significand÷2⋆s
+}
+
+dtypes←⟨
+  "BOOL", # Boolean type
+  "U8", # Unsigned byte
+  "I8", # Signed byte
+  "F8_E5M2", # FP8 <https://arxiv.org/pdf/2209.05433.pdf>
+  "F8_E4M3", # FP8 <https://arxiv.org/pdf/2209.05433.pdf>
+  "I16", # Signed integer (16-bit)
+  "U16", # Unsigned integer (16-bit)
+  "F16", # Half-precision floating point
+  "BF16", # Brain floating point
+  "I32", # Signed integer (32-bit)
+  "U32", # Unsigned integer (32-bit)
+  "F32", # Floating point (32-bit)
+  "F64", # Floating point (64-bit)
+  "I64", # Signed integer (64-bit)
+  "U64", # Unsigned integer (64-bit)
+⟩
+typeConversions←⟨
+  ⟨8‿'c', 1‿'u'⟩•bit._cast, # BOOL
+  8⊸ParseUint,              # U8
+  ⟨8‿'c', 8‿'i'⟩•bit._cast, # I8
+  5⊸ParseFloat˘∘‿1⊸⥊,       # F8_E5M2
+  4⊸ParseFloat˘∘‿1⊸⥊,       # F8_E4M4
+  ⟨8‿'c',16‿'i'⟩•bit._cast, # I16
+  16⊸ParseUint,             # U16
+  5⊸ParseFloat˘∘‿2⊸⥊,       # F16
+  8⊸ParseFloat˘∘‿2⊸⥊,       # BF16
+  ⟨8‿'c',32‿'i'⟩•bit._cast, # I32
+  32⊸ParseUint,             # U32
+  8⊸ParseFloat˘∘‿4⊸⥊,       # F32
+  ⟨8‿'c',64‿'f'⟩•bit._cast, # F64
+  ParseInt64,               # I64
+  ParseUint64,              # U64
+⟩
+
+GetArray←{bytes𝕊name:
+  header‿n←ExtractHeader bytes
+  byteBuf←n↓bytes
+  info←header JsonGet name
+  s‿e←info JsonGet "data_offsets"
+  shape←info JsonGet "shape"
+  dtypeIdx←⊑dtypes⊐<info JsonGet "dtype"
+  conv←dtypeIdx⊑typeConversions
+  shape⥊Conv s↓e↑byteBuf
+}
+
+SerializeArray←{
+  dtype←(∧´⌊⊸=⥊𝕩)⊑"F64"‿"I32"
+  shape←≢𝕩
+  data←(∧´⌊⊸=)◶⟨⟨64‿'f',8‿'c'⟩•bit._cast,⟨32‿'i',8‿'c'⟩•bit._cast⟩⥊𝕩
+  dtype‿shape‿data
+}
+
+SerializeArrays←{names𝕊arrs:
+  dtypes‿shapes‿datas←<˘⍉>SerializeArray¨arrs
+  dataOffsets←<˘2↕0∾+`≠¨datas
+  blocks←{𝕊name‿dtype‿shape‿dataOffset:
+    ["dtype"‿"shape"‿"data_offsets",dtype‿shape‿dataOffset]
+  }¨<˘⍉>names‿dtypes‿shapes‿dataOffsets
+  header←[names,blocks]
+  n←≠headerJson←Export header
+  nEncoded←⟨32‿'i',8‿'c'⟩•bit._cast⟨n,0⟩
+  nEncoded∾headerJson∾∾datas
+}