Lexical data inventory

Generated by tools/syntax-spec write from etc/syntax/lexical.x. See the lexical specification for explanations and the data model for notation and consumption.

input

(input (encoding bytes) (terminator 0) (utf8-validation none)
 (token-fields type text pos len line col)
 (position (pos 0 bytes) (len bytes) (line 1) (col 1 bytes))
 (advance (on 10 (increment line) (set col 1)) (otherwise (increment col)))
 (text copy) (trivia space comment) (retain preproc)
 (roots x2c lisp) (termination before-mode-dispatch))

classes

(classes (class letter (choice (range 65 90) (range 97 122)))
 (class digit (range 48 57))
 (class hex-digit (choice (ref digit) (range 65 70) (range 97 102)))
 (class octal-digit (range 48 55))
 (class binary-digit (choice (byte 48) (byte 49)))
 (class identifier-start (choice (ref letter) (byte 95)))
 (class identifier-rest (choice (ref identifier-start) (ref digit)))
 (class white-byte
  (choice (byte 32) (byte 9) (byte 13) (byte 11)
   (byte 12) (byte 10))) (class simple-escape (one-of-bytes "abfnrtv\'\"?\\"))
 (class atom-boundary (choice (one-of-bytes "()\'`,\"$@{[") (ref white-byte)))
 (class collection-boundary (one-of-bytes ",:]}"))
 (class set-boundary (choice (ref white-byte) (byte 36) (byte 64))))

rules

(rules
 (rule identifier (seq (ref identifier-start) (repeat 0 (ref identifier-rest))))
 (rule whitespace (repeat 1 (ref white-byte)) (emit space))
 (rule line-comment
  (seq (bytes "//") (until (choice (byte 10) (byte 0)) exclude))
  (emit comment))
 (rule block-comment (seq (bytes "/*") (until (bytes "*/") include))
  (nesting none) (on-nul incomplete) (emit comment))
 (rule preprocessor
  (seq (byte 35)
   (repeat 0 (ordered (ref continuation) (except (byte 10) (byte 0)))))
  (reject trailing-backslash) (emit preproc))
 (rule continuation (seq (byte 92) (choice (byte 10) (seq (byte 13) (byte 10)))))
 (rule unsigned-suffix (choice (bytes "u") (bytes "U")))
 (rule long-suffix (choice (bytes "l") (bytes "L")))
 (rule integer-suffix
  (optional
   (choice
    (seq (ref unsigned-suffix)
     (optional (seq (ref long-suffix) (optional (ref long-suffix)))))
    (seq (ref long-suffix) (optional (ref long-suffix))
     (optional (ref unsigned-suffix))))))
 (rule float-suffix (optional (one-of-bytes "fFlL")))
 (rule exponent-digits
  (seq (optional (one-of-bytes "+-")) (repeat 1 (ref digit))))
 (rule decimal-exponent (seq (one-of-bytes "eE") (ref exponent-digits)))
 (rule decimal-integer (seq (repeat 1 (ref digit)) (ref integer-suffix)))
 (rule binary-integer
  (seq (byte 48) (one-of-bytes "bB") (repeat 1 (ref binary-digit))
   (ref integer-suffix)))
 (rule octal-integer
  (seq (byte 48) (one-of-bytes "oO") (repeat 1 (ref octal-digit))
   (ref integer-suffix)))
 (rule legacy-octal
  (seq (byte 48) (range 49 55) (repeat 0 (ref octal-digit))
   (ref integer-suffix)))
 (rule hex-integer
  (seq (byte 48) (one-of-bytes "xX") (repeat 1 (ref hex-digit))
   (ref integer-suffix)))
 (rule decimal-float
  (seq (choice
        (seq (repeat 1 (ref digit)) (byte 46) (repeat 0 (ref digit))
         (optional (ref decimal-exponent)))
        (seq (byte 46) (repeat 1 (ref digit)) (optional (ref decimal-exponent)))
        (seq (repeat 1 (ref digit)) (ref decimal-exponent)))
   (ref float-suffix)))
 (rule hex-float
  (seq (byte 48) (one-of-bytes "xX")
   (choice
    (seq (repeat 1 (ref hex-digit))
     (optional (seq (byte 46) (repeat 0 (ref hex-digit)))))
    (seq (byte 46) (repeat 1 (ref hex-digit))))
   (one-of-bytes "pP") (ref exponent-digits)
   (ref float-suffix)))
 (rule number
  (seq (optional (one-of-bytes "+-"))
   (select-radix
    (ordered
     (when-prefix (seq (byte 48) (one-of-bytes "xX"))
      (choice (ref hex-float) (ref hex-integer)))
     (when-prefix (seq (byte 48) (one-of-bytes "bB")) (ref binary-integer))
     (when-prefix (seq (byte 48) (one-of-bytes "oO")) (ref octal-integer))
     (when (and (prefix (seq (byte 48) (range 49 55)))
            (not (after-decimal-digits (one-of-bytes ".eE"))))
      (ref legacy-octal))
     (otherwise (choice (ref decimal-float) (ref decimal-integer))))))
  (require (not-followed-by (ref identifier-rest)))
  (emit-by-form (integer lit-int) (float lit-float))
  (failure malformed))
 (rule octal-escape (seq (byte 92) (repeat-between 1 3 (ref octal-digit)))
  (greedy true) (when-count 3 (require-first (range 48 51))))
 (rule byte-escape
  (seq (byte 92) (one-of-bytes "xXuU") (repeat-between 1 2 (ref hex-digit))))
 (rule c-escape
  (ordered (seq (byte 92) (ref simple-escape))
   (ref continuation) (ref octal-escape)
   (seq (byte 92) (one-of-bytes "xX") (repeat 1 (ref hex-digit)))
   (seq (byte 92) (byte 117) (repeat-exact 4 (ref hex-digit)))
   (seq (byte 92) (byte 85) (repeat-exact 8 (ref hex-digit)))))
 (rule x2c-escape
  (ordered (seq (byte 92) (ref simple-escape))
   (ref continuation) (ref octal-escape) (ref byte-escape)))
 (rule c-string
  (seq (byte 34)
   (repeat 0
    (choice (ref c-escape)
     (except (byte 0) (byte 10) (byte 13) (byte 34)
      (byte 92)))) (byte 34)) (on-nul incomplete)
  (on-raw-newline malformed) (emit lit-char*))
 (rule c-character
  (seq (byte 39)
   (choice (ref c-escape)
    (except (byte 0) (byte 10) (byte 13) (byte 39)
     (byte 92))) (byte 39)) (failure malformed)
  (emit lit-char))
 (rule quoted-symbol-spelling
  (seq (byte 34)
   (repeat 0 (choice (ref x2c-escape) (except (byte 0) (byte 34) (byte 92))))
   (byte 34)) (on-nul incomplete))
 (rule symbol
  (ordered (seq (byte 60) (ref quoted-symbol-spelling) (byte 62))
   (seq (byte 60) (repeat 1 (except (byte 0) (byte 62) (ref white-byte)))
    (byte 62))) (on-nul incomplete) (emit lit-symbol))
 (rule atom-escape (seq (byte 92) (except (byte 0))) (on-nul incomplete))
 (rule atom
  (repeat 1
   (ordered (ref atom-escape)
    (except (byte 0) (ref atom-boundary) (bytes "//")
     (bytes "/*")))))
 (rule collection-atom (ref atom)
  (stop-before-unescaped (ref collection-boundary))
  (emit lit-atom))
 (rule set-atom
  (ordered (ref quoted-symbol-spelling)
   (repeat 0
    (ordered (ref atom-escape) (except (byte 0) (ref set-boundary) (bytes ">>")))))
  (emit lit-atom))
 (rule string-segment
  (repeat 1
   (ordered (seq (byte 92) (byte 36)) (ref x2c-escape)
    (bytes "$") (except (byte 0) (byte 34) (byte 36) (byte 92))))
  (stop-before (choice (byte 34) (byte 36)))
  (on-nul malformed) (emit segment))
 (rule common
  (ordered (ref whitespace) (ref preprocessor)
   (ref line-comment) (ref block-comment) (ref c-character)
   (when-prefix (ref digit) (ref number))
   (when-prefix (seq (byte 46) (ref digit))
    (ref number))))
 (rule named-reference (seq (one-of-bytes "$@") (optional (ref identifier)))
  (emit-sigil) (emit-identifier ident) (keyword-lookup false))
 (rule code-reference
  (ordered (seq (bytes "$(") (transition embedded-lisp))
   (seq (bytes "@(") (transition embedded-lisp-splice))
   (when-prefix (one-of-bytes "$@") (ref named-reference))))
 (rule percent-forms
  (ordered (seq (bytes "%\"") (transition percent-string))
   (seq (bytes "%[") (transition percent-array))
   (seq (bytes "%<<") (transition percent-set))
   (when (not (predicate percent-is-operator))
    (ordered (seq (bytes "%(") (transition percent-list))
     (seq (bytes "%{") (transition percent-map))
     (seq (bytes "%!") (emit "%!")) (seq (bytes "%<") (fail malformed))))))
 (rule angle-symbol (when (predicate can-start-symbol) (ref symbol))
  (failure malformed))
 (rule c-tokens
  (ordered (ref c-string) (seq (ref identifier) (emit-keyword-or-ident))
   (longest-prefix operators (emit-spelling) (apply-transitions))))
 (rule data-prefix
  (ordered
   (when-mode array map
    (seq (bytes "void") (not-followed-by (ref identifier-rest))
     (emit void))) (when-mode list (seq (bytes "?(") (transition capture)))
   (when-mode list
    (when-prefix
     (seq (byte 64)
      (lookahead (choice (byte 0) (byte 61) (byte 41) (ref white-byte))))
     (seq (byte 64) (optional (byte 61)) (emit lit-atom))))
   (when-mode list array map
    (when-prefix (byte 36)
     (ordered (seq (bytes "${") (transition data-code))
      (ref named-reference))))
   (when-mode list array map
    (when-prefix (byte 64)
     (ordered (seq (bytes "@{") (transition list-splice))
      (ref named-reference))))))
 (rule lisp-reference (when-prefix (byte 36) (ref named-reference)))
 (rule lisp-symbol (when (predicate lisp-angle-start) (ref symbol)))
 (rule signed-number
  (when-prefix (seq (one-of-bytes "+-") (ref digit))
   (ref number)))
 (rule data-punctuation
  (ordered (seq (one-of-bytes "()") (emit-spelling) (apply-transitions))
   (when-mode list array map (seq (one-of-bytes "\"{[") (apply-transitions)))
   (when-mode lisp macro-lisp (ref c-string))
   (when-mode array map
    (seq (one-of-bytes "]}:,") (emit-spelling)
     (apply-transitions))) (seq (one-of-bytes "\'`") (emit-spelling))
   (when-mode list lisp macro-lisp
    (seq (byte 44) (optional (byte 64)) (emit-spelling)))
   (ref lisp-symbol) (ref signed-number)))
 (rule list-atom (ref atom) (emit lit-atom))
 (rule lisp-atom (ref atom) (emit ident))
 (rule symbol-set-tokens
  (ordered (seq (bytes ">>") (transition close-set))
   (seq (one-of-bytes "$@") (emit-spelling))
   (ref symbol) (ref set-atom)) (failure malformed))
 (rule string-tokens
  (ordered (seq (bytes "${") (transition string-code))
   (when-prefix (byte 36) (ref named-reference))
   (seq (byte 34) (transition close-string))
   (ref string-segment))))

modes

(modes
 (mode x2c (ordered common code-reference percent-forms angle-symbol c-tokens))
 (mode x2c-par
  (ordered common code-reference percent-forms
   angle-symbol c-tokens))
 (mode array (ordered common data-prefix data-punctuation collection-atom))
 (mode map (ordered common data-prefix data-punctuation collection-atom))
 (mode list (ordered data-prefix data-punctuation common list-atom))
 (mode lisp (ordered lisp-reference data-punctuation common lisp-atom))
 (mode macro-lisp (ordered lisp-reference data-punctuation common lisp-atom))
 (mode symbol-set (ordered common symbol-set-tokens))
 (mode string (ordered string-tokens)))

transitions

(transitions
 (defaults (emit spelling) (width input) (unlisted-mode-action unchanged))
 (transition code-brace (from x2c x2c-par) (on "{") (push x2c))
 (transition close-code (from x2c x2c-par)
  (on "}") (pop when-depth-exceeds-one))
 (transition percent-list (from x2c x2c-par)
  (on "%(") (push list))
 (transition percent-array (from x2c x2c-par)
  (on "%[") (push array))
 (transition percent-map (from x2c x2c-par)
  (on "%{") (push map))
 (transition percent-set (from x2c x2c-par)
  (on "%<<") (push symbol-set))
 (transition percent-string (from x2c x2c-par)
  (on "%\"") (push string))
 (transition embedded-lisp (from x2c x2c-par)
  (on "$(") (push macro-lisp))
 (transition embedded-lisp-splice (from x2c x2c-par)
  (on "@(") (push macro-lisp))
 (transition code-parenthesis (from x2c-par)
  (on "(") (push x2c-par))
 (transition close-code-parenthesis (from x2c-par)
  (on ")") (pop))
 (transition nested-list (from list array map)
  (on "(") (push list))
 (transition nested-array (from list array map)
  (on "[") (emit "%[") (push array))
 (transition nested-map (from list array map)
  (on "{") (emit "%{") (push map))
 (transition nested-string (from list array map)
  (on "\"") (emit "%\"") (push string))
 (transition data-code (from list array map)
  (on "${") (push x2c))
 (transition list-splice (from list array map)
  (on "@{") (push x2c))
 (transition capture (from list) (on "?(")
  (push x2c-par)) (transition close-list (from list) (on ")") (pop))
 (transition close-array (from array) (on "]") (pop))
 (transition close-map (from map) (on "}") (pop))
 (transition close-set (from symbol-set) (on ">>") (pop))
 (transition string-code (from string) (on "${") (push x2c))
 (transition close-string (from string) (on "\"") (pop))
 (transition nested-lisp (from macro-lisp) (on "(") (push macro-lisp))
 (transition close-lisp (from macro-lisp) (on ")") (pop)))

predicates

(predicates
 (predicate percent-is-operator
  (and (not (predicate layout-statement-start))
   (previous-kind-in operand-end) (not (predicate closes-control))
   (not (predicate closes-statement-block))))
 (predicate can-start-symbol
  (or (predicate layout-statement-start) (not (previous-kind-in operand-end))
   (previous-ident "tag")
   (previous-sequence (kind-in operand-end)
    (ident "is") (optional (ident "not")))))
 (predicate layout-statement-start
  (and layout-enabled (last-stored-kind space)
   (last-stored-text-contains (byte 10)) (previous-significant-exists)
   (column-at-most first-significant-on-previous-token-physical-line)))
 (predicate closes-control
  (and (previous-kind ")")
   (before-matching-ordinary-group-kind-in "if"
    "while" "for" "switch")))
 (predicate closes-statement-block
  (and (previous-kind "}")
   (or (before-matching-ordinary-group-kind-in
        ";" ":" "{" "}" "]" "ident" "else" "do" "try"
        "finally" "defer")
    (before-matching-ordinary-group (kind ")")
     (before-matching-ordinary-group-kind-in "ident"
      "match" "if" "while" "for" "switch")))))
 (predicate lisp-angle-start
  (and (prefix (byte 60))
   (not (next-byte-in (byte 0) (byte 61) (ref white-byte)
         (one-of-bytes "()\'`,")))))
 (token-set operand-end ident lit-int lit-float
  lit-char* lit-char lit-atom lit-symbol ")"
  "]" "}" "\"" "++" "--")
 (token-set compiler-operand-end ident lit-int
  lit-float lit-char* lit-char lit-atom lit-symbol
  ")" "]")
 (token-set compiler-operand-start ident lit-int
  lit-float lit-char* lit-char lit-atom lit-symbol
  "(" "%(" "%[" "%{" "$(" "${" "$" "!" "-"
  "*" "&" "~" "++" "--"))

passes

(passes
 (pass pipeline
  (ordered frontend-shebang scan layout lexical-diagnostics
   conditionals keyword-retag parser))
 (pass frontend-shebang (external "src/frontend.x")
  (replace-with scripting-include) (preserve subsequent-line-numbers))
 (pass layout
  (enabled-by (extension ".xp")
   (directive-before-code "#pragma indent" trailing-whitespace-only))
  (state logical-lines bracket-depth pending-ternary indents closers enums)
  (depth-open (text-ends-in "([{") (except-kind lit-char* lit-char segment))
  (depth-close (kind-in ")" "]" "}"))
  (ordered (index-significant (exclude space comment) (retain-physical true))
   (split-lines (always directive)
    (otherwise
     (and physical-line-break depth-zero (not (leading "."))
      (or no-deeper-indent previous-nonternary-colon))))
   (check-tabs (immediately-preceding space)
    (after-last-lf) (except directive) (fail indent))
   (base-indent first-statement-column)
   (open-block (when final-nonternary-colon next-statement-deeper)
    (replace ":" "{") (except (depth-zero-word case default catch) (append "{"))
    (push next-statement-indent))
   (wrap-control-condition
    (last-depth-zero-word if while for foreach
     switch match) (exclude dotted-member) (unless already-parenthesized)
    (for written-parentheses)
    (inline-body (first-depth-zero-nonternary-colon)
     (stop-at-label) (remove-if control-header-or-preceding-else)))
   (select-closer
    (aggregate (contains struct union enum) (no ordinary-parameter-open))
    (default "}") (aggregate "};") (leading-typedef "}")
    (leading-static-typedef "}"))
   (bare-do (line "do:")
    (unless next-same-or-lower-line-is-same-indent-while-trailer)
    (remove "do"))
   (append-semicolon
    (unless directive final-semicolon enum-body
     whole-embedded-lisp whole-syntax-hole syntax-hole-after-close-paren
     decorator)) (decorator (leading "@" "$") (retag-first space))
   (indent-directive (retag comment))
   (dedent (append-stored-closers) (require remaining-level)
    (fail indent)) (eof close-all)) (external-details "lib/tokenizer.x:_Layout")
  (inserted-width 0) (reclassified-width preserve))
 (pass lexical-diagnostics (external "src/compiler.x:Compiler.tokenize"))
 (pass conditionals (external "src/preprocess.x:Compiler.scan_conditionals")
  (after scan) (hide inactive-branches) (retain conditional-structure))
 (pass keyword-retag
  (ordered
   (retag in ident
    (unless
     (and (previous-kind-in compiler-operand-end)
      (next-kind-in compiler-operand-start))))
   (retag match ident
    (unless (next-after-optional-parenthesized-group-kind-in case "{"))))))

keywords

(keywords (keyword "associated" associated)
 (keyword "auto" auto) (keyword "break" break)
 (keyword "case" case) (keyword "catch" catch)
 (keyword "char" char) (keyword "const" const)
 (keyword "continue" continue) (keyword "default" default)
 (keyword "defer" defer) (keyword "delegate" delegate)
 (keyword "do" do) (keyword "double" double)
 (keyword "else" else) (keyword "enum" enum)
 (keyword "extern" extern) (keyword "finally" finally)
 (keyword "float" float) (keyword "for" for)
 (keyword "goto" goto) (keyword "if" if) (keyword "import" import)
 (keyword "in" in) (keyword "inline" inline)
 (keyword "int" int) (keyword "long" long)
 (keyword "match" match) (keyword "protocol" protocol)
 (keyword "raise" raise) (keyword "register" register)
 (keyword "restrict" restrict) (keyword "return" return)
 (keyword "short" short) (keyword "signed" signed)
 (keyword "sizeof" sizeof) (keyword "static" static)
 (keyword "struct" struct) (keyword "switch" switch)
 (keyword "threaded" threaded) (keyword "try" try)
 (keyword "typedef" typedef) (keyword "union" union)
 (keyword "unsigned" unsigned) (keyword "void" void)
 (keyword "volatile" volatile) (keyword "while" while)
 (keyword "__inline" inline) (keyword "__inline__" inline)
 (keyword "__restrict" restrict) (keyword "__restrict__" restrict)
 (keyword "thread_local" threaded) (keyword "_Thread_local" threaded))

operators

(operators (selection longest-prefix) (operator "...")
 (operator ">>=") (operator "<<=") (operator "===")
 (operator "!==") (operator "->") (operator "++")
 (operator "--") (operator "<<") (operator ">>")
 (operator "<=") (operator ">=") (operator "==")
 (operator "!=") (operator "&&") (operator "||")
 (operator "+=") (operator "-=") (operator "*=")
 (operator "/=") (operator "%=") (operator "&=")
 (operator "^=") (operator "|=") (operator "-")
 (operator ",") (operator ";") (operator ":")
 (operator "!") (operator "?") (operator ".")
 (operator "(") (operator ")") (operator "[")
 (operator "]") (operator "{") (operator "}")
 (operator "*") (operator "/") (operator "&")
 (operator "%") (operator "^") (operator "+")
 (operator "<") (operator "=") (operator ">")
 (operator "|") (operator "~") (operator "@"))

failures

(failures
 (scanner-result (positive matched-byte-count)
  (zero no-match) (negative failure))
 (unsupported-start (scan_keyword -1l) (scan_c_operator -1l))
 (empty-input (scan_white_space -1l)) (status-aware ok incomplete malformed)
 (statusless-failure malformed) (record-first-status)
 (append (token error (width 0)) (token eof (width 0)))
 (callback-result 0) (dispatch-may-continue true)
 (semantic-stop first-eof) (unclosed-mode-at-eof parser-responsibility)
 (layout-status indent) (prefix-preconditions external-error)
 (numeric-signed-invalid-prefix may-return-zero)
 (string-leading-dollar dispatch-before-segment)
 (legacy-number-type-helper skip-minus-only))

sources

(sources (source "lib/scan.x") (source "lib/tokenizer.x")
 (source "src/compiler.x") (source "src/literals.x")
 (source "docs/src/reference/syntax/lexical.md"))

boundaries

(boundaries
 (external percent-string-decoding "src/literals.x:Compiler._parse_text_segment"
  (remove continuations) (normalize cr-and-crlf-to-lf)
  (decode escapes)) (external symbol-representability "src/literals.x")
 (external host-preprocessing host-preprocessor)
 (external native-c-acceptance host-compiler))