with-location := λ(: t Token)(: loc SourceLocation). (: (
   (match t (
      ()
      ( (Token( k _ _ )) (
         (set t (Token( k (iuid()) loc )))
      ))
   ))
   t
) Token);

without-location := λ(: t Token). (: (
   (match t (
      ()
      ( (Token( k ki _ )) (
         (set t (Token( k ki (SourceLocation( '_s 0_u64 0_u64 )) )))
      ))
   ))
   t
) Token);

with-key := λ(: t Token)(: k String). (: (
   (match t (
      ()
      ( (Token( _ _ loc )) (
         (set t (Token( k (iuid()) loc )))
      ))
   ))
   t
) Token);

location-of := λ(: t Token). (: (
   (let loc (.1( (as t Tag<Token>) )))
   loc
) SourceLocation);

id-of := λ(: t Token). (: (
   (let id (.2( (as t Tag<Token>) )))
   id
) U64);

key-of := λ(: t Token). (: (
   (let key (.3( (as t Tag<Token>) )))
   key
) String);

unique := λ(: t Token). (: (
   (match t (
      ()
      ( (Token( s id loc )) (
         (set t (Token( s (iuid()) loc )))
      ))
   ))
   t
) Token);

print-verbose := λ(: t Token). (: (
   (match t (
      ()
      ( (Token( s id loc )) (
         (print s)(print '\o_s)(print id)
      ))
   ))
) Nil);

token::new := λ(: s String). (: (
   (let tk (Token( s (iuid()) (location::new()) )))
   tk
) Token);

location::new := λ. (: (
   (let loc (SourceLocation( 'Unknown_s 0_u64 0_u64 )))
   loc
) SourceLocation);

non-zero := λ(: loc SourceLocation). (: (
   (let r 0_u64)
   (match loc (
      ()
      ( (SourceLocation( fp line col )) (
         (if (!=( line 0_u64 )) (set r 1_u64) ())
      ))
   ))
   r
) U64);

location-of := λ(: term AST). (: (
   (let r (SourceLocation( 'Unknown_s 0_u64 0_u64 )))
   (match term (
      ()
      ( (Var( _ v )) (set r (location-of v)) )
      ( (Lit( _ v )) (set r (location-of v)) )
      ( (App( tl tr )) (
         (set r (location-of tl))
         (if (not(non-zero r)) (set r (location-of tr)) ())
      ))
      ( (Abs( tl tr tlt )) (
         (set r (location-of tl))
         (if (not(non-zero r)) (set r (location-of tr)) ())
      ))
      ( (Seq( tl tr )) (
         (set r (location-of tl))
         (if (not(non-zero r)) (set r (location-of tr)) ())
      ))
      ( (Glb( tl tr )) (
         (set r (location-of tl))
         (if (not(non-zero r)) (set r (location-of tr)) ())
      ))
      ( (Typedef( tl tr )) (
         (set r (location-of tl))
         (if (not(non-zero r)) (set r (location-of tr)) ())
      ))
      ( (Frg( tl tr )) (
         (set r (location-of tl))
         (if (not(non-zero r)) (set r (location-of tr)) ())
      ))
      ( _ () )
   ))
   r
) SourceLocation);

already-tokenized := (: LEOF List<String>);
is-already-tokenized := λ(: fp String). (: (
   (let r 0_u64)
   (for-each (fpt in already-tokenized) (
      (if (==( fp fpt )) (set r 1_u64) ())
   ))
   r
) U64);

tokenize := λ(: fp String). (: (
   (let text '_s)
   (if (is-already-tokenized fp) () (
      (set text (read-file fp))
      (set already-tokenized (cons( fp already-tokenized )))
   ))
   (let in_comment False_u8)
   (let buffer SNil)
   (let current-line 1_u64)
   (let current-column 1_u64)
   (let token-line 1_u64)
   (let token-column 1_u64)
   (let parens-counter 0_i64)
   (while (head-string text) (
      (match (head-string text) (
         ()
         ( '\n_u8 (
            (set current-line (+( current-line 1_u64 )))
            (set current-column 1_u64)
         ))
         ( _ (
            (set current-column (+( current-column 1_u64 )))
         ))
      ))
      (match (head-string text) (
         ()
         ( \o_u8 (
            (if (non-zero buffer) (
               (let tk (Token( (clone-rope buffer) (iuid()) (SourceLocation( fp token-line token-column )) )))
               (set ast-tokenized-program (LCons(
                  tk
                  (close ast-tokenized-program)
               )))
               (set buffer SNil)
            ) ())
            (set token-line current-line)
            (set token-column current-column)
            (set in_comment True_u8)
         ))
         ( \n_u8 (
            (if (non-zero buffer) (
               (let tk (Token( (clone-rope buffer) (iuid()) (SourceLocation( fp token-line token-column )) )))
               (set ast-tokenized-program (LCons(
                  tk
                  (close ast-tokenized-program)
               )))
               (set buffer SNil)
            ) ())
            (set token-line current-line)
            (set token-column current-column)
            (set in_comment False_u8)
         ))
         ( \t_u8 (
            (if (non-zero buffer) (
               (let tk (Token( (clone-rope buffer) (iuid()) (SourceLocation( fp token-line token-column )) )))
               (set ast-tokenized-program (LCons(
                  tk
                  (close ast-tokenized-program)
               )))
               (set buffer SNil)
            ) ())
            (set token-line current-line)
            (set token-column current-column)
         ))
         ( \s_u8 (
            (if (non-zero buffer) (
               (let tk (Token( (clone-rope buffer) (iuid()) (SourceLocation( fp token-line token-column )) )))
               (set ast-tokenized-program (LCons(
                  tk
                  (close ast-tokenized-program)
               )))
               (set buffer SNil)
            ) ())
            (set token-line current-line)
            (set token-column current-column)
         ))

         ( \[_u8 (if (==( in_comment True_u8 )) () (
            (if (non-zero buffer) (
               (let tk (Token( (clone-rope buffer) (iuid()) (SourceLocation( fp token-line token-column )) )))
               (set ast-tokenized-program (LCons(
                  tk
                  (close ast-tokenized-program)
               )))
               (set buffer SNil)
            ) ())
            (set parens-counter (+( parens-counter 1_i64 )))
            (let tk-2 (Token( '\[_s (iuid()) (SourceLocation( fp token-line token-column )) )))
            (set ast-tokenized-program (LCons(
               tk-2
               (close ast-tokenized-program)
            )))
            (set token-line current-line)
            (set token-column current-column)
         )))
         ( \]_u8 (if (==( in_comment True_u8 )) () (
            (if (non-zero buffer) (
               (let tk (Token( (clone-rope buffer) (iuid()) (SourceLocation( fp token-line token-column )) )))
               (set ast-tokenized-program (LCons(
                  tk
                  (close ast-tokenized-program)
               )))
               (set buffer SNil)
            ) ())
            (set parens-counter (-( parens-counter 1_i64 )))
            (let tk-2 (Token( '\]_s (iuid()) (SourceLocation( fp token-line token-column )) )))
            (set ast-tokenized-program (LCons(
               tk-2
               (close ast-tokenized-program)
            )))
         )))
         ( \`_u8 (if (==( in_comment True_u8 )) () (
            (if (non-zero buffer) (
               (let tk (Token( (clone-rope buffer) (iuid()) (SourceLocation( fp token-line token-column )) )))
               (set ast-tokenized-program (LCons(
                  tk
                  (close ast-tokenized-program)
               )))
               (set buffer SNil)
            ) ())
            (let tk-2 (Token( '\`_s (iuid()) (SourceLocation( fp token-line token-column )) )))
            (set ast-tokenized-program (LCons(
               tk-2
               (close ast-tokenized-program)
            )))
         )))
         ( \:_u8 (if (==( in_comment True_u8 )) () (
            (if (non-zero buffer) (
               (let tk (Token( (clone-rope buffer) (iuid()) (SourceLocation( fp token-line token-column )) )))
               (set ast-tokenized-program (LCons(
                  tk
                  (close ast-tokenized-program)
               )))
               (set buffer SNil)
            ) ())
            (let tk-2 (Token( '\:_s (iuid()) (SourceLocation( fp token-line token-column )) )))
            (set ast-tokenized-program (LCons(
               tk-2
               (close ast-tokenized-program)
            )))
         )))

         ( c (if (==( in_comment True_u8 )) () (
            (set buffer (SCons(
               (close buffer)
               (close (SAtom (clone-rope c)))
            )))
            (if (==( '\l_s (clone-rope buffer) )) (
               (let tk (Token( '\l_s (iuid()) (SourceLocation( fp token-line token-column )) )))
               (set ast-tokenized-program (LCons(
                  tk
                  (close ast-tokenized-program)
               )))
               (set buffer SNil)
           ) ())
         )))
      ))
      (set text (tail-string text))
   ))

   (if (non-zero buffer) (
      (let tk (Token( (clone-rope buffer) (iuid()) (SourceLocation( fp token-line token-column )) )))
      (set ast-tokenized-program (LCons(
         tk
         (close ast-tokenized-program)
      )))
   ) ())

   (set ast-tokenized-program (reverse ast-tokenized-program))

   (if (==( parens-counter 0_i64 )) () (
      (eprint 'Hanging\sParentheses\sIn\sFile:\s_s)
      (eprint fp)
      (eprint '\sCount:\s_s)
      (print parens-counter)
      (eprint '\n_s)
      (exit 1_u64)
   ))

) Nil);