- let rec tokid acc = function
- | c :: t when Char.IsLetter(c) -> tokid(acc + (c.ToString())) t
- | '_' :: t -> tokid(acc + "_") t
- | c :: t when Char.IsDigit(c) -> tokid(acc + (c.ToString())) t
- | t -> acc, t
- let rec tokenize (acc : string list) = function
- | c :: t when Char.IsWhiteSpace(c) -> tokenize acc t //ignore whitespace
- | c :: t when Char.IsLetter(c) -> //starts with a letter so it’s keyword or name
- let s,t' = tokid "" t //read the full string
- match (c.ToString() + s) with
- | "sizeof" -> tokenize ("sizeof" :: acc) t' //is sizeof keyword
- | t -> tokenize (t :: "$id" :: acc) t' //is
- | '.' :: '.' :: '.' :: t -> tokenize ("..." :: acc) t
- | '<' :: '<' :: '=' :: t -> tokenize ("<<=" :: acc) t
- | '>' :: '>' :: '=' :: t -> tokenize (">>=" :: acc) t
- | [] -> List.rev acc
- | t -> failwith ("Invalid token near " + (seqstr t) )