package tokenizer import ( "fmt" "strconv" ) var undefToken = &Token{ id: -1, } // Token struct describe one token. type Token struct { id int key TokenKey value []byte line int offset int indent []byte string *StringSettings prev *Token next *Token } // addNext add new token as next node of dl-list. func (t *Token) addNext(next *Token) { next.prev = t t.next = next } // unlink remove token from dl-list and fix links of prev and next nodes. // Method returns next token or nil if no next token found. func (t *Token) unlink() *Token { next := t.next t.next.prev = nil t.next = nil t.prev = nil return next } // ID returns id of token. Id is the sequence number of tokens in the stream. func (t *Token) ID() int { return t.id } // String returns a multiline string with the token's information. func (t Token) String() string { return fmt.Sprintf("{\n\tId: %d\n\tKey: %d\n\tValue: %s\n\tPosition: %d\n\tIndent: %d bytes\n\tLine: %d\n}", t.id, t.key, t.value, t.offset, len(t.indent), t.line) } // IsValid checks if this token is valid — the key is not TokenUndef. func (t *Token) IsValid() bool { return t.key != TokenUndef } // IsKeyword checks if this is keyword — the key is TokenKeyword. func (t Token) IsKeyword() bool { return t.key == TokenKeyword } // IsNumber checks if this token is integer or float — the key is TokenInteger or TokenFloat. func (t Token) IsNumber() bool { return t.key == TokenInteger || t.key == TokenFloat } // IsFloat checks if this token is float — the key is TokenFloat. func (t Token) IsFloat() bool { return t.key == TokenFloat } // IsInteger checks if this token is integer — the key is TokenInteger. func (t Token) IsInteger() bool { return t.key == TokenInteger } // ValueInt returns value as int64. // If the token is float the result wild be round by math's rules. // If the token is not TokenInteger or TokenFloat zero will be returned. // Method doesn't use cache. Each call starts a number parser. func (t Token) ValueInt() int64 { if t.key == TokenInteger { num, _ := strconv.ParseInt(b2s(t.value), 10, 64) return num } else if t.key == TokenFloat { num, _ := strconv.ParseFloat(b2s(t.value), 64) return int64(num) } return 0 } // ValueFloat returns value as float64. // If the token is not TokenInteger or TokenFloat zero will be returned. // Method doesn't use cache. Each call starts a number parser. func (t *Token) ValueFloat() float64 { if t.key == TokenFloat { num, _ := strconv.ParseFloat(b2s(t.value), 64) return num } else if t.key == TokenInteger { num, _ := strconv.ParseInt(b2s(t.value), 10, 64) return float64(num) } return 0.0 } // Indent returns spaces before the token. func (t *Token) Indent() []byte { return t.indent } // Key returns the key of the token pointed to by the pointer. // If pointer is not valid (see IsValid) TokenUndef will be returned. func (t *Token) Key() TokenKey { return t.key } // Value returns value of current token as slice of bytes from source. // If current token is invalid value returns nil. // // Do not change bytes in the slice. Copy slice before change. func (t *Token) Value() []byte { return t.value } // ValueString returns value of the token as string. // If the token is TokenUndef method returns empty string. func (t *Token) ValueString() string { if t.value == nil { return "" } return b2s(t.value) } // Line returns line number in input string. // Line numbers starts from 1. func (t *Token) Line() int { return t.line } // Offset returns the byte position in input string (from start). func (t *Token) Offset() int { return t.offset } // StringSettings returns StringSettings structure if token is framed string. func (t *Token) StringSettings() *StringSettings { return t.string } // StringKey returns key of string. // If key not defined for string TokenString will be returned. func (t *Token) StringKey() TokenKey { if t.string != nil { return t.string.Key } return TokenString } // IsString checks if current token is a quoted string. // Token key may be TokenString or TokenStringFragment. func (t Token) IsString() bool { return t.key == TokenString || t.key == TokenStringFragment } // ValueUnescaped returns clear (unquoted) string // - without edge-tokens (quotes) // - with character escaping handling // // For example quoted string // "one \"two\"\t three" // transforms to // one "two" three // Method doesn't use cache. Each call starts a string parser. func (t *Token) ValueUnescaped() []byte { if t.string != nil { from := 0 to := len(t.value) if bytesStarts(t.string.StartToken, t.value) { from = len(t.string.StartToken) } if bytesEnds(t.string.EndToken, t.value) { to = len(t.value) - len(t.string.EndToken) } str := t.value[from:to] result := make([]byte, 0, len(str)) escaping := false start := 0 for i := 0; i < len(str); i++ { if escaping { if v, ok := t.string.SpecSymbols[str[i]]; ok { result = append(result, t.value[start:i]...) result = append(result, v) } start = i escaping = false } else if t.string.EscapeSymbol != 0 && str[i] == t.string.EscapeSymbol { escaping = true } } if start == 0 { // no one escapes return str } return result } return t.value } // ValueUnescapedString like as ValueUnescaped but returns string. func (t *Token) ValueUnescapedString() string { if s := t.ValueUnescaped(); s != nil { return b2s(s) } return "" } // Is checks if the token has any of these keys. func (t *Token) Is(key TokenKey, keys ...TokenKey) bool { if t.key == key { return true } if len(keys) > 0 { for _, k := range keys { if t.key == k { return true } } } return false }