feat tokenizer init once

cmd/main.go

-package main
-
-import (
-	"fmt"
-	"gitlab.schukai.com/oss/libraries/go/utilities/data.git"
-)
-
-func main() {
-
-	m := map[any]any{}
-
-	for i := 0; i < 100000; i++ {
-
-		tt := data.NewTransformer(m)
-		c, _ := tt.Transform("static:AtEst | strtolower | strtoupper | prefix:b | md5")
-		fmt.Println(c)
-
-	}
-
-}

go.mod

@@ -3,3 +3,5 @@ module gitlab.schukai.com/oss/libraries/go/utilities/data.git
 go 1.19
 
 require github.com/volker-schukai/tokenizer v1.0.0
+
+replace github.com/volker-schukai/tokenizer => /home/volker.schukai/projekte/github/tokenizer

transformer.go

@@ -22,6 +22,7 @@ import (
 type Transformer struct {
 	dataset map[any]any
 	errors  []error
+	parser  *tokenizer.Tokenizer
 }
 
 type Number interface {
@@ -29,8 +30,11 @@ type Number interface {
 }
 
 func NewTransformer(dataset map[any]any) *Transformer {
+	parser := initTokenizer()
+
 	return &Transformer{
 		dataset: dataset,
+		parser:  parser,
 	}
 }
 
@@ -51,10 +55,8 @@ func (t *Transformer) Transform(pipe string) (interface{}, error) {
 
 func (t *Transformer) tokenize(pipe string) (any, error) {
 
-	parser := initTokenizer()
-
 	// create tokens stream
-	stream := parser.ParseString(pipe)
+	stream := t.parser.ParseString(pipe)
 	defer stream.Close()
 
 	tokenMap := make([]TokenList, 0)