feat: new watcher lib #8

3eb35891 · Volker Schukai · 95b67e57 · 3eb35891 · 3eb35891 · 3eb35891
Verified Commit 3eb35891 authored 1 year ago by Volker Schukai
--- a/Taskfile.yml
+++ b/Taskfile.yml
@@ -7,11 +7,14 @@ tasks:
    cmds:
      - task --list-all
    silent: true
  test:
    desc: Execute unit tests in Go.
    cmds:  
      - echo "Execute unit tests in Go."
      - go test -cover -v ./...
+      - go test -bench .
+      - go test -race .
  test-fuzz:
    desc: Conduct fuzzing tests.#

--- a/go.mod
+++ b/go.mod
@@ -7,13 +7,14 @@ require (
 	github.com/imdario/mergo v0.3.16
 	github.com/kinbiko/jsonassert v1.1.1
 	github.com/magiconair/properties v1.8.7
-	github.com/pelletier/go-toml/v2 v2.0.9
+	github.com/pelletier/go-toml/v2 v2.1.0
 	github.com/r3labs/diff/v3 v3.0.1
 	github.com/stretchr/testify v1.8.4
 	gitlab.schukai.com/oss/libraries/go/application/xflags v1.9.0
-	gitlab.schukai.com/oss/libraries/go/network/http-negotiation v1.3.0
+	gitlab.schukai.com/oss/libraries/go/network/http-negotiation v1.3.1
 	gitlab.schukai.com/oss/libraries/go/utilities/pathfinder v0.5.2
-	golang.org/x/exp v0.0.0-20230811145659-89c5cff77bcb
+	gitlab.schukai.com/oss/libraries/go/utilities/watch v0.2.0
+	golang.org/x/exp v0.0.0-20230905200255-921286631fa9
 	gopkg.in/yaml.v3 v3.0.1
 )
@@ -26,7 +27,7 @@ require (
 	github.com/vmihailenco/msgpack/v5 v5.3.5 // indirect
 	github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
 	golang.org/x/net v0.8.0 // indirect
-	golang.org/x/sys v0.11.0 // indirect
+	golang.org/x/sys v0.12.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
 	google.golang.org/protobuf v1.29.0 // indirect
 )
--- a/go.sum
+++ b/go.sum
@@ -27,6 +27,8 @@ github.com/pelletier/go-toml/v2 v2.0.7 h1:muncTPStnKRos5dpVKULv2FVd4bMOhNePj9Cjg
 github.com/pelletier/go-toml/v2 v2.0.7/go.mod h1:eumQOmlWiOPt5WriQQqoM5y18pDHwha2N+QD+EUNTek=
 github.com/pelletier/go-toml/v2 v2.0.9 h1:uH2qQXheeefCCkuBBSLi7jCiSmj3VRh2+Goq2N7Xxu0=
 github.com/pelletier/go-toml/v2 v2.0.9/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc=
+github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4=
+github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/r3labs/diff/v3 v3.0.0 h1:ZhPwNxn9gW5WLPBV9GCYaVbMdLOSmJ0DeKdCiSbOLUI=
@@ -56,12 +58,18 @@ gitlab.schukai.com/oss/libraries/go/application/xflags v1.9.0 h1:bSnwEV56JZQWBQC
 gitlab.schukai.com/oss/libraries/go/application/xflags v1.9.0/go.mod h1:KN99uofMnTNcpfKwPbskucCTgwivJa3jfP2BHM4Ac+A=
 gitlab.schukai.com/oss/libraries/go/network/http-negotiation v1.3.0 h1:SZG0BW5ll3WK5ZIOTogjqX8oVHCTxANTDLPxUs7Rnx8=
 gitlab.schukai.com/oss/libraries/go/network/http-negotiation v1.3.0/go.mod h1:RS2rKf5O+rmSBshHLOgjG7dxg5N2MhNYokZOBcuXdX8=
+gitlab.schukai.com/oss/libraries/go/network/http-negotiation v1.3.1 h1:B6BZV3bURUew5u+L/QLaBjdqTlW7P3dHTO19QLkPSfI=
+gitlab.schukai.com/oss/libraries/go/network/http-negotiation v1.3.1/go.mod h1:RS2rKf5O+rmSBshHLOgjG7dxg5N2MhNYokZOBcuXdX8=
 gitlab.schukai.com/oss/libraries/go/utilities/pathfinder v0.3.0 h1:mSxk2q/npskmHMmw1oF4moccjGav5dL6qmff2njUV7A=
 gitlab.schukai.com/oss/libraries/go/utilities/pathfinder v0.3.0/go.mod h1:UvdD4NAf3gLKYafabJD7e9ZCOetzM9JZ9y4GkZukPVU=
 gitlab.schukai.com/oss/libraries/go/utilities/pathfinder v0.3.1 h1:oyElaqEiyr2XgaE1CYwD8LoeHsuR/vQD/p6k3jYbJFs=
 gitlab.schukai.com/oss/libraries/go/utilities/pathfinder v0.3.1/go.mod h1:UvdD4NAf3gLKYafabJD7e9ZCOetzM9JZ9y4GkZukPVU=
 gitlab.schukai.com/oss/libraries/go/utilities/pathfinder v0.5.2 h1:R+dL2NJCM+AQNPK4DPDmfvx1eomi1Xb1dl0XKEFj7Ek=
 gitlab.schukai.com/oss/libraries/go/utilities/pathfinder v0.5.2/go.mod h1:UvdD4NAf3gLKYafabJD7e9ZCOetzM9JZ9y4GkZukPVU=
+gitlab.schukai.com/oss/libraries/go/utilities/watch v0.1.0 h1:FAKHmf9p3NKyzuM0cIXYBxmhdQ7zJ+6wj5qqeoIMbGc=
+gitlab.schukai.com/oss/libraries/go/utilities/watch v0.1.0/go.mod h1:tMFl68peRKHgFQLltrTN3JLredofMqvGi3C0SEAj73Y=
+gitlab.schukai.com/oss/libraries/go/utilities/watch v0.2.0 h1:tLjN9Wyv+LJhtiiQDzdzaDelEq2LVCDP3Ndo7ZPIWfQ=
+gitlab.schukai.com/oss/libraries/go/utilities/watch v0.2.0/go.mod h1:tMFl68peRKHgFQLltrTN3JLredofMqvGi3C0SEAj73Y=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/exp v0.0.0-20221012211006-4de253d81b95 h1:sBdrWpxhGDdTAYNqbgBLAR+ULAPPhfgncLr1X0lyWtg=
 golang.org/x/exp v0.0.0-20221012211006-4de253d81b95/go.mod h1:cyybsKvd6eL0RnXn6p/Grxp8F5bW7iYuBgsNCOHpMYE=
@@ -75,6 +83,8 @@ golang.org/x/exp v0.0.0-20230810033253-352e893a4cad h1:g0bG7Z4uG+OgH2QDODnjp6ggk
 golang.org/x/exp v0.0.0-20230810033253-352e893a4cad/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc=
 golang.org/x/exp v0.0.0-20230811145659-89c5cff77bcb h1:mIKbk8weKhSeLH2GmUTrvx8CjkyJmnU1wFmg59CUjFA=
 golang.org/x/exp v0.0.0-20230811145659-89c5cff77bcb/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc=
+golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjsbSXD66ic0XW0js0R9g=
+golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k=
 golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
 golang.org/x/net v0.0.0-20221014081412-f15817d10f9b h1:tvrvnPFcdzp294diPnrdZZZ8XUt2Tyj7svb7X52iDuU=
 golang.org/x/net v0.0.0-20221014081412-f15817d10f9b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
@@ -92,6 +102,8 @@ golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM=
 golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o=
+golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=

--- a/import.go
+++ b/import.go
@@ -172,32 +172,9 @@ func handleField(p string, r reflect.Value) {
 	}
 }
-//func forEachElem(r reflect.Value, fn func(e reflect.Value)) {
-//	switch r.Kind() {
-//	case reflect.Slice:
-//		for i := 0; i < r.Len(); i++ {
-//			fn(r.Index(i))
-//		}
-//	case reflect.Map:
-//		for _, k := range r.MapKeys() {
-//			fn(r.MapIndex(k))
-//		}
-//	case reflect.Ptr, reflect.Interface:
-//		if !r.IsNil() {
-//			fn(r.Elem())
-//		}
-//	}
-//}
 func (s *Settings[C]) importFiles() {
-	s.fileWatch.Lock()
+        defer func() {
-	defer func() {
-		s.fileWatch.Unlock()
-	}()
-	defer func() {
 		s.notifyErrorHooks()
 	}()

--- a/settings.go
+++ b/settings.go
@@ -4,21 +4,12 @@
 package configuration
 import (
+	"gitlab.schukai.com/oss/libraries/go/utilities/watch"
 	"reflect"
 	"strconv"
 	"sync"
-	"github.com/fsnotify/fsnotify"
 )
-type fileWatch struct {
-	sync.Mutex
-	watcher     *fsnotify.Watcher
-	watchList   map[string]bool
-	cancelWatch chan bool
-	onWatch     bool
-}
 // Settings is the main struct for the configuration
 type Settings[C any] struct {
 	files  fileBackend
@@ -34,7 +25,7 @@ type Settings[C any] struct {
 		postprocessing []PostprocessingHook
 	}
-	fileWatch fileWatch
+	fileWatch watch.Lighthouse
 }
 func (s *Settings[C]) initDefaults() *Settings[C] {

--- a/watch.go
+++ b/watch.go
@@ -4,72 +4,34 @@
 package configuration
 import (
-	"github.com/fsnotify/fsnotify"
+	"gitlab.schukai.com/oss/libraries/go/utilities/watch"
-	"os"
 	"path"
 )
-func (s *Settings[C]) initWatch() *Settings[C] {
-	var err error
-	defer func() {
-		s.notifyErrorHooks()
-	}()
-	if s.fileWatch.watcher != nil {
-		s.errors = append(s.errors, WatchAlreadyInitializedError)
-		return s
-	}
-	s.fileWatch.watcher, err = fsnotify.NewWatcher()
-	if err != nil {
-		s.errors = append(s.errors, err)
-		return s
-	}
-	return s
-}
 func (s *Settings[C]) StopWatching() *Settings[C] {
-	s.fileWatch.Lock()
+	if s.fileWatch != nil {
-	defer s.fileWatch.Unlock()
+		err := s.fileWatch.StopWatching()
+		if err != nil {
-	defer func() {
+			s.errors = append(s.errors, err)
-		s.notifyErrorHooks()
+			return s
-	}()
+		}
-	if s.fileWatch.watcher == nil {
-		s.errors = append(s.errors, WatchNotInitializedError)
-		return s
 	}
-	if !s.fileWatch.onWatch {
+	// remove all files from watch list
-		s.errors = append(s.errors, WatchNotRunningError)
+	for _, f := range s.files.files {
-		return s
+		d := path.Dir(f.path)
+		err := s.fileWatch.Remove(d)
+		if err != nil {
+			s.errors = append(s.errors, err)
+		}
 	}
-	s.fileWatch.cancelWatch <- true
-	return s
-}
-func (s *Settings[C]) buildWatchList() *Settings[C] {
-	s.fileWatch.Lock()
-	defer s.fileWatch.Unlock()
-	s.fileWatch.watchList = make(map[string]bool)
 	for _, d := range s.files.directories {
-		fn := path.Join(d, s.files.name+s.files.format.Extension())
+		err := s.fileWatch.Remove(d)
-		s.fileWatch.watchList[fn] = true
+		if err != nil {
-	}
+			s.errors = append(s.errors, err)
+		}
-	for _, f := range s.files.files {
-		s.fileWatch.watchList[f.path] = true
 	}
 	return s
@@ -78,99 +40,52 @@ func (s *Settings[C]) buildWatchList() *Settings[C] {
 // Watch the given file for changes
 func (s *Settings[C]) Watch() *Settings[C] {
-	s.buildWatchList()
+	if s.fileWatch == nil {
+		s.fileWatch = watch.NewLighthouse()
-	s.fileWatch.Lock()
-	defer s.fileWatch.Unlock()
-	defer func() {
-		s.notifyErrorHooks()
-	}()
-	if s.fileWatch.watcher == nil {
-		s.initWatch()
-	}
-	if s.fileWatch.watchList == nil {
-		s.errors = append(s.errors, WatchListNotInitializedError)
-		return s
 	}
-	if s.fileWatch.onWatch == true {
+	err := s.fileWatch.StartWatching()
-		s.errors = append(s.errors, WatchAlreadyRunningError)
+	if err != nil {
+		s.errors = append(s.errors, err)
 		return s
 	}
-	s.fileWatch.onWatch = true
+	for _, d := range s.files.directories {
-	s.fileWatch.cancelWatch = make(chan bool)
-	// remove all files from the watch list
-	for _, file := range s.fileWatch.watcher.WatchList() {
-		s.fileWatch.watcher.Remove(file)
-	}
-	// add all files to the watch list
-	for filePath := range s.fileWatch.watchList {
-		fileInfo, err := os.Stat(filePath)
-		if err != nil {
-			s.errors = append(s.errors, err)
-			continue
-		}
-		if fileInfo.IsDir() {
+		w := &watch.Watch{
-			err = s.fileWatch.watcher.Add(filePath)
+			Path: d,
-		} else {
+			OnChange: func(x string) {
-			err = s.fileWatch.watcher.Add(path.Dir(filePath))
+				if x == d {
+					s.Import()
+				}
+			},
 		}
+		err := s.fileWatch.Add(w)
 		if err != nil {
 			s.errors = append(s.errors, err)
 		}
 	}
-	if len(s.fileWatch.watcher.WatchList()) == 0 {
+	for _, f := range s.files.files {
-		s.errors = append(s.errors, NoFilesToWatchError)
-		s.Unlock()
-		return s
-	}
-	go func() {
-	finished:
-		for {
-			select {
-			case event, ok := <-s.fileWatch.watcher.Events:
-				if !ok {
-					return
-				}
-				_, exist := s.fileWatch.watchList[event.Name]
+		p := path.Dir(f.path)
-				if !exist {
-					continue
-				}
-				if event.Op&fsnotify.Write == fsnotify.Write {
+		w := &watch.Watch{
-					s.Import()
+			Path: p,
-				} else if event.Op&fsnotify.Remove == fsnotify.Remove {
+			OnChange: func(x string) {
-					s.Import()
+				if x == f.path {
-				} else if event.Op&fsnotify.Rename == fsnotify.Rename {
 					s.Import()
 				}
-			case err, ok := <-s.fileWatch.watcher.Errors:
+			},
-				if !ok {
-					return
-				}
-				s.errors = append(s.errors, err)
-			case <-s.fileWatch.cancelWatch:
-				break finished
-			}
 		}
-		s.fileWatch.onWatch = false
+		err := s.fileWatch.Add(w)
-	}()
+		if err != nil {
+			s.errors = append(s.errors, err)
+		}
+	}
 	return s

--- a/watch_test.go
+++ b/watch_test.go
@@ -6,6 +6,7 @@ package configuration
 import (
 	"github.com/stretchr/testify/assert"
 	"os"
+	"sync"
 	"testing"
 	"time"
 )
@@ -35,11 +36,10 @@ func createTestFileForWatch1() (string, error) {
 		return "", err
 	}
-	f.WriteString("Host: \"127.0.0.1\"")
+	_, _ = f.WriteString("Host: \"127.0.0.1\"")
-	f.Close()
+	_ = f.Close()
 	return f.Name(), nil
 }
 type testHostTimeout struct {
@@ -55,7 +55,12 @@ func TestMultiChange(t *testing.T) {
 		return
 	}
-	defer os.Remove(tmpFn)
+	defer func() {
+		err := os.Remove(tmpFn)
+		if err != nil {
+			t.Error(err)
+		}
+	}()
 	config := struct {
 		Host string `yaml:"Host"`
@@ -67,13 +72,16 @@ func TestMultiChange(t *testing.T) {
 	c.SetMnemonic("my-app")
 	assert.Equal(t, c.Config().Host, "localhost")
+	var mu sync.Mutex
 	result := []string{}
 	signal := make(chan string)
 	var h ChangeHook
 	h = &ChangeEventHandler{
 		Callback: func(event ChangeEvent) {
+			mu.Lock()
 			result = append(result, event.Changelog[0].To.(string))
+			mu.Unlock()
 			signal <- event.Changelog[0].To.(string)
 		},
 	}
@@ -96,25 +104,27 @@ func TestMultiChange(t *testing.T) {
 	data := []testHostTimeout{
+		// important to have a timeout > 500ms, because the decoupled file watcher will only trigger every 500ms
 		{
 			host:    "1.org",
-			timeout: time.Millisecond * 100,
+			timeout: time.Millisecond * 550,
 		},
 		{
 			host:    "2.org",
-			timeout: time.Millisecond * 10,
+			timeout: time.Millisecond * 610,
 		},
 		{
 			host:    "3.org",
-			timeout: time.Millisecond * 2,
+			timeout: time.Millisecond * 602,
 		},
 		{
 			host:    "4.org",
-			timeout: time.Millisecond * 100,
+			timeout: time.Millisecond * 700,
 		},
 		{
 			host:    "9.org",
-			timeout: time.Millisecond * 100,
+			timeout: time.Millisecond * 700,
 		},
 	}
@@ -122,14 +132,18 @@ func TestMultiChange(t *testing.T) {
 	go runTestFilesChange(tmpFn, data, t)
-	for loop := true; loop; {
+	mu.Lock()
+	length := len(result)
+	mu.Unlock()
+	for length < len(data) {
 		select {
 		case <-signal:
-			if len(result) == len(data) {
+			mu.Lock()
-				loop = false
+			length = len(result)
-				break
+			mu.Unlock()
-			}
+			continue
-		case <-time.After(time.Second * 10):
+		case <-time.After(time.Second * 20):
 			t.Log(result)
 			t.Fatalf("Timeout")
 		}
@@ -201,7 +215,10 @@ func TestSettingStopWatching(t *testing.T) {
 		return
 	}
-	defer os.Remove(f.Name())
+	defer func() {
+		_ = os.Remove(f.Name())
+		_ = f.Close()
+	}()
 	config := struct {
 		Host string `yaml:"Host"`
@@ -236,8 +253,8 @@ func TestSettingStopWatchingNotOnWatch(t *testing.T) {
 	c := New(config)
 	c.StopWatching()
-	if !c.HasErrors() {
+	if c.HasErrors() {
-		t.Error("Expected to have an error")
+		t.Error("Expected to have no error")
 	}
 }
@@ -250,7 +267,10 @@ func TestSettingStopWatchingTwice(t *testing.T) {
 		return
 	}
-	defer os.Remove(f.Name())
+	defer func() {
+		_ = os.Remove(f.Name())
+		_ = f.Close()
+	}()
 	config := struct {
 		Host string `yaml:"Host"`
@@ -258,7 +278,7 @@ func TestSettingStopWatchingTwice(t *testing.T) {
 		Host: "localhost",
 	}
-	f.WriteString("Host: example.org")
+	_, _ = f.WriteString("Host: example.org")
 	c := New(config)
 	c.AddFile(f.Name(), Yaml)
@@ -273,8 +293,7 @@ func TestSettingStopWatchingTwice(t *testing.T) {
 		t.Error("Expected to have an error")
 	}
-	if e[0] != WatchNotRunningError {
+	errText := e[0].Error()
-		t.Error("Expected to have an error")
+	assert.Equal(t, errText, "Watcher is not active")
-	}
 }