From 3eb3589186ad5b3bd473a48740669b4ba437854b Mon Sep 17 00:00:00 2001
From: Volker Schukai <volker.schukai@schukai.com>
Date: Sun, 10 Sep 2023 20:07:09 +0200
Subject: [PATCH] feat: new watcher lib #8
---
Taskfile.yml | 3 +
go.mod | 9 +--
go.sum | 12 ++++
import.go | 25 +-------
settings.go | 13 +---
watch.go | 173 +++++++++++++-------------------------------------
watch_test.go | 65 ++++++++++++-------
7 files changed, 109 insertions(+), 191 deletions(-)
diff --git a/Taskfile.yml b/Taskfile.yml
index 3465c69..8fd6c58 100644
--- a/Taskfile.yml
+++ b/Taskfile.yml
@@ -7,11 +7,14 @@ tasks:
cmds:
- task --list-all
silent: true
+
test:
desc: Execute unit tests in Go.
cmds:
- echo "Execute unit tests in Go."
- go test -cover -v ./...
+ - go test -bench .
+ - go test -race .
test-fuzz:
desc: Conduct fuzzing tests.#
diff --git a/go.mod b/go.mod
index a4e546e..fe1395c 100644
--- a/go.mod
+++ b/go.mod
@@ -7,13 +7,14 @@ require (
github.com/imdario/mergo v0.3.16
github.com/kinbiko/jsonassert v1.1.1
github.com/magiconair/properties v1.8.7
- github.com/pelletier/go-toml/v2 v2.0.9
+ github.com/pelletier/go-toml/v2 v2.1.0
github.com/r3labs/diff/v3 v3.0.1
github.com/stretchr/testify v1.8.4
gitlab.schukai.com/oss/libraries/go/application/xflags v1.9.0
- gitlab.schukai.com/oss/libraries/go/network/http-negotiation v1.3.0
+ gitlab.schukai.com/oss/libraries/go/network/http-negotiation v1.3.1
gitlab.schukai.com/oss/libraries/go/utilities/pathfinder v0.5.2
- golang.org/x/exp v0.0.0-20230811145659-89c5cff77bcb
+ gitlab.schukai.com/oss/libraries/go/utilities/watch v0.2.0
+ golang.org/x/exp v0.0.0-20230905200255-921286631fa9
gopkg.in/yaml.v3 v3.0.1
)
@@ -26,7 +27,7 @@ require (
github.com/vmihailenco/msgpack/v5 v5.3.5 // indirect
github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
golang.org/x/net v0.8.0 // indirect
- golang.org/x/sys v0.11.0 // indirect
+ golang.org/x/sys v0.12.0 // indirect
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/protobuf v1.29.0 // indirect
)
diff --git a/go.sum b/go.sum
index 2bcbd7a..be8d433 100644
--- a/go.sum
+++ b/go.sum
@@ -27,6 +27,8 @@ github.com/pelletier/go-toml/v2 v2.0.7 h1:muncTPStnKRos5dpVKULv2FVd4bMOhNePj9Cjg
github.com/pelletier/go-toml/v2 v2.0.7/go.mod h1:eumQOmlWiOPt5WriQQqoM5y18pDHwha2N+QD+EUNTek=
github.com/pelletier/go-toml/v2 v2.0.9 h1:uH2qQXheeefCCkuBBSLi7jCiSmj3VRh2+Goq2N7Xxu0=
github.com/pelletier/go-toml/v2 v2.0.9/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc=
+github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4=
+github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/r3labs/diff/v3 v3.0.0 h1:ZhPwNxn9gW5WLPBV9GCYaVbMdLOSmJ0DeKdCiSbOLUI=
@@ -56,12 +58,18 @@ gitlab.schukai.com/oss/libraries/go/application/xflags v1.9.0 h1:bSnwEV56JZQWBQC
gitlab.schukai.com/oss/libraries/go/application/xflags v1.9.0/go.mod h1:KN99uofMnTNcpfKwPbskucCTgwivJa3jfP2BHM4Ac+A=
gitlab.schukai.com/oss/libraries/go/network/http-negotiation v1.3.0 h1:SZG0BW5ll3WK5ZIOTogjqX8oVHCTxANTDLPxUs7Rnx8=
gitlab.schukai.com/oss/libraries/go/network/http-negotiation v1.3.0/go.mod h1:RS2rKf5O+rmSBshHLOgjG7dxg5N2MhNYokZOBcuXdX8=
+gitlab.schukai.com/oss/libraries/go/network/http-negotiation v1.3.1 h1:B6BZV3bURUew5u+L/QLaBjdqTlW7P3dHTO19QLkPSfI=
+gitlab.schukai.com/oss/libraries/go/network/http-negotiation v1.3.1/go.mod h1:RS2rKf5O+rmSBshHLOgjG7dxg5N2MhNYokZOBcuXdX8=
gitlab.schukai.com/oss/libraries/go/utilities/pathfinder v0.3.0 h1:mSxk2q/npskmHMmw1oF4moccjGav5dL6qmff2njUV7A=
gitlab.schukai.com/oss/libraries/go/utilities/pathfinder v0.3.0/go.mod h1:UvdD4NAf3gLKYafabJD7e9ZCOetzM9JZ9y4GkZukPVU=
gitlab.schukai.com/oss/libraries/go/utilities/pathfinder v0.3.1 h1:oyElaqEiyr2XgaE1CYwD8LoeHsuR/vQD/p6k3jYbJFs=
gitlab.schukai.com/oss/libraries/go/utilities/pathfinder v0.3.1/go.mod h1:UvdD4NAf3gLKYafabJD7e9ZCOetzM9JZ9y4GkZukPVU=
gitlab.schukai.com/oss/libraries/go/utilities/pathfinder v0.5.2 h1:R+dL2NJCM+AQNPK4DPDmfvx1eomi1Xb1dl0XKEFj7Ek=
gitlab.schukai.com/oss/libraries/go/utilities/pathfinder v0.5.2/go.mod h1:UvdD4NAf3gLKYafabJD7e9ZCOetzM9JZ9y4GkZukPVU=
+gitlab.schukai.com/oss/libraries/go/utilities/watch v0.1.0 h1:FAKHmf9p3NKyzuM0cIXYBxmhdQ7zJ+6wj5qqeoIMbGc=
+gitlab.schukai.com/oss/libraries/go/utilities/watch v0.1.0/go.mod h1:tMFl68peRKHgFQLltrTN3JLredofMqvGi3C0SEAj73Y=
+gitlab.schukai.com/oss/libraries/go/utilities/watch v0.2.0 h1:tLjN9Wyv+LJhtiiQDzdzaDelEq2LVCDP3Ndo7ZPIWfQ=
+gitlab.schukai.com/oss/libraries/go/utilities/watch v0.2.0/go.mod h1:tMFl68peRKHgFQLltrTN3JLredofMqvGi3C0SEAj73Y=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/exp v0.0.0-20221012211006-4de253d81b95 h1:sBdrWpxhGDdTAYNqbgBLAR+ULAPPhfgncLr1X0lyWtg=
golang.org/x/exp v0.0.0-20221012211006-4de253d81b95/go.mod h1:cyybsKvd6eL0RnXn6p/Grxp8F5bW7iYuBgsNCOHpMYE=
@@ -75,6 +83,8 @@ golang.org/x/exp v0.0.0-20230810033253-352e893a4cad h1:g0bG7Z4uG+OgH2QDODnjp6ggk
golang.org/x/exp v0.0.0-20230810033253-352e893a4cad/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc=
golang.org/x/exp v0.0.0-20230811145659-89c5cff77bcb h1:mIKbk8weKhSeLH2GmUTrvx8CjkyJmnU1wFmg59CUjFA=
golang.org/x/exp v0.0.0-20230811145659-89c5cff77bcb/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc=
+golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjsbSXD66ic0XW0js0R9g=
+golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k=
golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
golang.org/x/net v0.0.0-20221014081412-f15817d10f9b h1:tvrvnPFcdzp294diPnrdZZZ8XUt2Tyj7svb7X52iDuU=
golang.org/x/net v0.0.0-20221014081412-f15817d10f9b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=
@@ -92,6 +102,8 @@ golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM=
golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o=
+golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
diff --git a/import.go b/import.go
index 2a83855..8fddd9f 100644
--- a/import.go
+++ b/import.go
@@ -172,32 +172,9 @@ func handleField(p string, r reflect.Value) {
}
}
-//func forEachElem(r reflect.Value, fn func(e reflect.Value)) {
-// switch r.Kind() {
-// case reflect.Slice:
-// for i := 0; i < r.Len(); i++ {
-// fn(r.Index(i))
-// }
-// case reflect.Map:
-// for _, k := range r.MapKeys() {
-// fn(r.MapIndex(k))
-// }
-// case reflect.Ptr, reflect.Interface:
-// if !r.IsNil() {
-// fn(r.Elem())
-// }
-// }
-//}
-
func (s *Settings[C]) importFiles() {
- s.fileWatch.Lock()
-
- defer func() {
- s.fileWatch.Unlock()
- }()
-
- defer func() {
+ defer func() {
s.notifyErrorHooks()
}()
diff --git a/settings.go b/settings.go
index 2014632..55288ec 100644
--- a/settings.go
+++ b/settings.go
@@ -4,21 +4,12 @@
package configuration
import (
+ "gitlab.schukai.com/oss/libraries/go/utilities/watch"
"reflect"
"strconv"
"sync"
-
- "github.com/fsnotify/fsnotify"
)
-type fileWatch struct {
- sync.Mutex
- watcher *fsnotify.Watcher
- watchList map[string]bool
- cancelWatch chan bool
- onWatch bool
-}
-
// Settings is the main struct for the configuration
type Settings[C any] struct {
files fileBackend
@@ -34,7 +25,7 @@ type Settings[C any] struct {
postprocessing []PostprocessingHook
}
- fileWatch fileWatch
+ fileWatch watch.Lighthouse
}
func (s *Settings[C]) initDefaults() *Settings[C] {
diff --git a/watch.go b/watch.go
index 5b90b6e..2dd96a2 100644
--- a/watch.go
+++ b/watch.go
@@ -4,72 +4,34 @@
package configuration
import (
- "github.com/fsnotify/fsnotify"
- "os"
+ "gitlab.schukai.com/oss/libraries/go/utilities/watch"
"path"
)
-func (s *Settings[C]) initWatch() *Settings[C] {
-
- var err error
-
- defer func() {
- s.notifyErrorHooks()
- }()
-
- if s.fileWatch.watcher != nil {
- s.errors = append(s.errors, WatchAlreadyInitializedError)
- return s
- }
-
- s.fileWatch.watcher, err = fsnotify.NewWatcher()
- if err != nil {
- s.errors = append(s.errors, err)
- return s
- }
-
- return s
-
-}
-
func (s *Settings[C]) StopWatching() *Settings[C] {
- s.fileWatch.Lock()
- defer s.fileWatch.Unlock()
-
- defer func() {
- s.notifyErrorHooks()
- }()
-
- if s.fileWatch.watcher == nil {
- s.errors = append(s.errors, WatchNotInitializedError)
- return s
+ if s.fileWatch != nil {
+ err := s.fileWatch.StopWatching()
+ if err != nil {
+ s.errors = append(s.errors, err)
+ return s
+ }
}
- if !s.fileWatch.onWatch {
- s.errors = append(s.errors, WatchNotRunningError)
- return s
+ // remove all files from watch list
+ for _, f := range s.files.files {
+ d := path.Dir(f.path)
+ err := s.fileWatch.Remove(d)
+ if err != nil {
+ s.errors = append(s.errors, err)
+ }
}
- s.fileWatch.cancelWatch <- true
-
- return s
-}
-
-func (s *Settings[C]) buildWatchList() *Settings[C] {
-
- s.fileWatch.Lock()
- defer s.fileWatch.Unlock()
-
- s.fileWatch.watchList = make(map[string]bool)
-
for _, d := range s.files.directories {
- fn := path.Join(d, s.files.name+s.files.format.Extension())
- s.fileWatch.watchList[fn] = true
- }
-
- for _, f := range s.files.files {
- s.fileWatch.watchList[f.path] = true
+ err := s.fileWatch.Remove(d)
+ if err != nil {
+ s.errors = append(s.errors, err)
+ }
}
return s
@@ -78,99 +40,52 @@ func (s *Settings[C]) buildWatchList() *Settings[C] {
// Watch the given file for changes
func (s *Settings[C]) Watch() *Settings[C] {
- s.buildWatchList()
-
- s.fileWatch.Lock()
- defer s.fileWatch.Unlock()
-
- defer func() {
- s.notifyErrorHooks()
- }()
-
- if s.fileWatch.watcher == nil {
- s.initWatch()
- }
-
- if s.fileWatch.watchList == nil {
- s.errors = append(s.errors, WatchListNotInitializedError)
- return s
+ if s.fileWatch == nil {
+ s.fileWatch = watch.NewLighthouse()
}
- if s.fileWatch.onWatch == true {
- s.errors = append(s.errors, WatchAlreadyRunningError)
+ err := s.fileWatch.StartWatching()
+ if err != nil {
+ s.errors = append(s.errors, err)
return s
}
- s.fileWatch.onWatch = true
- s.fileWatch.cancelWatch = make(chan bool)
-
- // remove all files from the watch list
- for _, file := range s.fileWatch.watcher.WatchList() {
- s.fileWatch.watcher.Remove(file)
- }
-
- // add all files to the watch list
- for filePath := range s.fileWatch.watchList {
-
- fileInfo, err := os.Stat(filePath)
- if err != nil {
- s.errors = append(s.errors, err)
- continue
- }
+ for _, d := range s.files.directories {
- if fileInfo.IsDir() {
- err = s.fileWatch.watcher.Add(filePath)
- } else {
- err = s.fileWatch.watcher.Add(path.Dir(filePath))
+ w := &watch.Watch{
+ Path: d,
+ OnChange: func(x string) {
+ if x == d {
+ s.Import()
+ }
+ },
}
+ err := s.fileWatch.Add(w)
if err != nil {
s.errors = append(s.errors, err)
}
}
- if len(s.fileWatch.watcher.WatchList()) == 0 {
- s.errors = append(s.errors, NoFilesToWatchError)
- s.Unlock()
- return s
- }
-
- go func() {
- finished:
-
- for {
- select {
- case event, ok := <-s.fileWatch.watcher.Events:
- if !ok {
- return
- }
+ for _, f := range s.files.files {
- _, exist := s.fileWatch.watchList[event.Name]
- if !exist {
- continue
- }
+ p := path.Dir(f.path)
- if event.Op&fsnotify.Write == fsnotify.Write {
- s.Import()
- } else if event.Op&fsnotify.Remove == fsnotify.Remove {
- s.Import()
- } else if event.Op&fsnotify.Rename == fsnotify.Rename {
+ w := &watch.Watch{
+ Path: p,
+ OnChange: func(x string) {
+ if x == f.path {
s.Import()
}
- case err, ok := <-s.fileWatch.watcher.Errors:
- if !ok {
- return
- }
- s.errors = append(s.errors, err)
-
- case <-s.fileWatch.cancelWatch:
- break finished
- }
+ },
}
- s.fileWatch.onWatch = false
- }()
+ err := s.fileWatch.Add(w)
+ if err != nil {
+ s.errors = append(s.errors, err)
+ }
+ }
return s
diff --git a/watch_test.go b/watch_test.go
index 91e4bff..9c7a2bd 100644
--- a/watch_test.go
+++ b/watch_test.go
@@ -6,6 +6,7 @@ package configuration
import (
"github.com/stretchr/testify/assert"
"os"
+ "sync"
"testing"
"time"
)
@@ -35,11 +36,10 @@ func createTestFileForWatch1() (string, error) {
return "", err
}
- f.WriteString("Host: \"127.0.0.1\"")
- f.Close()
+ _, _ = f.WriteString("Host: \"127.0.0.1\"")
+ _ = f.Close()
return f.Name(), nil
-
}
type testHostTimeout struct {
@@ -55,7 +55,12 @@ func TestMultiChange(t *testing.T) {
return
}
- defer os.Remove(tmpFn)
+ defer func() {
+ err := os.Remove(tmpFn)
+ if err != nil {
+ t.Error(err)
+ }
+ }()
config := struct {
Host string `yaml:"Host"`
@@ -67,13 +72,16 @@ func TestMultiChange(t *testing.T) {
c.SetMnemonic("my-app")
assert.Equal(t, c.Config().Host, "localhost")
+ var mu sync.Mutex
result := []string{}
signal := make(chan string)
var h ChangeHook
h = &ChangeEventHandler{
Callback: func(event ChangeEvent) {
+ mu.Lock()
result = append(result, event.Changelog[0].To.(string))
+ mu.Unlock()
signal <- event.Changelog[0].To.(string)
},
}
@@ -96,25 +104,27 @@ func TestMultiChange(t *testing.T) {
data := []testHostTimeout{
+ // important to have a timeout > 500ms, because the decoupled file watcher will only trigger every 500ms
+
{
host: "1.org",
- timeout: time.Millisecond * 100,
+ timeout: time.Millisecond * 550,
},
{
host: "2.org",
- timeout: time.Millisecond * 10,
+ timeout: time.Millisecond * 610,
},
{
host: "3.org",
- timeout: time.Millisecond * 2,
+ timeout: time.Millisecond * 602,
},
{
host: "4.org",
- timeout: time.Millisecond * 100,
+ timeout: time.Millisecond * 700,
},
{
host: "9.org",
- timeout: time.Millisecond * 100,
+ timeout: time.Millisecond * 700,
},
}
@@ -122,14 +132,18 @@ func TestMultiChange(t *testing.T) {
go runTestFilesChange(tmpFn, data, t)
- for loop := true; loop; {
+ mu.Lock()
+ length := len(result)
+ mu.Unlock()
+
+ for length < len(data) {
select {
case <-signal:
- if len(result) == len(data) {
- loop = false
- break
- }
- case <-time.After(time.Second * 10):
+ mu.Lock()
+ length = len(result)
+ mu.Unlock()
+ continue
+ case <-time.After(time.Second * 20):
t.Log(result)
t.Fatalf("Timeout")
}
@@ -201,7 +215,10 @@ func TestSettingStopWatching(t *testing.T) {
return
}
- defer os.Remove(f.Name())
+ defer func() {
+ _ = os.Remove(f.Name())
+ _ = f.Close()
+ }()
config := struct {
Host string `yaml:"Host"`
@@ -236,8 +253,8 @@ func TestSettingStopWatchingNotOnWatch(t *testing.T) {
c := New(config)
c.StopWatching()
- if !c.HasErrors() {
- t.Error("Expected to have an error")
+ if c.HasErrors() {
+ t.Error("Expected to have no error")
}
}
@@ -250,7 +267,10 @@ func TestSettingStopWatchingTwice(t *testing.T) {
return
}
- defer os.Remove(f.Name())
+ defer func() {
+ _ = os.Remove(f.Name())
+ _ = f.Close()
+ }()
config := struct {
Host string `yaml:"Host"`
@@ -258,7 +278,7 @@ func TestSettingStopWatchingTwice(t *testing.T) {
Host: "localhost",
}
- f.WriteString("Host: example.org")
+ _, _ = f.WriteString("Host: example.org")
c := New(config)
c.AddFile(f.Name(), Yaml)
@@ -273,8 +293,7 @@ func TestSettingStopWatchingTwice(t *testing.T) {
t.Error("Expected to have an error")
}
- if e[0] != WatchNotRunningError {
- t.Error("Expected to have an error")
- }
+ errText := e[0].Error()
+ assert.Equal(t, errText, "Watcher is not active")
}
--
GitLab