Skip to content

Commit 49baf22

Browse files
lsierantlsierant
committed
Observability tool
1 parent 917723b commit 49baf22

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

65 files changed

+5346
-0
lines changed

‎.gitignore‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -84,6 +84,8 @@ docker/mongodb-kubernetes-tests/.test_identifiers*
8484

8585
logs-debug/
8686
/ssdlc-report/*
87+
tools/mdbdebug/bin*
88+
tools/diffwatch/bin*
8789
.gocache/
8890

8991
docs/**/log/*

‎controllers/operator/mongodbshardedcluster_controller.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2882,6 +2882,10 @@ func (r *ShardedClusterReconcileHelper) statefulsetLabels() map[string]string {
28822882
return merge.StringToStringMap(r.sc.Labels, r.sc.GetOwnerLabels())
28832883
}
28842884

2885+
func (r *ShardedClusterReconcileHelper) DesiredShardsConfiguration() map[int]*mdbv1.ShardedClusterComponentSpec {
2886+
return r.desiredShardsConfiguration
2887+
}
2888+
28852889
func (r *ShardedClusterReconcileHelper) ShardsMemberClustersMap() map[int][]multicluster.MemberCluster {
28862890
return r.shardsMemberClustersMap
28872891
}

‎tools/diffwatch/Dockerfile‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
FROM alpine:latest
2+
3+
RUN apk add --update --no-cache python3 py3-pip && ln -sf python3 /usr/bin/python
4+
RUN apk add bash tmux kubectl htop less fzf yq lnav jq curl
5+
RUN pip install tmuxp --break-system-packages
6+
7+
# Create directory for lnav formats
8+
RUN mkdir -p /root/.lnav/formats/installed/
9+
10+
COPY bin_linux/diffwatch /usr/local/bin/
11+
COPY lnav/*.json /root/.lnav/formats/installed/
12+
ADD retry_cmd.sh /usr/local/bin/
13+
RUN chmod +x /usr/local/bin/retry_cmd.sh
14+
15+
CMD ["/bin/bash"]

‎tools/diffwatch/build.sh‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
#!/usr/bin/env bash
2+
3+
set -Eeou pipefail
4+
5+
script_name=$(readlink -f "${BASH_SOURCE[0]}")
6+
script_dir=$(dirname "${script_name}")
7+
8+
pushd "${script_dir}" >/dev/null 2>&1
9+
mkdir bin bin_linux >/dev/null 2>&1 || true
10+
11+
echo "Building diffwatch from $(pwd) directory"
12+
GOOS=linux GOARCH=amd64 go build -o bin_linux ./...
13+
go build -o bin ./...
14+
15+
echo "Copying diffwatch from $(pwd) to ${PROJECT_DIR}/bin"
16+
cp bin/diffwatch "${PROJECT_DIR}"/bin

‎tools/diffwatch/build_docker.sh‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
#!/usr/bin/env bash
2+
3+
set -Eeou pipefail
4+
5+
export TAG=${TAG:-"latest"}
6+
7+
docker build --platform linux/amd64 -t "quay.io/lsierant/diffwatch:${TAG}" .
8+
docker push "quay.io/lsierant/diffwatch:${TAG}"

‎tools/diffwatch/cmd/diffwatch/diffwatch.go‎

Lines changed: 105 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,105 @@
1+
package main
2+
3+
import (
4+
"context"
5+
"flag"
6+
"fmt"
7+
"io"
8+
"log"
9+
"os"
10+
"os/signal"
11+
"path"
12+
"strings"
13+
"syscall"
14+
15+
"github.com/mongodb/mongodb-kubernetes/diffwatch/pkg/diffwatch"
16+
)
17+
18+
type arrayFlags []string
19+
20+
func (i *arrayFlags) String() string {
21+
return strings.Join(*i, ",")
22+
}
23+
24+
func (i *arrayFlags) Set(value string) error {
25+
*i = append(*i, value)
26+
return nil
27+
}
28+
29+
func main() {
30+
ctx, cancel := context.WithCancel(context.Background())
31+
32+
signalChan := make(chan os.Signal, 1)
33+
signal.Notify(signalChan, syscall.SIGINT, syscall.SIGTERM)
34+
35+
go func() {
36+
<-signalChan
37+
cancel()
38+
}()
39+
40+
readFromStdin := isInPipeMode()
41+
var inputStream io.Reader
42+
if readFromStdin {
43+
inputStream = os.Stdin
44+
}
45+
46+
var filePath string
47+
var destDir string
48+
var linesAfter int
49+
var linesBefore int
50+
var linesContext int
51+
var ignores arrayFlags
52+
flag.StringVar(&filePath, "file", "", "Path to the JSON file that will be periodically observed for changes. Optional when the content is piped on stdin. Required when -destDir is specified. "+
53+
"If reading from stdin, then path is not relevant (file won't be read), but the file name will be used for the diff files prefix stored in destDir.")
54+
flag.StringVar(&destDir, "destDir", "", "Path to the destination directory to store diffs. Optional. If not set, then diff files won't be created. "+
55+
"If specified, then -file parameter is required. The files will be prefixed with file name of the -file parameter.")
56+
flag.IntVar(&linesAfter, "A", 0, "Number of lines printed after a match (default 0)")
57+
flag.IntVar(&linesBefore, "B", 0, "Number of lines printed before a match (default 0)")
58+
flag.IntVar(&linesContext, "C", 3, "Number of context lines printed before and after (equivalent to setting -A and -B) (default = 3)")
59+
flag.Var(&ignores, "ignore", "Regex pattern to ignore triggering diff if the only changes are ignored ones; you can specify multiple --ignore parameters, e.g. --ignore timestamp --ignore '\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}\\.\\d{3}Z' (ignore all lines with changed timestamp)")
60+
flag.Parse()
61+
62+
for ignore := range ignores {
63+
fmt.Println("ignore = ", ignore)
64+
}
65+
66+
if linesBefore == 0 {
67+
linesBefore = linesContext
68+
}
69+
if linesAfter == 0 {
70+
linesAfter = linesContext
71+
}
72+
73+
if err := watchChanges(ctx, filePath, destDir, inputStream, linesBefore, linesAfter, ignores); err != nil {
74+
cancel()
75+
if err == io.EOF {
76+
log.Printf("Reached end of stream. Exiting.")
77+
} else {
78+
log.Printf("Error: %v", err)
79+
}
80+
os.Exit(1)
81+
}
82+
}
83+
84+
func isInPipeMode() bool {
85+
stat, _ := os.Stdin.Stat()
86+
return (stat.Mode() & os.ModeCharDevice) == 0
87+
}
88+
89+
func watchChanges(ctx context.Context, filePath string, destDir string, inputStream io.Reader, linesBefore int, linesAfter int, ignores []string) error {
90+
diffWriterFunc := diffwatch.WriteDiffFiles(destDir, path.Base(filePath))
91+
jsonDiffer, err := diffwatch.NewJsonDiffer(linesBefore, linesAfter, diffWriterFunc, ignores)
92+
if err != nil {
93+
return err
94+
}
95+
96+
// parsedFileChannel is filled in the background by reading from stream or watching the file periodically
97+
parsedFileChannel := make(chan diffwatch.ParsedFileWrapper)
98+
if inputStream != nil {
99+
go diffwatch.ReadAndParseFromStream(ctx, inputStream, filePath, parsedFileChannel)
100+
} else {
101+
go diffwatch.ReadAndParseFilePeriodically(ctx, filePath, diffwatch.DefaultWatchInterval, parsedFileChannel)
102+
}
103+
104+
return diffwatch.WatchFileChangesPeriodically(ctx, filePath, parsedFileChannel, jsonDiffer.FileChangedHandler)
105+
}

‎tools/diffwatch/cmd/diffwatch/diffwatch_int_test.go‎

Lines changed: 120 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,120 @@
1+
package main
2+
3+
import (
4+
"bytes"
5+
"context"
6+
"fmt"
7+
"io"
8+
"os"
9+
"testing"
10+
"time"
11+
12+
"github.com/mongodb/mongodb-kubernetes/diffwatch/pkg/diffwatch"
13+
"github.com/stretchr/testify/assert"
14+
"github.com/stretchr/testify/require"
15+
)
16+
17+
// ignored project-root/tmp dir where test outputs will be stored
18+
const tmpDir = "../../../../tmp/diffwatch"
19+
const cleanupAfterTest = false
20+
21+
// TestDiffWatcherFromFile is a manual test that triggers simulated sequence of changes.
22+
// Intended for manual inspection of files.
23+
//
24+
// How to run:
25+
// 1. Create ops-manager-kubernetes/tmp directory
26+
// 2. Comment t.Skip and run the test
27+
// 3. View latest files:
28+
// find $(find tmp/diffwatch -d 1 -type d | sort -n | tail -n 1) -type f | sort -rV | fzf --preview 'cat {}'
29+
func TestDiffWatcherFromFile(t *testing.T) {
30+
t.Skip("Test intended to manual run, comment skip to run")
31+
ctx, cancel := context.WithCancel(context.Background())
32+
defer cancel()
33+
34+
require.NoError(t, os.MkdirAll(tmpDir, 0770))
35+
tempDir, err := os.MkdirTemp(tmpDir, time.Now().Format("20060102_150405"))
36+
require.NoError(t, err)
37+
defer func() {
38+
if cleanupAfterTest {
39+
_ = os.RemoveAll(tempDir)
40+
}
41+
}()
42+
43+
watchedFile := fmt.Sprintf("%s/watched.json", tempDir)
44+
go watchChanges(ctx, watchedFile, tempDir, nil, 4, 4, []string{"ignoredField", `\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}`})
45+
46+
diffwatch.DefaultWatchInterval = time.Millisecond * 100
47+
diffwatch.DefaultProgressInterval = time.Millisecond * 200
48+
applyChange(t, "resources/base.json", watchedFile)
49+
applyChange(t, "resources/changed_1.json", watchedFile)
50+
applyChange(t, "resources/changed_2.json", watchedFile)
51+
time.Sleep(diffwatch.DefaultProgressInterval * 2)
52+
applyChange(t, "resources/changed_3.json", watchedFile)
53+
applyChange(t, "resources/changed_3_ignored_only.json", watchedFile)
54+
applyChange(t, "resources/changed_3_ignored_only_2.json", watchedFile)
55+
applyChange(t, "resources/changed_4_ignored_only_ts.json", watchedFile)
56+
applyChange(t, "resources/changed_4_ignored_only_ts_and_other.json", watchedFile)
57+
applyChange(t, "resources/changed_5.json", watchedFile)
58+
}
59+
60+
func TestDiffWatcherFromStream(t *testing.T) {
61+
t.Skip("Test intended to manual run, comment skip to run")
62+
ctx, cancel := context.WithCancel(context.Background())
63+
defer cancel()
64+
65+
buf := bytes.Buffer{}
66+
files := []string{
67+
"resources/base.json",
68+
"resources/changed_1.json",
69+
"resources/changed_2.json",
70+
"resources/changed_3.json",
71+
"resources/changed_4_ignored_only_ts.json",
72+
"resources/changed_4_ignored_only_ts_and_other.json",
73+
"resources/changed_5.json",
74+
}
75+
for _, file := range files {
76+
fileBytes, err := os.ReadFile(file)
77+
require.NoError(t, err)
78+
buf.Write(fileBytes)
79+
}
80+
81+
require.NoError(t, os.MkdirAll(tmpDir, 0770))
82+
tempDir, err := os.MkdirTemp(tmpDir, time.Now().Format("20060102_150405"))
83+
require.NoError(t, err)
84+
defer func() {
85+
if cleanupAfterTest {
86+
_ = os.RemoveAll(tempDir)
87+
}
88+
}()
89+
90+
watchedFile := "watched.file"
91+
_ = watchChanges(ctx, watchedFile, tempDir, &buf, 2, 2, []string{"ignoredField", `\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}`})
92+
cancel()
93+
time.Sleep(time.Second * 1)
94+
}
95+
96+
func applyChange(t *testing.T, srcFilePath string, dstFilePath string) {
97+
assert.NoError(t, copyFile(srcFilePath, dstFilePath, 0660))
98+
time.Sleep(diffwatch.DefaultWatchInterval * 2)
99+
}
100+
101+
func copyFile(srcFilePath string, dstFilePath string, mode os.FileMode) error {
102+
source, err := os.Open(srcFilePath)
103+
if err != nil {
104+
return err
105+
}
106+
defer func() {
107+
_ = source.Close()
108+
}()
109+
110+
destination, err := os.OpenFile(dstFilePath, os.O_RDWR|os.O_CREATE|os.O_TRUNC, mode)
111+
if err != nil {
112+
return err
113+
}
114+
defer func() {
115+
_ = destination.Close()
116+
}()
117+
118+
_, err = io.Copy(destination, source)
119+
return err
120+
}

‎tools/diffwatch/cmd/diffwatch/resources/base.json‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
{
2+
"version": 1,
3+
"processes": [
4+
{
5+
"name": "om-backup-db-0-0",
6+
"disabled": false,
7+
"hostname": "om-backup-db-0-0-svc.lsierant-10.svc.cluster.local",
8+
"args2_6": {
9+
"net": {
10+
"port": 27017,
11+
"tls": {
12+
"certificateKeyFile": "/var/lib/mongodb-automation/secrets/certs/LFKN25MS7RP2OSSJM3ORWIGPUW7VHJ24MOYDC2IXP77ADT45OR3A",
13+
"mode": "requireTLS"
14+
}
15+
},
16+
"replication": {
17+
"replSetName": "om-backup-db"
18+
},
19+
"storage": {
20+
"dbPath": "/data"
21+
},
22+
"systemLog": {
23+
"destination": "file",
24+
"logAppend": false,
25+
"path": "/var/log/mongodb-mms-automation/mongodb.log"
26+
}
27+
},
28+
"featureCompatibilityVersion": "6.0",
29+
"processType": "mongod",
30+
"version": "6.0.5-ent",
31+
"authSchemaVersion": 5,
32+
"LogRotate": {
33+
"timeThresholdHrs": 0,
34+
"sizeThresholdMB": 0
35+
}
36+
}
37+
]
38+
}

‎tools/diffwatch/cmd/diffwatch/resources/changed_1.json‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
{
2+
"version": 1,
3+
"processes": [
4+
{
5+
"name": "om-backup-db-0-0",
6+
"disabled": true,
7+
"hostname": "om-backup-db-0-0-svc.lsierant-10.svc.cluster.local",
8+
"args2_6": {
9+
"net": {
10+
"port": 27017,
11+
"tls": {
12+
"certificateKeyFile": "/var/lib/mongodb-automation/secrets/certs/LFKN25MS7RP2OSSJM3ORWIGPUW7VHJ24MOYDC2IXP77ADT45OR3A",
13+
"mode": "preferTLS"
14+
}
15+
},
16+
"replication": {
17+
"replSetName": "om-backup-db"
18+
},
19+
"storage": {
20+
"dbPath": "/data"
21+
},
22+
"systemLog": {
23+
"destination": "file",
24+
"logAppend": false,
25+
"path": "/var/log/mongodb-mms-automation/mongodb.log"
26+
}
27+
},
28+
"featureCompatibilityVersion": "6.0",
29+
"processType": "mongod",
30+
"version": "6.0.6-ent",
31+
"authSchemaVersion": 5,
32+
"newField1": 1,
33+
"newField2": 2,
34+
"newField3": 3,
35+
"newField4": 4,
36+
"newField5": 5,
37+
"LogRotate": {
38+
"timeThresholdHrs": 0,
39+
"sizeThresholdMB": 0
40+
},
41+
"newField6": 6
42+
}
43+
]
44+
}

0 commit comments

Comments
 (0)