PMM-7 Add flag to enable collstats details and disable them by default. (#997)

Author: BupycHuk

exporter/collstats_collector.go

@@ -31,13 +31,14 @@ type collstatsCollector struct {
 
 	compatibleMode  bool
 	discoveringMode bool
+	enableDetails   bool
 	topologyInfo    labelsGetter
 
 	collections []string
 }
 
 // newCollectionStatsCollector creates a collector for statistics about collections.
-func newCollectionStatsCollector(ctx context.Context, client *mongo.Client, logger *logrus.Logger, discovery bool, topology labelsGetter, collections []string) *collstatsCollector {
+func newCollectionStatsCollector(ctx context.Context, client *mongo.Client, logger *logrus.Logger, discovery bool, topology labelsGetter, collections []string, enableDetails bool) *collstatsCollector {
 	return &collstatsCollector{
 		ctx:  ctx,
 		base: newBaseCollector(client, logger.WithFields(logrus.Fields{"collector": "collstats"})),
@@ -46,7 +47,8 @@ func newCollectionStatsCollector(ctx context.Context, client *mongo.Client, logg
 		discoveringMode: discovery,
 		topologyInfo:    topology,
 
-		collections: collections,
+		collections:   collections,
+		enableDetails: enableDetails,
 	}
 }
 
@@ -110,6 +112,18 @@ func (d *collstatsCollector) collect(ch chan<- prometheus.Metric) {
 
 		pipeline := mongo.Pipeline{aggregation}
 
+		if !d.enableDetails {
+			project := bson.D{
+				{
+					Key: "$project", Value: bson.M{
+						"storageStats.wiredTiger":   0,
+						"storageStats.indexDetails": 0,
+					},
+				},
+			}
+			pipeline = append(pipeline, project)
+		}
+
 		cursor, err := client.Database(database).Collection(collection).Aggregate(d.ctx, pipeline)
 		if err != nil {
 			logger.Errorf("cannot get $collstats cursor for collection %s.%s: %s", database, collection, err)

exporter/collstats_collector_test.go

@@ -54,7 +54,7 @@ func TestCollStatsCollector(t *testing.T) {
 
 	collection := []string{"testdb.testcol_00", "testdb.testcol_01", "testdb.testcol_02"}
 	logger := logrus.New()
-	c := newCollectionStatsCollector(ctx, client, logger, false, ti, collection)
+	c := newCollectionStatsCollector(ctx, client, logger, false, ti, collection, false)
 
 	// The last \n at the end of this string is important
 	expected := strings.NewReader(`

exporter/exporter.go

@@ -45,19 +45,13 @@ type Exporter struct {
 
 // Opts holds new exporter options.
 type Opts struct {
-	// Only get stats for the collections matching this list of namespaces.
-	// Example: db1.col1,db.col1
-	CollStatsNamespaces    []string
-	CollStatsLimit         int
 	CompatibleMode         bool
 	DirectConnect          bool
 	ConnectTimeoutMS       int
 	DisableDefaultRegistry bool
 	DiscoveringMode        bool
 	GlobalConnPool         bool
-	ProfileTimeTS          int
 	TimeoutOffset          int
-	CurrentOpSlowTime      string
 
 	CollectAll               bool
 	EnableDBStats            bool
@@ -72,14 +66,20 @@ type Opts struct {
 	EnableProfile            bool
 	EnableShards             bool
 	EnableFCV                bool // Feature Compatibility Version.
+	EnablePBMMetrics         bool
 
 	EnableOverrideDescendingIndex bool
 
-	// Enable metrics for Percona Backup for MongoDB (PBM).
-	EnablePBMMetrics bool
+	// Only get stats for the collections matching this list of namespaces.
+	// Example: db1.col1,db.col1
+	CollStatsNamespaces    []string
+	CollStatsLimit         int
+	CollStatsEnableDetails bool
+	IndexStatsCollections  []string
+	CurrentOpSlowTime      string
+	ProfileTimeTS          int
 
-	IndexStatsCollections []string
-	Logger                *logrus.Logger
+	Logger *logrus.Logger
 
 	URI      string
 	NodeName string
@@ -192,7 +192,7 @@ func (e *Exporter) makeRegistry(ctx context.Context, client *mongo.Client, topol
 	if (len(e.opts.CollStatsNamespaces) > 0 || e.opts.DiscoveringMode) && e.opts.EnableCollStats && limitsOk && requestOpts.EnableCollStats {
 		cc := newCollectionStatsCollector(ctx, client, e.opts.Logger,
 			e.opts.DiscoveringMode,
-			topologyInfo, e.opts.CollStatsNamespaces)
+			topologyInfo, e.opts.CollStatsNamespaces, e.opts.CollStatsEnableDetails)
 		registry.MustRegister(cc)
 	}
 

main.go

@@ -70,7 +70,8 @@ type GlobalFlags struct {
 
 	CollectAll bool `name:"collect-all" help:"Enable all collectors. Same as specifying all --collector.<name>"`
 
-	CollStatsLimit int `name:"collector.collstats-limit" help:"Disable collstats, dbstats, topmetrics and indexstats collector if there are more than <n> collections. 0=No limit" default:"0"`
+	CollStatsLimit         int  `name:"collector.collstats-limit" help:"Disable collstats, dbstats, topmetrics and indexstats collector if there are more than <n> collections. 0=No limit" default:"0"`
+	CollStatsEnableDetails bool `name:"collector.collstats-enable-details" help:"Enable collecting index details and wired tiger metrics from $collStats" default:"false"`
 
 	ProfileTimeTS int `name:"collector.profile-time-ts" help:"Set time for scrape slow queries." default:"30"`
 
@@ -192,10 +193,11 @@ func buildExporter(opts GlobalFlags, uri string, log *logrus.Logger) *exporter.E
 
 		EnableOverrideDescendingIndex: opts.EnableOverrideDescendingIndex,
 
-		CollStatsLimit:    opts.CollStatsLimit,
-		CollectAll:        opts.CollectAll,
-		ProfileTimeTS:     opts.ProfileTimeTS,
-		CurrentOpSlowTime: opts.CurrentOpSlowTime,
+		CollStatsLimit:         opts.CollStatsLimit,
+		CollStatsEnableDetails: opts.CollStatsEnableDetails,
+		CollectAll:             opts.CollectAll,
+		ProfileTimeTS:          opts.ProfileTimeTS,
+		CurrentOpSlowTime:      opts.CurrentOpSlowTime,
 	}
 
 	e := exporter.New(exporterOpts)