diff --git a/collector/indices.go b/collector/indices.go index fdad33f..4e2080c 100644 --- a/collector/indices.go +++ b/collector/indices.go @@ -267,6 +267,26 @@ var ( "Total indexing throttle time in seconds", indicesLabels, nil, ) + indicesIndexingIndexFailed = prometheus.NewDesc( + prometheus.BuildFQName(namespace, "index_stats", "indexing_index_failed_total"), + "Total number of failed indexing operations", + indicesLabels, nil, + ) + indicesIndexingDeleteCurrent = prometheus.NewDesc( + prometheus.BuildFQName(namespace, "index_stats", "indexing_delete_current"), + "The number of documents currently being deleted from an index", + indicesLabels, nil, + ) + indicesIndexingWriteLoad = prometheus.NewDesc( + prometheus.BuildFQName(namespace, "index_stats", "indexing_write_load"), + "Write load for indexing operations", + indicesLabels, nil, + ) + indicesIndexingIsThrottled = prometheus.NewDesc( + prometheus.BuildFQName(namespace, "indices", "indexing_is_throttled"), + "Whether indexing is currently throttled for an index (1=throttled, 0=not throttled)", + indicesLabels, nil, + ) indicesGetTimeTotal = prometheus.NewDesc( prometheus.BuildFQName(namespace, "index_stats", "get_time_seconds_total"), "Total get time in seconds", @@ -536,6 +556,10 @@ func (i *Indices) Describe(ch chan<- *prometheus.Desc) { ch <- indicesIndexingDeleteTotal ch <- indicesIndexingNoopUpdateTotal ch <- indicesIndexingThrottleSecondsTotal + ch <- indicesIndexingIndexFailed + ch <- indicesIndexingDeleteCurrent + ch <- indicesIndexingWriteLoad + ch <- indicesIndexingIsThrottled ch <- indicesGetTimeTotal ch <- indicesGetTotal ch <- indicesMergeTimeTotal @@ -1053,6 +1077,42 @@ func (i *Indices) Collect(ch chan<- prometheus.Metric) { i.getClusterName(), ) + if indexStats.Total.Indexing.IndexFailed != nil { + ch <- prometheus.MustNewConstMetric( + indicesIndexingIndexFailed, + prometheus.CounterValue, + float64(*indexStats.Total.Indexing.IndexFailed), + indexName, + i.getClusterName(), + ) + } + + ch <- prometheus.MustNewConstMetric( + indicesIndexingDeleteCurrent, + prometheus.GaugeValue, + float64(indexStats.Total.Indexing.DeleteCurrent), + indexName, + i.getClusterName(), + ) + + if indexStats.Total.Indexing.WriteLoad != nil { + ch <- prometheus.MustNewConstMetric( + indicesIndexingWriteLoad, + prometheus.GaugeValue, + *indexStats.Total.Indexing.WriteLoad, + indexName, + i.getClusterName(), + ) + } + + ch <- prometheus.MustNewConstMetric( + indicesIndexingIsThrottled, + prometheus.GaugeValue, + bool2Float(indexStats.Total.Indexing.IsThrottled), + indexName, + i.getClusterName(), + ) + ch <- prometheus.MustNewConstMetric( indicesGetTimeTotal, prometheus.CounterValue, diff --git a/collector/indices_response.go b/collector/indices_response.go index d42773b..b277e5a 100644 --- a/collector/indices_response.go +++ b/collector/indices_response.go @@ -89,16 +89,17 @@ type IndexStatsIndexStoreResponse struct { // IndexStatsIndexIndexingResponse defines index stats index indexing information structure type IndexStatsIndexIndexingResponse struct { - IndexTotal int64 `json:"index_total"` - IndexTimeInMillis int64 `json:"index_time_in_millis"` - IndexCurrent int64 `json:"index_current"` - IndexFailed int64 `json:"index_failed"` - DeleteTotal int64 `json:"delete_total"` - DeleteTimeInMillis int64 `json:"delete_time_in_millis"` - DeleteCurrent int64 `json:"delete_current"` - NoopUpdateTotal int64 `json:"noop_update_total"` - IsThrottled bool `json:"is_throttled"` - ThrottleTimeInMillis int64 `json:"throttle_time_in_millis"` + IndexTotal int64 `json:"index_total"` + IndexTimeInMillis int64 `json:"index_time_in_millis"` + IndexCurrent int64 `json:"index_current"` + IndexFailed *int64 `json:"index_failed,omitempty"` + DeleteTotal int64 `json:"delete_total"` + DeleteTimeInMillis int64 `json:"delete_time_in_millis"` + DeleteCurrent int64 `json:"delete_current"` + NoopUpdateTotal int64 `json:"noop_update_total"` + IsThrottled bool `json:"is_throttled"` + ThrottleTimeInMillis int64 `json:"throttle_time_in_millis"` + WriteLoad *float64 `json:"write_load,omitempty"` } // IndexStatsIndexGetResponse defines index stats index get information structure diff --git a/collector/indices_test.go b/collector/indices_test.go index 9c246aa..76bfdab 100644 --- a/collector/indices_test.go +++ b/collector/indices_test.go @@ -103,6 +103,14 @@ func TestIndices(t *testing.T) { # TYPE elasticsearch_index_stats_indexing_throttle_time_seconds_total counter elasticsearch_index_stats_indexing_throttle_time_seconds_total{cluster="unknown_cluster",index="foo_1"} 0 elasticsearch_index_stats_indexing_throttle_time_seconds_total{cluster="unknown_cluster",index="foo_2"} 0 + # HELP elasticsearch_index_stats_indexing_delete_current The number of documents currently being deleted from an index + # TYPE elasticsearch_index_stats_indexing_delete_current gauge + elasticsearch_index_stats_indexing_delete_current{cluster="unknown_cluster",index="foo_1"} 0 + elasticsearch_index_stats_indexing_delete_current{cluster="unknown_cluster",index="foo_2"} 0 + # HELP elasticsearch_indices_indexing_is_throttled Whether indexing is currently throttled for an index (1=throttled, 0=not throttled) + # TYPE elasticsearch_indices_indexing_is_throttled gauge + elasticsearch_indices_indexing_is_throttled{cluster="unknown_cluster",index="foo_1"} 0 + elasticsearch_indices_indexing_is_throttled{cluster="unknown_cluster",index="foo_2"} 0 # HELP elasticsearch_index_stats_merge_auto_throttle_bytes_total Total bytes that were auto-throttled during merging # TYPE elasticsearch_index_stats_merge_auto_throttle_bytes_total counter elasticsearch_index_stats_merge_auto_throttle_bytes_total{cluster="unknown_cluster",index="foo_1"} 0 @@ -409,6 +417,18 @@ func TestIndices(t *testing.T) { # TYPE elasticsearch_index_stats_indexing_throttle_time_seconds_total counter elasticsearch_index_stats_indexing_throttle_time_seconds_total{cluster="unknown_cluster",index="foo_1"} 0 elasticsearch_index_stats_indexing_throttle_time_seconds_total{cluster="unknown_cluster",index="foo_2"} 0 + # HELP elasticsearch_index_stats_indexing_delete_current The number of documents currently being deleted from an index + # TYPE elasticsearch_index_stats_indexing_delete_current gauge + elasticsearch_index_stats_indexing_delete_current{cluster="unknown_cluster",index="foo_1"} 0 + elasticsearch_index_stats_indexing_delete_current{cluster="unknown_cluster",index="foo_2"} 0 + # HELP elasticsearch_index_stats_indexing_index_failed_total Total number of failed indexing operations + # TYPE elasticsearch_index_stats_indexing_index_failed_total counter + elasticsearch_index_stats_indexing_index_failed_total{cluster="unknown_cluster",index="foo_1"} 0 + elasticsearch_index_stats_indexing_index_failed_total{cluster="unknown_cluster",index="foo_2"} 0 + # HELP elasticsearch_indices_indexing_is_throttled Whether indexing is currently throttled for an index (1=throttled, 0=not throttled) + # TYPE elasticsearch_indices_indexing_is_throttled gauge + elasticsearch_indices_indexing_is_throttled{cluster="unknown_cluster",index="foo_1"} 0 + elasticsearch_indices_indexing_is_throttled{cluster="unknown_cluster",index="foo_2"} 0 # HELP elasticsearch_index_stats_merge_auto_throttle_bytes_total Total bytes that were auto-throttled during merging # TYPE elasticsearch_index_stats_merge_auto_throttle_bytes_total counter elasticsearch_index_stats_merge_auto_throttle_bytes_total{cluster="unknown_cluster",index="foo_1"} 1.048576e+08 @@ -755,6 +775,27 @@ func TestIndices(t *testing.T) { elasticsearch_index_stats_indexing_throttle_time_seconds_total{cluster="unknown_cluster",index=".watches"} 0 elasticsearch_index_stats_indexing_throttle_time_seconds_total{cluster="unknown_cluster",index="foo_1"} 0 elasticsearch_index_stats_indexing_throttle_time_seconds_total{cluster="unknown_cluster",index="foo_2"} 0 + # HELP elasticsearch_index_stats_indexing_delete_current The number of documents currently being deleted from an index + # TYPE elasticsearch_index_stats_indexing_delete_current gauge + elasticsearch_index_stats_indexing_delete_current{cluster="unknown_cluster",index=".monitoring-data-2"} 0 + elasticsearch_index_stats_indexing_delete_current{cluster="unknown_cluster",index=".monitoring-es-2-2017.08.23"} 0 + elasticsearch_index_stats_indexing_delete_current{cluster="unknown_cluster",index=".watches"} 0 + elasticsearch_index_stats_indexing_delete_current{cluster="unknown_cluster",index="foo_1"} 0 + elasticsearch_index_stats_indexing_delete_current{cluster="unknown_cluster",index="foo_2"} 0 + # HELP elasticsearch_index_stats_indexing_index_failed_total Total number of failed indexing operations + # TYPE elasticsearch_index_stats_indexing_index_failed_total counter + elasticsearch_index_stats_indexing_index_failed_total{cluster="unknown_cluster",index=".monitoring-data-2"} 0 + elasticsearch_index_stats_indexing_index_failed_total{cluster="unknown_cluster",index=".monitoring-es-2-2017.08.23"} 0 + elasticsearch_index_stats_indexing_index_failed_total{cluster="unknown_cluster",index=".watches"} 0 + elasticsearch_index_stats_indexing_index_failed_total{cluster="unknown_cluster",index="foo_1"} 0 + elasticsearch_index_stats_indexing_index_failed_total{cluster="unknown_cluster",index="foo_2"} 0 + # HELP elasticsearch_indices_indexing_is_throttled Whether indexing is currently throttled for an index (1=throttled, 0=not throttled) + # TYPE elasticsearch_indices_indexing_is_throttled gauge + elasticsearch_indices_indexing_is_throttled{cluster="unknown_cluster",index=".monitoring-data-2"} 0 + elasticsearch_indices_indexing_is_throttled{cluster="unknown_cluster",index=".monitoring-es-2-2017.08.23"} 0 + elasticsearch_indices_indexing_is_throttled{cluster="unknown_cluster",index=".watches"} 0 + elasticsearch_indices_indexing_is_throttled{cluster="unknown_cluster",index="foo_1"} 0 + elasticsearch_indices_indexing_is_throttled{cluster="unknown_cluster",index="foo_2"} 0 # HELP elasticsearch_index_stats_merge_auto_throttle_bytes_total Total bytes that were auto-throttled during merging # TYPE elasticsearch_index_stats_merge_auto_throttle_bytes_total counter elasticsearch_index_stats_merge_auto_throttle_bytes_total{cluster="unknown_cluster",index=".monitoring-data-2"} 2.097152e+07 @@ -1271,6 +1312,24 @@ func TestIndices(t *testing.T) { elasticsearch_index_stats_indexing_throttle_time_seconds_total{cluster="unknown_cluster",index="foo_1"} 0 elasticsearch_index_stats_indexing_throttle_time_seconds_total{cluster="unknown_cluster",index="foo_2"} 0 elasticsearch_index_stats_indexing_throttle_time_seconds_total{cluster="unknown_cluster",index="foo_3"} 0 + # HELP elasticsearch_index_stats_indexing_delete_current The number of documents currently being deleted from an index + # TYPE elasticsearch_index_stats_indexing_delete_current gauge + elasticsearch_index_stats_indexing_delete_current{cluster="unknown_cluster",index=".geoip_databases"} 0 + elasticsearch_index_stats_indexing_delete_current{cluster="unknown_cluster",index="foo_1"} 0 + elasticsearch_index_stats_indexing_delete_current{cluster="unknown_cluster",index="foo_2"} 0 + elasticsearch_index_stats_indexing_delete_current{cluster="unknown_cluster",index="foo_3"} 0 + # HELP elasticsearch_index_stats_indexing_index_failed_total Total number of failed indexing operations + # TYPE elasticsearch_index_stats_indexing_index_failed_total counter + elasticsearch_index_stats_indexing_index_failed_total{cluster="unknown_cluster",index=".geoip_databases"} 0 + elasticsearch_index_stats_indexing_index_failed_total{cluster="unknown_cluster",index="foo_1"} 0 + elasticsearch_index_stats_indexing_index_failed_total{cluster="unknown_cluster",index="foo_2"} 0 + elasticsearch_index_stats_indexing_index_failed_total{cluster="unknown_cluster",index="foo_3"} 0 + # HELP elasticsearch_indices_indexing_is_throttled Whether indexing is currently throttled for an index (1=throttled, 0=not throttled) + # TYPE elasticsearch_indices_indexing_is_throttled gauge + elasticsearch_indices_indexing_is_throttled{cluster="unknown_cluster",index=".geoip_databases"} 0 + elasticsearch_indices_indexing_is_throttled{cluster="unknown_cluster",index="foo_1"} 0 + elasticsearch_indices_indexing_is_throttled{cluster="unknown_cluster",index="foo_2"} 0 + elasticsearch_indices_indexing_is_throttled{cluster="unknown_cluster",index="foo_3"} 0 # HELP elasticsearch_index_stats_merge_auto_throttle_bytes_total Total bytes that were auto-throttled during merging # TYPE elasticsearch_index_stats_merge_auto_throttle_bytes_total counter elasticsearch_index_stats_merge_auto_throttle_bytes_total{cluster="unknown_cluster",index=".geoip_databases"} 2.097152e+07 @@ -1726,6 +1785,24 @@ func TestIndices(t *testing.T) { elasticsearch_index_stats_indexing_throttle_time_seconds_total{cluster="unknown_cluster",index="foo_1"} 0 elasticsearch_index_stats_indexing_throttle_time_seconds_total{cluster="unknown_cluster",index="foo_2"} 0 elasticsearch_index_stats_indexing_throttle_time_seconds_total{cluster="unknown_cluster",index="foo_3"} 0 + # HELP elasticsearch_index_stats_indexing_delete_current The number of documents currently being deleted from an index + # TYPE elasticsearch_index_stats_indexing_delete_current gauge + elasticsearch_index_stats_indexing_delete_current{cluster="unknown_cluster",index=".geoip_databases"} 0 + elasticsearch_index_stats_indexing_delete_current{cluster="unknown_cluster",index="foo_1"} 0 + elasticsearch_index_stats_indexing_delete_current{cluster="unknown_cluster",index="foo_2"} 0 + elasticsearch_index_stats_indexing_delete_current{cluster="unknown_cluster",index="foo_3"} 0 + # HELP elasticsearch_index_stats_indexing_index_failed_total Total number of failed indexing operations + # TYPE elasticsearch_index_stats_indexing_index_failed_total counter + elasticsearch_index_stats_indexing_index_failed_total{cluster="unknown_cluster",index=".geoip_databases"} 0 + elasticsearch_index_stats_indexing_index_failed_total{cluster="unknown_cluster",index="foo_1"} 0 + elasticsearch_index_stats_indexing_index_failed_total{cluster="unknown_cluster",index="foo_2"} 0 + elasticsearch_index_stats_indexing_index_failed_total{cluster="unknown_cluster",index="foo_3"} 0 + # HELP elasticsearch_indices_indexing_is_throttled Whether indexing is currently throttled for an index (1=throttled, 0=not throttled) + # TYPE elasticsearch_indices_indexing_is_throttled gauge + elasticsearch_indices_indexing_is_throttled{cluster="unknown_cluster",index=".geoip_databases"} 0 + elasticsearch_indices_indexing_is_throttled{cluster="unknown_cluster",index="foo_1"} 0 + elasticsearch_indices_indexing_is_throttled{cluster="unknown_cluster",index="foo_2"} 0 + elasticsearch_indices_indexing_is_throttled{cluster="unknown_cluster",index="foo_3"} 0 # HELP elasticsearch_index_stats_merge_auto_throttle_bytes_total Total bytes that were auto-throttled during merging # TYPE elasticsearch_index_stats_merge_auto_throttle_bytes_total counter elasticsearch_index_stats_merge_auto_throttle_bytes_total{cluster="unknown_cluster",index=".geoip_databases"} 2.097152e+07