Read metrics from file (v2) (cs3org#1118)

changelog/unreleased/read-metrics-from-file-2.md

@@ -0,0 +1,5 @@
+Enhancement: Metrics module can be configured to retrieve metrics data from file
+
+- Export site metrics in Prometheus #698
+
+https://github.com/cs3org/reva/pull/1118

cmd/revad/runtime/loader.go

@@ -29,7 +29,7 @@ import (
 	_ "github.com/cs3org/reva/internal/http/services/loader"
 	_ "github.com/cs3org/reva/pkg/auth/manager/loader"
 	_ "github.com/cs3org/reva/pkg/auth/registry/loader"
-	_ "github.com/cs3org/reva/pkg/metrics"
+	_ "github.com/cs3org/reva/pkg/metrics/driver/loader"
 	_ "github.com/cs3org/reva/pkg/ocm/invite/manager/loader"
 	_ "github.com/cs3org/reva/pkg/ocm/provider/authorizer/loader"
 	_ "github.com/cs3org/reva/pkg/ocm/share/manager/loader"

examples/metrics/metrics.toml

@@ -1,7 +1,14 @@
 [shared]
-jwt_secret = "Pive-Fumkiu4"
+
+[http.services.metrics]
+# one of dummy, json.
+metrics_data_driver_type = "dummy"
+# if left unspecified location /var/tmp/reva/metrics/metricsdata.json is used (for driver type json).
+metrics_data_location = ""
+# metrics recording interval in milliseconds
+metrics_record_interval = 5000
 
 [http.services.prometheus]
 
 [http]
-address = "0.0.0.0:5550"
+address = "0.0.0.0:5550"

internal/http/services/loader/loader.go

@@ -25,6 +25,7 @@ import (
 	_ "github.com/cs3org/reva/internal/http/services/helloworld"
 	_ "github.com/cs3org/reva/internal/http/services/mentix"
 	_ "github.com/cs3org/reva/internal/http/services/meshdirectory"
+	_ "github.com/cs3org/reva/internal/http/services/metrics"
 	_ "github.com/cs3org/reva/internal/http/services/ocmd"
 	_ "github.com/cs3org/reva/internal/http/services/oidcprovider"
 	_ "github.com/cs3org/reva/internal/http/services/owncloud/ocdav"

internal/http/services/metrics/metrics.go

@@ -0,0 +1,92 @@
+// Copyright 2018-2020 CERN
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// In applying this license, CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+package metrics
+
+/*
+This service initializes the metrics package according to the metrics configuration.
+*/
+import (
+	"net/http"
+	"os"
+
+	"github.com/cs3org/reva/pkg/logger"
+	"github.com/mitchellh/mapstructure"
+	"github.com/rs/zerolog"
+
+	"github.com/cs3org/reva/pkg/metrics"
+	"github.com/cs3org/reva/pkg/metrics/config"
+	"github.com/cs3org/reva/pkg/rhttp/global"
+)
+
+func init() {
+	global.Register(serviceName, New)
+}
+
+const (
+	serviceName = "metrics"
+)
+
+// Close is called when this service is being stopped.
+func (s *svc) Close() error {
+	return nil
+}
+
+// Prefix returns the main endpoint of this service.
+func (s *svc) Prefix() string {
+	return ""
+}
+
+// Unprotected returns all endpoints that can be queried without prior authorization.
+func (s *svc) Unprotected() []string {
+	return []string{}
+}
+
+// Handler serves all HTTP requests.
+func (s *svc) Handler() http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		log := logger.New().With().Int("pid", os.Getpid()).Logger()
+		if _, err := w.Write([]byte("This is the metrics service.\n")); err != nil {
+			log.Error().Err(err).Msg("error writing metrics response")
+		}
+	})
+}
+
+// New returns a new metrics service.
+func New(m map[string]interface{}, log *zerolog.Logger) (global.Service, error) {
+	// Prepare the configuration
+	conf := &config.Config{}
+	if err := mapstructure.Decode(m, conf); err != nil {
+		return nil, err
+	}
+
+	conf.Init()
+
+	// initialize metrics using the configuration
+	err := metrics.Init(conf)
+	if err != nil {
+		return nil, err
+	}
+
+	// Create the service
+	s := &svc{}
+	return s, nil
+}
+
+type svc struct {
+}

internal/http/services/prometheus/prometheus.go

@@ -28,8 +28,6 @@ import (
 	"go.opencensus.io/stats/view"
 
 	"github.com/cs3org/reva/pkg/rhttp/global"
-	// Initializes goroutines which periodically update stats
-	_ "github.com/cs3org/reva/pkg/metrics/reader/dummy"
 )
 
 func init() {

pkg/metrics/config/config.go

@@ -0,0 +1,39 @@
+// Copyright 2018-2020 CERN
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// In applying this license, CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+package config
+
+// Config holds the config options that need to be passed down to the metrics reader(driver)
+type Config struct {
+	MetricsDataDriverType string `mapstructure:"metrics_data_driver_type"`
+	MetricsDataLocation   string `mapstructure:"metrics_data_location"`
+	MetricsRecordInterval int    `mapstructure:"metrics_record_interval"`
+}
+
+// Init sets sane defaults
+func (c *Config) Init() {
+	if c.MetricsDataDriverType == "json" {
+		// default values
+		if c.MetricsDataLocation == "" {
+			c.MetricsDataLocation = "/var/tmp/reva/metrics/metricsdata.json"
+		}
+	}
+	if c.MetricsRecordInterval == 0 {
+		c.MetricsRecordInterval = 5000
+	}
+}

pkg/metrics/driver/dummy/dummy.go

@@ -0,0 +1,60 @@
+// Copyright 2018-2020 CERN
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// In applying this license, CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+package dummy
+
+import (
+	"math/rand"
+
+	"github.com/cs3org/reva/pkg/metrics/config"
+	"github.com/cs3org/reva/pkg/metrics/driver/registry"
+)
+
+func init() {
+	driver := &MetricsDummyDriver{}
+	registry.Register(driverName(), driver)
+}
+
+func driverName() string {
+	return "dummy"
+}
+
+// MetricsDummyDriver the MetricsDummyDriver struct
+type MetricsDummyDriver struct {
+}
+
+// Configure configures this driver
+func (d *MetricsDummyDriver) Configure(c *config.Config) error {
+	// no configuration necessary
+	return nil
+}
+
+// GetNumUsers returns the number of site users; it's a random number
+func (d *MetricsDummyDriver) GetNumUsers() int64 {
+	return int64(rand.Intn(30000))
+}
+
+// GetNumGroups returns the number of site groups; it's a random number
+func (d *MetricsDummyDriver) GetNumGroups() int64 {
+	return int64(rand.Intn(200))
+}
+
+// GetAmountStorage returns the amount of site storage used; it's a random amount
+func (d *MetricsDummyDriver) GetAmountStorage() int64 {
+	return int64(rand.Intn(70000000000))
+}

pkg/metrics/driver/json/json.go

@@ -0,0 +1,98 @@
+// Copyright 2018-2020 CERN
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// In applying this license, CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+package json
+
+import (
+	"encoding/json"
+	"errors"
+	"io/ioutil"
+	"os"
+
+	"github.com/cs3org/reva/pkg/metrics/driver/registry"
+
+	"github.com/cs3org/reva/pkg/logger"
+	"github.com/cs3org/reva/pkg/metrics/config"
+	"github.com/rs/zerolog"
+)
+
+var log zerolog.Logger
+
+func init() {
+	log = logger.New().With().Int("pid", os.Getpid()).Logger()
+	driver := &MetricsJSONDriver{}
+	registry.Register(driverName(), driver)
+}
+
+func driverName() string {
+	return "json"
+}
+
+// readJSON always returns a data object but logs the error in case reading the json fails.
+func readJSON(driver *MetricsJSONDriver) *data {
+	data := &data{}
+
+	file, err := ioutil.ReadFile(driver.metricsDataLocation)
+	if err != nil {
+		log.Error().Err(err).Str("location", driver.metricsDataLocation).Msg("Unable to read json file from location.")
+	}
+	err = json.Unmarshal(file, data)
+	if err != nil {
+		log.Error().Err(err).Msg("Unable to unmarshall json file.")
+	}
+
+	return data
+}
+
+type data struct {
+	NumUsers      int64 `json:"cs3_org_sciencemesh_site_total_num_users"`
+	NumGroups     int64 `json:"cs3_org_sciencemesh_site_total_num_groups"`
+	AmountStorage int64 `json:"cs3_org_sciencemesh_site_total_amount_storage"`
+}
+
+// MetricsJSONDriver the JsonDriver struct
+type MetricsJSONDriver struct {
+	metricsDataLocation string
+}
+
+// Configure configures this driver
+func (d *MetricsJSONDriver) Configure(c *config.Config) error {
+	if c.MetricsDataLocation == "" {
+		err := errors.New("Unable to initialize a metrics data driver, has the data location (metrics_data_location) been configured?")
+		return err
+	}
+
+	d.metricsDataLocation = c.MetricsDataLocation
+
+	return nil
+}
+
+// GetNumUsers returns the number of site users
+func (d *MetricsJSONDriver) GetNumUsers() int64 {
+	return readJSON(d).NumUsers
+}
+
+// GetNumGroups returns the number of site groups
+func (d *MetricsJSONDriver) GetNumGroups() int64 {
+	return readJSON(d).NumGroups
+}
+
+// GetAmountStorage returns the amount of site storage used
+func (d *MetricsJSONDriver) GetAmountStorage() int64 {
+	return readJSON(d).AmountStorage
+}

pkg/metrics/driver/loader/loader.go

@@ -0,0 +1,26 @@
+// Copyright 2018-2020 CERN
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+//
+// In applying this license, CERN does not waive the privileges and immunities
+// granted to it by virtue of its status as an Intergovernmental Organization
+// or submit itself to any jurisdiction.
+
+package loader
+
+import (
+	// Load metrics drivers.
+	_ "github.com/cs3org/reva/pkg/metrics/driver/dummy"
+	_ "github.com/cs3org/reva/pkg/metrics/driver/json"
+	// Add your own here
+)