artin/exportarr
1
0
Fork 0
mirror of https://github.com/onedr0p/exportarr.git synced 2026-02-06 10:57:32 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add --backfill & --backfill-since-date options for Prowlarr. (#117)

Browse Source 
* Add backfill options to Prowlarr

Signed-off-by: Russell Troxel <russell.troxel@segment.com>

* Update Readme

Signed-off-by: Russell Troxel <russelltroxel@gmail.com>

---------

Signed-off-by: Russell Troxel <russell.troxel@segment.com>
Signed-off-by: Russell Troxel <russelltroxel@gmail.com>
This commit is contained in:
Russell Troxel 2023-03-18 09:29:56 -07:00 committed by GitHub
parent 7500b12561
commit e3edbb2a65
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 178 additions and 31 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
README.md
View File

@ -146,9 +146,15 @@ Visit http://127.0.0.1:9709/metrics to see Radarr metrics
./exportarr prowlarr \
--port 9710 \
--url http://x.x.x.x:9696 \
--api-key amlmndfb503rfqaa5ln5hj5qkmu3hy18
--api-key amlmndfb503rfqaa5ln5hj5qkmu3hy18 \
--backfill \
--backfill-since-date 2023-03-01
```
Backfill options are optional.
Visit http://127.0.0.1:9710/metrics to see Prowlarr metrics
#### Readarr (Experimental)
```sh
@ -160,7 +166,7 @@ Visit http://127.0.0.1:9709/metrics to see Radarr metrics
--api-key amlmndfb503rfqaa5ln5hj5qkmu3hy18
```
Visit http://127.0.0.1:9710/metrics to see Prowlarr metrics
Visit http://127.0.0.1:9711/metrics to see Readarr metrics
## Configuration
@ -169,18 +175,25 @@ Visit http://127.0.0.1:9710/metrics to see Prowlarr metrics
| `PORT` | `--port` or `-p` | The port exportarr will listen on | | ✅ |
| `URL` | `--url` or `-u` | The full URL to Sonarr, Radarr, or Lidarr | | ✅ |
| `APIKEY` | `--api-key` or `-a` | API Key for Sonarr, Radarr or Lidarr | | ❌ |
| `APIKEY_FILE` | `--api-key-file` | API Key file location for Sonarr, Radarr or Lidarr | | ❌ |
| `API_KEY_FILE` | `--api-key-file` | API Key file location for Sonarr, Radarr or Lidarr | | ❌ |
| `CONFIG` | `--config` or `-c` | Path to Sonarr, Radarr or Lidarr's `config.xml` (advanced) | | ❌ |
| `INTERFACE` | `--interface` or `-i` | The interface IP exportarr will listen on | `0.0.0.0` | ❌ |
| `LOG_LEVEL` | `--log-level` or `-l` | Set the default Log Level | `INFO` | ❌ |
| `DISABLE_SSL_VERIFY` | `--disable-ssl-verify` | Set to `true` to disable SSL verification | `false` | ❌ |
| `BASIC_AUTH_PASSWORD` | `--basic-auth-password` | Set to your basic auth password | | ❌ |
| `BASIC_AUTH_USERNAME` | `--basic-auth-username` | Set to your basic auth username | | ❌ |
| `AUTH_PASSWORD` | `--auth-password` | Set to your basic or form auth password | | ❌ |
| `AUTH_USERNAME` | `--auth-username` | Set to your basic or form auth username | | ❌ |
| `FORM_AUTH` | `--form-auth` | Use Form Auth instead of basic auth | `false` | ❌ |
| `ENABLE_ADDITIONAL_METRICS` | `--enable-additional-metrics` | Set to `true` to enable gathering of additional metrics (slow) | `false` | ❌ |
| `ENABLE_UNKNOWN_QUEUE_ITEMS` | `--enable-unknown-queue-items` | Set to `true` to enable gathering unknown queue items | `false` | ❌ |
| `PROWLARR__BACKFILL` | `--backfill` | Set to `true` to enable backfill of historical metrics | `false` | ❌ |
| `PROWLARR__BACKFILL_SINCE_DATE` | `--backfill-since-date` | Set a date from which to start the backfill | `1970-01-01` (epoch) | ❌ |
### Prowlarr Backfill
The prowlarr collector is a little different than other collectors as it's hitting an actual "stats" endpoint, collecting counters of events that happened in a small time window, rather than getting all-time statistics like the other collectors. This means that by default, when you start the prowlarr collector, collected stats will start from that moment (all counters will start from zero). `ENABLE_ADDITIONAL_METRIC` will tell the collector that it should backfill all-time metrics the first time it's queried. The actual API call this makes is *really* slow. On my prowlarr instance which is only a few months old, this took more than 5 seconds, so if you have a very active or very old prowlarr server this might time out, and need to be turned off.
The prowlarr collector is a little different than other collectors as it's hitting an actual "stats" endpoint, collecting counters of events that happened in a small time window, rather than getting all-time statistics like the other collectors. This means that by default, when you start the prowlarr collector, collected stats will start from that moment (all counters will start from zero).
Note that if you have to turn off backfill that your counters will reset to zero every time you restart the collector. This isn't a huge problem, prometheus expects counters to reset on restart, so normal aggregations like `rate()`, `increase()`, `irate()`, etc will all still work fine. The only thing that will be tough will be getting all-time stats (you can still do it, but you'll have to do something like `increase(stat_you_want_to_see[<timeserverhasbeenalive>])`, which will likely be slow)
To backill all Prowlarr Data, either use `PROWLARR__BACKFILL` or `--backfill`.
Note that the first request can be extremely slow, depending on how long your prowlarr instance has been running. You can also specify a start date to limit the backfill if the backfill is timing out:
`PROWLARR__BACKFILL_DATE_SINCE=2023-03-01` or `--backfill-date-since=2023-03-01`

9
internal/collector/prowlarr/stats.go
View File

@ -116,12 +116,9 @@ type prowlarrCollector struct {
}
func NewProwlarrCollector(c *config.Config) *prowlarrCollector {
var lastStatUpdate time.Time
if c.EnableAdditionalMetrics {
// If additional metrics are enabled, backfill the cache.
lastStatUpdate = time.Time{}
} else {
lastStatUpdate = time.Now()
lastStatUpdate := time.Now()
if c.Prowlarr.Backfill || !c.Prowlarr.BackfillSinceTime.IsZero() {
lastStatUpdate = c.Prowlarr.BackfillSinceTime
}
return &prowlarrCollector{
config: c,

7
internal/commands/prowlarr.go
View File

@ -10,6 +10,9 @@ import (
func init() {
rootCmd.AddCommand(prowlarrCmd)
prowlarrCmd.PersistentFlags().Bool("backfill", false, "Backfill Prowlarr")
prowlarrCmd.PersistentFlags().String("backfill-since-date", "", "Date from which to start Prowlarr Backfill")
}
var prowlarrCmd = &cobra.Command{
@ -19,6 +22,10 @@ var prowlarrCmd = &cobra.Command{
RunE: func(cmd *cobra.Command, args []string) error {
conf.Arr = "prowlarr"
conf.ApiVersion = "v1"
conf.LoadProwlarrFlags(cmd.PersistentFlags())
if err := conf.Prowlarr.Validate(); err != nil {
return err
}
serveHttp(func(r *prometheus.Registry) {
r.MustRegister(
prowlarrCollector.NewProwlarrCollector(conf),

62
internal/config/config.go
View File

@ -18,22 +18,24 @@ import (
)
type Config struct {
Arr string `koanf:"arr"`
LogLevel string `koanf:"log-level" validate:"ValidateLogLevel"`
LogFormat string `koanf:"log-format" validate:"in:console,json"`
URL string `koanf:"url" validate:"required|url"`
ApiKey string `koanf:"api-key" validate:"required|regex:([a-z0-9]{32})"`
ApiKeyFile string `koanf:"api-key-file"`
ApiVersion string `koanf:"api-version" validate:"required|in:v3,v4"`
XMLConfig string `koanf:"config"`
Port int `koanf:"port" validate:"required"`
Interface string `koanf:"interface" validate:"required|ip"`
DisableSSLVerify bool `koanf:"disable-ssl-verify"`
AuthUsername string `koanf:"auth-username"`
AuthPassword string `koanf:"auth-password"`
FormAuth bool `koanf:"form-auth"`
EnableUnknownQueueItems bool `koanf:"enable-unknown-queue-items"`
EnableAdditionalMetrics bool `koanf:"enable-additional-metric"`
Arr string `koanf:"arr"`
LogLevel string `koanf:"log-level" validate:"ValidateLogLevel"`
LogFormat string `koanf:"log-format" validate:"in:console,json"`
URL string `koanf:"url" validate:"required|url"`
ApiKey string `koanf:"api-key" validate:"required|regex:([a-z0-9]{32})"`
ApiKeyFile string `koanf:"api-key-file"`
ApiVersion string `koanf:"api-version" validate:"required|in:v3,v4"`
XMLConfig string `koanf:"config"`
Port int `koanf:"port" validate:"required"`
Interface string `koanf:"interface" validate:"required|ip"`
DisableSSLVerify bool `koanf:"disable-ssl-verify"`
AuthUsername string `koanf:"auth-username"`
AuthPassword string `koanf:"auth-password"`
FormAuth bool `koanf:"form-auth"`
EnableUnknownQueueItems bool `koanf:"enable-unknown-queue-items"`
EnableAdditionalMetrics bool `koanf:"enable-additional-metric"`
Prowlarr ProwlarrConfig `koanf:"prowlarr"`
k *koanf.Koanf
}
func (c *Config) UseBasicAuth() bool {
@ -57,6 +59,7 @@ func (c *Config) URLLabel() string {
}
return c.URL
}
func LoadConfig(flags *flag.FlagSet) (*Config, error) {
k := koanf.New(".")
@ -74,7 +77,10 @@ func LoadConfig(flags *flag.FlagSet) (*Config, error) {
// Environment
err = k.Load(env.Provider("", ".", func(s string) string {
return strings.Replace(strings.ToLower(s), "_", "-", -1)
s = strings.ToLower(s)
s = strings.Replace(s, "__", ".", -1)
s = strings.Replace(s, "_", "-", -1)
return s
}), nil)
if err != nil {
return nil, err
@ -109,7 +115,7 @@ func LoadConfig(flags *flag.FlagSet) (*Config, error) {
if err := k.Unmarshal("", &out); err != nil {
return nil, err
}
out.k = k
return &out, nil
}
@ -146,3 +152,23 @@ func (c *Config) Messages() map[string]string {
"LogLevel.validateLogLevel": "Log Level must be one of: debug, info, warn, error, dpanic, panic, fatal",
}
}
func (c *Config) Translates() map[string]string {
return validate.MS{
"LogLevel": "log-level",
"LogFormat": "log-format",
"URL": "url",
"ApiKey": "api-key",
"ApiKeyFile": "api-key-file",
"ApiVersion": "api-version",
"XMLConfig": "config",
"Port": "port",
"Interface": "interface",
"DisableSSLVerify": "disable-ssl-verify",
"AuthUsername": "auth-username",
"AuthPassword": "auth-password",
"FormAuth": "form-auth",
"EnableUnknownQueueItems": "enable-unknown-queue-items",
"EnableAdditionalMetrics": "enable-additional-metric",
}
}

58
internal/config/prowlarr.go Normal file
View File

@ -0,0 +1,58 @@
package config
import (
"fmt"
"time"
"github.com/gookit/validate"
"github.com/knadh/koanf/providers/posflag"
"github.com/knadh/koanf/v2"
flag "github.com/spf13/pflag"
)
type ProwlarrConfig struct {
Backfill bool `koanf:"backfill"`
BackfillSinceDate string `koanf:"backfill-since-date" validate:"date"`
BackfillSinceTime time.Time
}
func (p ProwlarrConfig) Validate() error {
v := validate.Struct(p)
if !v.Validate() {
return v.Errors
}
if p.BackfillSinceDate != "" && p.BackfillSinceTime.IsZero() {
// Should be unreachable as long as we validate that the date is valid in LoadProwlarrFlags/Validate
return fmt.Errorf("backfill-since-date is not a valid date")
}
return nil
}
func (p ProwlarrConfig) Messages() map[string]string {
return validate.MS{
"backfill-since-date": "backfill-since-date is not a valid date",
}
}
func (p ProwlarrConfig) Translates() map[string]string {
return validate.MS{
"BackfillSinceDate": "backfill-since-date",
}
}
func (c *Config) LoadProwlarrFlags(flags *flag.FlagSet) error {
err := c.k.Load(posflag.Provider(flags, ".", c.k), nil, koanf.WithMergeFunc(func(src, dest map[string]interface{}) error {
dest["prowlarr"] = src
return nil
}))
if err != nil {
return err
}
err = c.k.Unmarshal("prowlarr", &c.Prowlarr)
c.Prowlarr.BackfillSinceTime = c.k.Time("prowlarr.backfill-since-date", "2006-01-02")
if err != nil {
return err
}
return nil
}

46
internal/config/prowlarr_test.go Normal file
View File

@ -0,0 +1,46 @@
package config
import (
"testing"
"time"
"github.com/stretchr/testify/require"
)
func TestValidateProwlarr(t *testing.T) {
tm, _ := time.Parse("2006-01-02", "2021-01-01")
parameters := []struct {
name string
config *ProwlarrConfig
shouldError bool
}{
{
name: "good",
config: &ProwlarrConfig{
Backfill: true,
BackfillSinceTime: tm,
BackfillSinceDate: "2021-01-01",
},
},
{
name: "bad-date",
config: &ProwlarrConfig{
Backfill: true,
BackfillSinceTime: tm,
BackfillSinceDate: "2021-31-31",
},
shouldError: true,
},
}
for _, parameter := range parameters {
t.Run(parameter.name, func(t *testing.T) {
err := parameter.config.Validate()
if parameter.shouldError {
require.Error(t, err)
} else {
require.NoError(t, err)
}
})
}
}