artin/API-Manager
1
0
Fork 0
mirror of https://github.com/OpenBankProject/API-Manager.git synced 2026-02-06 15:06:48 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #286 from Reena-cell/develop

Browse Source 
improved include_app_names
This commit is contained in:
Simon Redfern 2023-02-09 14:29:10 +01:00 committed by GitHub
commit f7caf7f83e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 141 additions and 67 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
apimanager/apimanager/settings.py
View File

@ -27,7 +27,7 @@ BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
SECRET_KEY = None
# SECURITY WARNING: don't run with debug turned on in production!
DEBUG = True
DEBUG = False

41
apimanager/metrics/forms.py
View File

@ -212,18 +212,16 @@ class CustomSummaryForm(forms.Form):
required=True,
initial=(datetime.now() - timedelta(6)).strftime(API_MANAGER_DATE_FORMAT),
)
#exclude_app_names = forms.CharField(
# label=_('Exclude App Names'),
# widget=forms.TextInput(
# attrs={
# 'class': FORM_CONTROL,
# }
# ),
# required=False,
# initial='API-Manager',
#)
#include_obp_apps = forms.BooleanField(required=False, label=_('Include System Data'))
include_obp_apps = forms.BooleanField(required=False)
include_app_names = forms.CharField(
label=_('Include App Names'),
widget=forms.TextInput(
attrs={
'class': FORM_CONTROL,
}
),
required=False,
)
def __init__(self, *args, **kwargs):
kwargs.setdefault('label_suffix', '')
@ -237,16 +235,15 @@ class MonthlyMetricsSummaryForm(forms.Form):
#initial=str(datetime.now().strftime('%Y-%m-%dT%H:%M:%S.%fZ')),
initial=str(datetime.now().strftime(API_MANAGER_DATE_FORMAT)),
)
#exclude_app_names = forms.CharField(
# label=_('Exclude App Names'),
# widget=forms.TextInput(
# attrs={
# 'class': FORM_CONTROL,
# }
# ),
# required=False,
# initial='API-Manager',
#)
include_app_names = forms.CharField(
label=_('Include App Names'),
widget=forms.TextInput(
attrs={
'class': FORM_CONTROL,
}
),
required=False,
)
def __init__(self, *args, **kwargs):
kwargs.setdefault('label_suffix', '')

6
apimanager/metrics/templates/metrics/custom_summary.html
View File

@ -50,10 +50,10 @@
</div>
</div>
<div class="col-xs-4 col-sm-3">
{% if form.exclude_app_names.errors %}<div class="alert alert-danger">{{ form.exclude_app_names.errors }}</div>{% endif %}
{% if form.include_app_names.errors %}<div class="alert alert-danger">{{ form.include_app_names.errors }}</div>{% endif %}
<div class="form-group">
{{ form.exclude_app_names.label_tag }}
{{ form.exclude_app_names }}
{{ form.include_app_names.label_tag }}
{{ form.include_app_names }}
</div>
</div>
</div>

6
apimanager/metrics/templates/metrics/daily_summary.html
View File

@ -50,10 +50,10 @@
</div>
</div>
<div class="col-xs-4 col-sm-3">
{% if form.exclude_app_names.errors %}<div class="alert alert-danger">{{ form.exclude_app_names.errors }}</div>{% endif %}
{% if form.include_app_names.errors %}<div class="alert alert-danger">{{ form.include_app_names.errors }}</div>{% endif %}
<div class="form-group">
{{ form.exclude_app_names.label_tag }}
{{ form.exclude_app_names }}
{{ form.include_app_names.label_tag }}
{{ form.include_app_names }}
</div>
</div>
</div>

6
apimanager/metrics/templates/metrics/hourly_summary.html
View File

@ -49,10 +49,10 @@
</div>
</div>
<div class="col-xs-4 col-sm-3">
{% if form.exclude_app_names.errors %}<div class="alert alert-danger">{{ form.exclude_app_names.errors }}</div>{% endif %}
{% if form.include_app_names.errors %}<div class="alert alert-danger">{{ form.include_app_names.errors }}</div>{% endif %}
<div class="form-group">
{{ form.exclude_app_names.label_tag }}
{{ form.exclude_app_names }}
{{ form.include_app_names.label_tag }}
{{ form.include_app_names }}
</div>
</div>
</div>

6
apimanager/metrics/templates/metrics/monthly_summary.html
View File

@ -45,10 +45,10 @@
</div>
</div>
<div class="col-xs-4 col-sm-3">
{% if form.exclude_app_names.errors %}<div class="alert alert-danger">{{ form.exclude_app_names.errors }}</div>{% endif %}
{% if form.include_app_names.errors %}<div class="alert alert-danger">{{ form.include_app_names.errors }}</div>{% endif %}
<div class="form-group">
{{ form.exclude_app_names.label_tag }}
{{ form.exclude_app_names }}
{{ form.include_app_names.label_tag }}
{{ form.include_app_names }}
</div>
</div>
</div>

6
apimanager/metrics/templates/metrics/quarterly_summary.html
View File

@ -49,10 +49,10 @@
</div>
</div>
<div class="col-xs-4 col-sm-3">
{% if form.exclude_app_names.errors %}<div class="alert alert-danger">{{ form.exclude_app_names.errors }}</div>{% endif %}
{% if form.include_app_names.errors %}<div class="alert alert-danger">{{ form.include_app_names.errors }}</div>{% endif %}
<div class="form-group">
{{ form.exclude_app_names.label_tag }}
{{ form.exclude_app_names }}
{{ form.include_app_names.label_tag }}
{{ form.include_app_names }}
</div>
</div>
</div>

6
apimanager/metrics/templates/metrics/weekly_summary.html
View File

@ -51,10 +51,10 @@
</div>
</div>
<div class="col-xs-4 col-sm-3">
{% if form.exclude_app_names.errors %}<div class="alert alert-danger">{{ form.exclude_app_names.errors }}</div>{% endif %}
{% if form.include_app_names.errors %}<div class="alert alert-danger">{{ form.include_app_names.errors }}</div>{% endif %}
<div class="form-group">
{{ form.exclude_app_names.label_tag }}
{{ form.exclude_app_names }}
{{ form.include_app_names.label_tag }}
{{ form.include_app_names }}
</div>
</div>
</div>

6
apimanager/metrics/templates/metrics/yearly_summary.html
View File

@ -51,10 +51,10 @@
</div>
</div>
<div class="col-xs-4 col-sm-3">
{% if form.exclude_app_names.errors %}<div class="alert alert-danger">{{ form.exclude_app_names.errors }}</div>{% endif %}
{% if form.include_app_names.errors %}<div class="alert alert-danger">{{ form.include_app_names.errors }}</div>{% endif %}
<div class="form-group">
{{ form.exclude_app_names.label_tag }}
{{ form.exclude_app_names }}
{{ form.include_app_names.label_tag }}
{{ form.include_app_names }}
</div>
</div>
</div>

123
apimanager/metrics/views.py
View File

@ -11,7 +11,7 @@ from enum import Enum
from django.conf import settings
from apimanager import local_settings
from apimanager.settings import API_HOST, EXCLUDE_APPS, EXCLUDE_FUNCTIONS, EXCLUDE_URL_PATTERN, API_EXPLORER_APP_NAME, API_DATE_FORMAT, API_DATE_TIME_FORMAT
from apimanager.settings import API_HOST, EXCLUDE_APPS, EXCLUDE_FUNCTIONS, EXCLUDE_URL_PATTERN, API_EXPLORER_APP_NAME, API_DATE_FORMAT, API_DATE_TIME_FORMAT, DEBUG
from django.contrib import messages
from django.contrib.auth.mixins import LoginRequiredMixin
from django.views.generic import TemplateView
@ -33,6 +33,8 @@ import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
import statistics
import urllib.parse
CACHE_SETTING_URL_MSG = "The cache setting url is"
CACHE_SETTING_KEY_MSG = "The cache setting key is"
@ -146,10 +148,16 @@ class MetricsView(LoginRequiredMixin, TemplateView):
metrics = api.get(urlpath)
metrics = self.to_django(metrics['metrics'])
except APIError as err:
if DEBUG:
raise(err)
error_once_only(self.request, err)
except KeyError as err:
if DEBUG:
raise(err)
error_once_only(self.request, metrics['message'])
except Exception as err:
if DEBUG:
raise(err)
error_once_only(self.request, err)
return metrics
@ -246,14 +254,45 @@ class MonthlyMetricsSummaryView(LoginRequiredMixin, TemplateView):
params = '&'.join(params)
return params
def get_aggregate_metrics(self, from_date, to_date):
def get_app_name_parameters(self, include_app_names):
#if len(app_names) == 1:
# return
#1. Parse include_app_names create a list using a commo (,) separator
#2. Trim each word (remove space),
#3. Then is one word, &app_name = thing
#4. IF IT IS MORE than one word then return number of app without space
#5. return either &app_name=thing
#6. Or
#7. return &include_app_names=thing1,thing2,
#8. url encode
#app_names = []
#input_string = "simon says, foo, bar , App 2 "
input_string = include_app_names.strip()
result = ""
if input_string != "":
input_list = input_string.strip().split(",")
#print("input_list is:", input_list)
cleaned_list = [item.strip() for item in input_list]
#print("cleaned_list is: ", cleaned_list)
cleaned_string=', '.join([str(item) for item in cleaned_list])
#print("cleaned_string is:", cleaned_string)
url_encoded_string = urllib.parse.quote(cleaned_string)
#print("url_encoded_string is:", url_encoded_string)
if len(cleaned_list) == 0:
result = ""
elif len(cleaned_list) == 1:
result = "&app_name={}".format(url_encoded_string)
else:
result = "&include_app_names={}".format(url_encoded_string)
return result
def get_aggregate_metrics(self, from_date, to_date, include_app_names):
"""
Gets the metrics from the API, using given parameters,
There are different use cases, so we accept different parameters.
only_show_api_explorer_metrics has the default value False, because it is just used for app = API_Explorer.
"""
try:
print("get_app_name_parameters is: ", self.get_app_name_parameters(include_app_names))
api_calls_total = 0
average_response_time = 0
url_path = '/management/aggregate-metrics'
@ -264,7 +303,9 @@ class MonthlyMetricsSummaryView(LoginRequiredMixin, TemplateView):
# from_date, to_date, ",".join(local_settings.EXCLUDE_FUNCTIONS), ",".join(local_settings.EXCLUDE_URL_PATTERN))
#
#else:
url_path = url_path + '?from_date={}&to_date={}'.format(from_date, to_date)
url_path = url_path + '?from_date={}&to_date={}{}'.format(from_date, to_date, self.get_app_name_parameters(include_app_names))
#print("get_app_name_parameters(include_app_names) is:", self.get_app_name_parameters(include_app_names))
#print("url_path is: ", url_path)
cache_key = get_cache_key_for_current_call(self.request, url_path)
api_cache = None
try:
@ -284,8 +325,12 @@ class MonthlyMetricsSummaryView(LoginRequiredMixin, TemplateView):
api_calls_total, average_response_time, cache_key, from_date, metrics, to_date, url_path)
return api_calls_total, average_response_time, int(average_calls_per_day)
except APIError as err:
if DEBUG:
raise(err)
error_once_only(self.request, err)
except Exception as err:
if DEBUG:
raise(err)
error_once_only(self.request, err)
def get_internal_api_call_metrics(self, api_calls_total, average_response_time, cache_key, from_date, metrics,
@ -317,6 +362,8 @@ class MonthlyMetricsSummaryView(LoginRequiredMixin, TemplateView):
except APIError as err:
error_once_only(self.request, err)
except Exception as err:
if DEBUG:
raise(err)
error_once_only(self.request, err)
else:
urlpath = '/management/metrics/top-consumers?from_date={}&to_date={}&exclude_implemented_by_partial_functions={}&exclude_url_pattern={}'.format(
@ -326,8 +373,12 @@ class MonthlyMetricsSummaryView(LoginRequiredMixin, TemplateView):
apps = api.get(urlpath)
active_apps_list = list(apps['top_consumers'])
except APIError as err:
if DEBUG:
raise(err)
error_once_only(self.request, err)
except Exception as err:
if DEBUG:
raise(err)
error_once_only(self.request, err)
return active_apps_list
@ -391,12 +442,16 @@ class MonthlyMetricsSummaryView(LoginRequiredMixin, TemplateView):
LOGGER.warning('{0}: {1}'.format(CACHE_SETTING_URL_MSG, urlpath))
LOGGER.warning('{0}: {1}'.format(CACHE_SETTING_KEY_MSG, cache_key))
except APIError as err:
if DEBUG:
raise(err)
error_once_only(self.request, err)
except Exception as err:
if DEBUG:
raise(err)
error_once_only(self.request, err)
return apps_list
def calls_per_delta(self, from_date_string, to_date_string, **delta ):
def calls_per_delta(self, from_date_string, to_date_string, include_app_names, **delta ):
"""
return how many calls were made in total per given delta.
Here we need to convert date_string to datetime object, and calculate the dates.
@ -415,12 +470,14 @@ class MonthlyMetricsSummaryView(LoginRequiredMixin, TemplateView):
# here we need to first convert datetime object to String
form_date= from_datetime_object.strftime(API_DATE_FORMAT)
to_date= time_delta_in_loop.strftime(API_DATE_FORMAT)
aggregate_metrics = self.get_aggregate_metrics(form_date, to_date, )
aggregate_metrics = self.get_aggregate_metrics(form_date, to_date, include_app_names)
result = aggregate_metrics[0]
result_list_pure.append(result)
result_list.append('{} - {} # {}'.format(from_datetime_object, time_delta_in_loop, result))
date_list.append(from_datetime_object)
except Exception as err:
if DEBUG:
raise(err)
error_once_only(self.request, err)
break
@ -430,20 +487,20 @@ class MonthlyMetricsSummaryView(LoginRequiredMixin, TemplateView):
return (result_list, result_list_pure, date_list)
def calls_per_month(self,from_date, to_date):
def calls_per_month(self,from_date, to_date, include_app_names):
"""
Convenience function to print number of calls per month
It is actually 30 days, not a month
"""
calls_per_month_list, calls_per_month, month_list = self.calls_per_delta(from_date, to_date, days=30)
calls_per_month_list, calls_per_month, month_list = self.calls_per_delta(from_date, to_date, include_app_names, days=30)
return calls_per_month_list, calls_per_month, month_list
def calls_per_day(self,from_date, to_date):
def calls_per_day(self,from_date, to_date, include_app_names):
"""
Convenience function to print number of calls per day
"""
calls_per_day, calls_per_day_pure, date_list = self.calls_per_delta(from_date, to_date, days=1)
calls_per_day, calls_per_day_pure, date_list = self.calls_per_delta(from_date, to_date, include_app_names, days=1)
if len(calls_per_day) >= 90:
calls_per_day = calls_per_day[-90:]
@ -457,18 +514,18 @@ class MonthlyMetricsSummaryView(LoginRequiredMixin, TemplateView):
return calls_per_day, calls_per_day_pure, date_list
def calls_per_half_day(self,from_date):
def calls_per_half_day(self,from_date, include_app_names):
"""
Convenience function to print number of calls per half day
"""
return self.calls_per_delta(from_date, hours=12)
def calls_per_hour(self,from_date, to_date):
def calls_per_hour(self,from_date, to_date, include_app_names):
"""
Convenience function to print number of calls per hour
"""
calls_per_hour_list, calls_per_hour, hour_list = self.calls_per_delta(from_date, to_date, hours=1)
calls_per_hour_list, calls_per_hour, hour_list = self.calls_per_delta(from_date, to_date, include_app_names, hours=1)
return calls_per_hour_list, calls_per_hour, hour_list
def plot_line_chart(self, plot_data, date_month_list, period):
@ -602,6 +659,8 @@ class MonthlyMetricsSummaryView(LoginRequiredMixin, TemplateView):
except KeyError as err:
messages.error(self.request, 'KeyError: {}'.format(err))
except Exception as err:
if DEBUG:
raise(err)
error_once_only(self.request, err)
user_email_cansearchwarehouse = dict(zip(users_with_cansearchwarehouse, email_with_cansearchwarehouse))
@ -626,8 +685,12 @@ class MonthlyMetricsSummaryView(LoginRequiredMixin, TemplateView):
else:
data = data[data_key]
except APIError as err:
if DEBUG:
raise(err)
error_once_only(self.request, err)
except Exception as err:
if DEBUG:
raise(err)
error_once_only(self.request, err)
return data
@ -680,8 +743,12 @@ class MonthlyMetricsSummaryView(LoginRequiredMixin, TemplateView):
if "elasticSearchWarehouse" in api['Implemented_by_partial_function']:
top_warehouse_calls.append(api)
except APIError as err:
if DEBUG:
raise(err)
error_once_only(self.request, err)
except Exception as err:
if DEBUG:
raise(err)
error_once_only(self.request, err)
return top_warehouse_calls
@ -699,8 +766,12 @@ class MonthlyMetricsSummaryView(LoginRequiredMixin, TemplateView):
else:
top_apps_using_warehouse = top_apps_using_warehouse["top_consumers"][:2]
except APIError as err:
if DEBUG:
raise(err)
error_once_only(self.request, err)
except Exception as err:
if DEBUG:
raise(err)
error_once_only(self.request, err)
return top_apps_using_warehouse
@ -757,8 +828,12 @@ class MonthlyMetricsSummaryView(LoginRequiredMixin, TemplateView):
except APIError as err:
if DEBUG:
raise(err)
error_once_only(self.request, err)
except Exception as err:
if DEBUG:
raise(err)
error_once_only(self.request, 'Unknown Error. {}'.format(err))
if times_to_first_call:
@ -781,7 +856,7 @@ class MonthlyMetricsSummaryView(LoginRequiredMixin, TemplateView):
per_hour_chart=[]
if form.is_valid():
# = form.cleaned_data.get('include_obp_apps')
#exclude_app_names = form.cleaned_data.get("exclude_app_names")
include_app_names = form.cleaned_data.get("include_app_names")
#if exclude_app_names not in local_settings.EXCLUDE_APPS:
# error_once_only(self.request, "Invalid Exclude App Name, Please select" + str(local_settings.EXCLUDE_APPS) + "Anyone of these")
form_to_date_string = form.data['to_date']
@ -790,31 +865,31 @@ class MonthlyMetricsSummaryView(LoginRequiredMixin, TemplateView):
if (web_page_type == SummaryType.DAILY):
# for one day, the from_date is 1 day ago.
from_date = return_to_days_ago(to_date, 1)
calls_per_hour_list, calls_per_hour, hour_list = self.calls_per_hour(from_date, to_date)
calls_per_hour_list, calls_per_hour, hour_list = self.calls_per_hour(from_date, to_date, include_app_names)
per_hour_chart = self.plot_line_chart(calls_per_hour, hour_list, 'hour')
if (web_page_type == SummaryType.WEEKLY):
# for one month, the from_date is 7 days ago.
from_date = return_to_days_ago(to_date, 7)
calls_per_day_list, calls_per_day, date_list = self.calls_per_day(from_date, to_date)
calls_per_day_list, calls_per_day, date_list = self.calls_per_day(from_date, to_date, include_app_names)
per_day_chart = self.plot_line_chart(calls_per_day, date_list, "day")
if (web_page_type == SummaryType.MONTHLY):
# for one month, the from_date is 30 days ago.
from_date = return_to_days_ago(to_date, 30)
calls_per_day_list, calls_per_day, date_list = self.calls_per_day(from_date, to_date)
calls_per_day_list, calls_per_day, date_list = self.calls_per_day(from_date, to_date, include_app_names)
per_day_chart = self.plot_line_chart(calls_per_day, date_list, "day")
if (web_page_type == SummaryType.QUARTERLY):
# for one quarter, the from_date is 90 days ago.
from_date = (datetime.datetime.strptime(to_date, API_DATE_FORMAT) - timedelta(90)).strftime(API_DATE_FORMAT)
calls_per_month_list, calls_per_month, month_list = self.calls_per_month(from_date, to_date)
calls_per_month_list, calls_per_month, month_list = self.calls_per_month(from_date, to_date, include_app_names)
per_month_chart = self.plot_line_chart(calls_per_month, month_list, 'month')
if (web_page_type == SummaryType.YEARLY):
from_date = return_to_days_ago(to_date, 365)
#calls_per_month_list, calls_per_month, month_list = self.calls_per_month(, from_date, to_date)
calls_per_month_list, calls_per_month, month_list = self.calls_per_month(from_date, to_date)
calls_per_month_list, calls_per_month, month_list = self.calls_per_month(from_date, to_date, include_app_names)
per_month_chart = self.plot_line_chart(calls_per_month, month_list, "month")
if (web_page_type == SummaryType.CUSTOM):
@ -822,7 +897,7 @@ class MonthlyMetricsSummaryView(LoginRequiredMixin, TemplateView):
form_from_date_string = form.data['from_date_custom']
from_date = convert_form_date_to_obpapi_datetime_format(form_from_date_string)
#calls_per_day_list, calls_per_day, date_list = self.calls_per_day(, from_date, to_date)
calls_per_day_list, calls_per_day, date_list = self.calls_per_day(from_date, to_date)
calls_per_day_list, calls_per_day, date_list = self.calls_per_day(from_date, to_date,include_app_names)
per_day_chart = self.plot_line_chart(calls_per_day, date_list, "day")
api_host_name = API_HOST
@ -835,7 +910,7 @@ class MonthlyMetricsSummaryView(LoginRequiredMixin, TemplateView):
top_consumers = self.get_top_consumers(form.cleaned_data, from_date, to_date)
top_consumers_bar_chart = self.plot_topconsumer_bar_chart(top_consumers)
top_warehouse_calls = self.get_top_warehouse_calls(form.cleaned_data, from_date, to_date)
api_calls, average_response_time, average_calls_per_day = self.get_aggregate_metrics(from_date, to_date)
api_calls, average_response_time, average_calls_per_day = self.get_aggregate_metrics(from_date, to_date, include_app_names)
unique_app_names, number_of_apps_with_unique_app_name, number_of_apps_with_unique_developer_email = self.get_total_number_of_apps(
form.cleaned_data, from_date, to_date)
active_apps_list = self.get_active_apps(from_date, to_date)
@ -871,6 +946,8 @@ class MonthlyMetricsSummaryView(LoginRequiredMixin, TemplateView):
else:
error_once_only(self.request, str(form.errors))
except Exception as err:
if DEBUG:
raise(err)
error_once_only(self.request, err)
def _daily_and_weekly(self, web_page_type,to_date, per_hour_chart, per_day_chart, from_date):
if (web_page_type == SummaryType.DAILY):
@ -897,7 +974,7 @@ class MonthlyMetricsSummaryView(LoginRequiredMixin, TemplateView):
if (web_page_type == SummaryType.QUARTERLY):
# for one quarter, the from_date is 90 days ago.
from_date = (datetime.datetime.strptime(to_date, API_DATE_FORMAT) - timedelta(90)).strftime(API_DATE_FORMAT)
calls_per_month_list, calls_per_month, month_list = self.calls_per_month(from_date, to_date)
calls_per_month_list, calls_per_month, month_list = self.calls_per_month(from_date, to_date, include_app_names)
per_month_chart = self.plot_line_chart(calls_per_month, month_list, 'month')
return (from_date, per_day_chart, per_month_chart)
@ -905,7 +982,7 @@ class MonthlyMetricsSummaryView(LoginRequiredMixin, TemplateView):
def _yearly_and_custom(self, web_page_type,to_date, per_month_chart, per_day_chart, from_date):
if (web_page_type == SummaryType.YEARLY):
from_date = return_to_days_ago(to_date, 365)
calls_per_month_list, calls_per_month, month_list = self.calls_per_month(from_date, to_date)
calls_per_month_list, calls_per_month, month_list = self.calls_per_month(from_date, to_date, include_app_names)
per_month_chart = self.plot_line_chart(calls_per_month, month_list, "month")
if (web_page_type == SummaryType.CUSTOM):