From 150c243b2245580a18394bf28f3ddb4eb878aa35 Mon Sep 17 00:00:00 2001 From: Hongwei Date: Wed, 21 Oct 2020 14:55:33 +0200 Subject: [PATCH 01/15] feature/added the cache for getUser --- apimanager/base/context_processors.py | 84 ++++++++++++++++++--------- 1 file changed, 55 insertions(+), 29 deletions(-) diff --git a/apimanager/base/context_processors.py b/apimanager/base/context_processors.py index 44e616e..1cd6e7d 100644 --- a/apimanager/base/context_processors.py +++ b/apimanager/base/context_processors.py @@ -7,7 +7,7 @@ from django.conf import settings from django.contrib import messages from obp.api import API, APIError - +from django.core.cache import cache def api_root(request): """Returns the configured API_ROOT""" @@ -17,39 +17,65 @@ def api_root(request): def api_username(request): """Returns the API username/email of the logged-in user""" nametodisplay = 'not authenticated' - if request.user.is_authenticated: - try: - api = API(request.session.get('obp')) - data = api.get('/users/current') - username = data['username'] - email = data['email'] - provider = data['provider'] - if "google" in provider: - nametodisplay = email - elif "yahoo" in provider: - nametodisplay = email - else: - nametodisplay = username - except APIError as err: - messages.error(request, err) - except Exception as err: - messages.error(request, err) - return {'API_USERNAME': nametodisplay} + get_current_user_api_url = '/users/current' + #Here we can not get the user from obp-api side, so we use the django auth user id here. + cache_key_django_user_id = request.session._session.get('_auth_user_id') + cache_key = '{},{},{}'.format('api_username',get_current_user_api_url, cache_key_django_user_id) + apicaches=None + try: + apicaches=cache.get(cache_key) + except Exception as err: + apicaches=None + if not apicaches is None: + return apicaches + else: + if request.user.is_authenticated: + try: + api = API(request.session.get('obp')) + data = api.get(get_current_user_api_url) + username = data['username'] + email = data['email'] + provider = data['provider'] + if "google" in provider: + nametodisplay = email + elif "yahoo" in provider: + nametodisplay = email + else: + nametodisplay = username + apicaches=cache.set(cache_key, {'API_USERNAME': nametodisplay}) + except APIError as err: + messages.error(request, err) + except Exception as err: + messages.error(request, err) + return {'API_USERNAME': nametodisplay} def api_user_id(request): """Returns the API user id of the logged-in user""" user_id = 'not authenticated' - if request.user.is_authenticated: - try: - api = API(request.session.get('obp')) - data = api.get('/users/current') - user_id = data['user_id'] - except APIError as err: - messages.error(request, err) - except Exception as err: - messages.error(request, err) - return {'API_USER_ID': user_id} + get_current_user_api_url = '/users/current' + #Here we can not get the user from obp-api side, so we use the django auth user id here. + cache_key_django_user_id = request.session._session.get('_auth_user_id') + cache_key = '{},{},{}'.format('api_user_id',get_current_user_api_url, cache_key_django_user_id) + apicaches=None + try: + apicaches=cache.get(cache_key) + except Exception as err: + apicaches=None + if not apicaches is None: + return apicaches + else: + if request.user.is_authenticated: + try: + api = API(request.session.get('obp')) + data = api.get('/users/current') + user_id = data['user_id'] + apicaches=cache.set(cache_key, {'API_USER_ID': user_id}) + except APIError as err: + messages.error(request, err) + except Exception as err: + messages.error(request, err) + return {'API_USER_ID': user_id} def api_tester_url(request): From 7b3a9ebda501252e41d766f796270722a0323e11 Mon Sep 17 00:00:00 2001 From: Hongwei Date: Wed, 21 Oct 2020 16:07:18 +0200 Subject: [PATCH 02/15] feature/move the error handling to ErrorHandler --- apimanager/branches/views.py | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/apimanager/branches/views.py b/apimanager/branches/views.py index e27e9ed..94d0866 100644 --- a/apimanager/branches/views.py +++ b/apimanager/branches/views.py @@ -16,19 +16,6 @@ from obp.api import API, APIError from .forms import CreateBranchForm - -def error_once_only(request, err): - """ - Just add the error once - :param request: - :param err: - :return: - """ - storage = messages.get_messages(request) - if str(err) not in [str(m.message) for m in storage]: - messages.error(request, err) - - class IndexBranchesView(LoginRequiredMixin, FormView): """Index view for branches""" template_name = "branches/index.html" From 5112ca87e74965e53c0cf70f54496deca5965a15 Mon Sep 17 00:00:00 2001 From: Hongwei Date: Thu, 22 Oct 2020 10:49:27 +0200 Subject: [PATCH 03/15] feature/added the log for the error_once_only method --- apimanager/utils/ErrorHandler.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/apimanager/utils/ErrorHandler.py b/apimanager/utils/ErrorHandler.py index b8afc50..7b58f58 100644 --- a/apimanager/utils/ErrorHandler.py +++ b/apimanager/utils/ErrorHandler.py @@ -1,6 +1,6 @@ from django.contrib import messages import functools -from obp.api import API, APIError +from obp.api import API, APIError, LOGGER from django.http import JsonResponse import traceback @@ -11,6 +11,7 @@ def error_once_only(request, err): :param err: :return: """ + LOGGER.exception('error_once_only - Error Message: {}'.format(err)) storage = messages.get_messages(request) if str(err) not in [str(m.message) for m in storage]: messages.error(request, err) From f5aad88cd143fe60425c8bf3bc92c602303b023a Mon Sep 17 00:00:00 2001 From: Hongwei Date: Thu, 22 Oct 2020 14:01:40 +0200 Subject: [PATCH 04/15] feature/fixed the cache CACHES setting and set the default API_EXPLORER_APP_NAME --- apimanager/apimanager/settings.py | 33 +++++++++++++------------------ 1 file changed, 14 insertions(+), 19 deletions(-) diff --git a/apimanager/apimanager/settings.py b/apimanager/apimanager/settings.py index fd3970e..c618e76 100644 --- a/apimanager/apimanager/settings.py +++ b/apimanager/apimanager/settings.py @@ -71,6 +71,19 @@ MIDDLEWARE = [ 'django.middleware.clickjacking.XFrameOptionsMiddleware', ] +# TIMEOUT is 2592000 seconds = 60*60*24*30 (1 month) +# MAX_ENTRIES is 1000000 entities +CACHES = { + 'default': { + 'BACKEND': 'django.core.cache.backends.locmem.LocMemCache', + 'LOCATION': 'unique-snowflake', + 'TIMEOUT': 600000, + 'OPTIONS': { + 'MAX_ENTRIES': 10000000 + } + } +} + ROOT_URLCONF = 'apimanager.urls' TEMPLATES = [ @@ -228,14 +241,6 @@ DIRECTLOGIN_PATH = '/my/logins/direct' GATEWAYLOGIN_HAS_CBS = False - -CACHES = { - 'default': { - 'BACKEND': 'django.core.cache.backends.memcached.MemcachedCache', - 'LOCATION': 'unix:/tmp/memcached.sock', - } -} - # Use BOOTSTRAP3 if you are using Bootstrap 3 BOOTSTRAP4 = { 'include_jquery': True, @@ -248,17 +253,7 @@ EXCLUDE_FUNCTIONS = [] # Url Patterns to exclude when reqeust to OBP-API's api EXCLUDE_URL_PATTERN = [] # App Name to aggregate metrics -API_EXPLORER_APP_NAME = 'xxx' - -CACHES = { - 'default': { - 'BACKEND': 'django.core.cache.backends.locmem.LocMemCache', - 'LOCATION': 'unique-snowflake', - } -} -CACHE_DATEFORMAT = '%Y%m%d%H%M%S' -CACHE_TIME = 3600 -CACHE_TIME_SHORT = 300 +API_EXPLORER_APP_NAME = 'API Explorer' # Local settings can override anything in here try: From db2d6d467f2b28a7d0313c88a4e426b2468f0a84 Mon Sep 17 00:00:00 2001 From: Hongwei Date: Thu, 22 Oct 2020 14:03:03 +0200 Subject: [PATCH 05/15] bugfix/used the DateField for the Form fields --- apimanager/metrics/forms.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/apimanager/metrics/forms.py b/apimanager/metrics/forms.py index 17a0784..3d53e94 100644 --- a/apimanager/metrics/forms.py +++ b/apimanager/metrics/forms.py @@ -211,7 +211,7 @@ class ConnectorMetricsForm(MetricsForm): class CustomSummaryForm(forms.Form): - to_date = forms.DateTimeField( + to_date = forms.DateField( label='To Date', # input_formats=[settings.API_DATEFORMAT], # widget=forms.DateTimeInput( @@ -220,12 +220,12 @@ class CustomSummaryForm(forms.Form): # 'class': 'form-control', # } # ), - widget=DateTimePickerInput(format='%Y-%m-%d %H:%M:%S'), + widget=DatePickerInput(format='%Y-%m-%d'), required=True, - initial=str(datetime.now().strftime('%Y-%m-%d %H:00:00')), + initial=str(datetime.now().strftime('%Y-%m-%d')), ) - from_date_custom = forms.DateTimeField( + from_date_custom = forms.DateField( label='From Date', # input_formats=[settings.API_DATEFORMAT], # widget=forms.DateTimeInput( @@ -234,9 +234,9 @@ class CustomSummaryForm(forms.Form): # 'class': 'form-control', # } # ), - widget=DateTimePickerInput(format='%Y-%m-%d %H:%M:%S'), + widget=DatePickerInput(format='%Y-%m-%d'), required=True, - initial=(datetime.now() - timedelta(6)).strftime('%Y-%m-%d %H:00:00'), + initial=(datetime.now() - timedelta(6)).strftime('%Y-%m-%d'), ) include_obp_apps = forms.BooleanField(required=False) @@ -247,7 +247,7 @@ class CustomSummaryForm(forms.Form): class MetricsSummaryForm(forms.Form): - to_date = forms.DateTimeField( + to_date = forms.DateField( label='To Date', # input_formats=[settings.API_DATEFORMAT], # widget=forms.DateTimeInput( @@ -256,10 +256,10 @@ class MetricsSummaryForm(forms.Form): # 'class': 'form-control', # } # ), - widget=DateTimePickerInput(format='%Y-%m-%d %H:%M:%S'), + widget=DatePickerInput(format='%Y-%m-%d'), required=True, # initial=str(datetime.now().strftime('%Y-%m-%dT%H:%M:%S.%fZ')), - initial=str(datetime.now().strftime('%Y-%m-%d %H:00:00')), + initial=str(datetime.now().strftime('%Y-%m-%d')), ) include_obp_apps = forms.BooleanField(required=False) From 18079b2037126bae341d8fc00997742f3b5b572f Mon Sep 17 00:00:00 2001 From: Hongwei Date: Thu, 22 Oct 2020 14:03:32 +0200 Subject: [PATCH 06/15] feature/added the logging for the cache --- apimanager/base/context_processors.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/apimanager/base/context_processors.py b/apimanager/base/context_processors.py index 1cd6e7d..ec04f66 100644 --- a/apimanager/base/context_processors.py +++ b/apimanager/base/context_processors.py @@ -6,7 +6,7 @@ Context processors for base app from django.conf import settings from django.contrib import messages -from obp.api import API, APIError +from obp.api import API, APIError, LOGGER from django.core.cache import cache def api_root(request): @@ -43,6 +43,8 @@ def api_username(request): else: nametodisplay = username apicaches=cache.set(cache_key, {'API_USERNAME': nametodisplay}) + LOGGER.warning('The cache is setting try to api_user_name:') + LOGGER.warning('The cache is setting key is: {}'.format(cache_key)) except APIError as err: messages.error(request, err) except Exception as err: @@ -71,6 +73,8 @@ def api_user_id(request): data = api.get('/users/current') user_id = data['user_id'] apicaches=cache.set(cache_key, {'API_USER_ID': user_id}) + LOGGER.warning('The cache is setting try to api_user_id:') + LOGGER.warning('The cache is setting key is: {}'.format(cache_key)) except APIError as err: messages.error(request, err) except Exception as err: From 4c04363b1590de1954a375d90522f0ff92e5b09b Mon Sep 17 00:00:00 2001 From: Hongwei Date: Thu, 22 Oct 2020 14:04:00 +0200 Subject: [PATCH 07/15] feature/tweaked the summary.html remove the unused tag --- apimanager/metrics/templates/metrics/summary.html | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/apimanager/metrics/templates/metrics/summary.html b/apimanager/metrics/templates/metrics/summary.html index 0043e99..320cafe 100644 --- a/apimanager/metrics/templates/metrics/summary.html +++ b/apimanager/metrics/templates/metrics/summary.html @@ -55,12 +55,16 @@
{{ form.include_obp_apps }} Include System Calls - {{ excluded_functions }} - {{ excluded_url_pattern }}
{% if excluded_apps %} {{ excluded_apps }} {% endif %} + {% if excluded_functions %} + {{ excluded_functions }} + {% endif %} + {% if excluded_url_pattern %} + {{ excluded_url_pattern }} + {% endif %}
From 88c1c75318d9df78940e4e9b19150635987ccc3a Mon Sep 17 00:00:00 2001 From: Hongwei Date: Thu, 22 Oct 2020 14:04:25 +0200 Subject: [PATCH 08/15] feature/added the error handling for the get method --- apimanager/obp/api.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/apimanager/obp/api.py b/apimanager/obp/api.py index efcf578..d640d77 100644 --- a/apimanager/obp/api.py +++ b/apimanager/obp/api.py @@ -71,8 +71,11 @@ class API(object): Convenience call which uses API_ROOT from settings """ url = settings.API_ROOT + urlpath - response = self.call('GET', url) - return self.handle_response(response) + response = self.handle_response(self.call('GET', url)) + if response is not None and 'code' in response: + raise APIError(response['message']) + else: + return response def delete(self, urlpath): """ From 62189a95fd1c9463e9bc104af2ac5341f5736616 Mon Sep 17 00:00:00 2001 From: Hongwei Date: Thu, 22 Oct 2020 14:04:58 +0200 Subject: [PATCH 09/15] feature/used the utils and put the get_cache_key_for_current_call method there --- apimanager/{base => utils}/utils.py | 7 +++++++ 1 file changed, 7 insertions(+) rename apimanager/{base => utils}/utils.py (52%) diff --git a/apimanager/base/utils.py b/apimanager/utils/utils.py similarity index 52% rename from apimanager/base/utils.py rename to apimanager/utils/utils.py index 6c7419a..25c4c21 100644 --- a/apimanager/base/utils.py +++ b/apimanager/utils/utils.py @@ -5,6 +5,7 @@ Base utilities from datetime import datetime from django.contrib.humanize.templatetags.humanize import naturaltime +from base import context_processors def json_serial(obj): @@ -14,3 +15,9 @@ def json_serial(obj): serial = naturaltime(obj) return serial raise TypeError('Type not serializable') + +def get_cache_key_for_current_call(request, urlpath): + """we will generate the cache key by login username+urlpath + url path may contain lots of special characters, here we use the hash method first. + """ + return context_processors.api_username(request).get('API_USERNAME') + str(hash(urlpath)) \ No newline at end of file From 33ec1d14c2d459fb0497a44585fa35c3e8e2f98b Mon Sep 17 00:00:00 2001 From: Hongwei Date: Thu, 22 Oct 2020 14:19:16 +0200 Subject: [PATCH 10/15] feature/clean the get_aggregate_metrics method and set the proper cache for it. --- apimanager/metrics/views.py | 349 +++++++++++++----------------------- 1 file changed, 129 insertions(+), 220 deletions(-) diff --git a/apimanager/metrics/views.py b/apimanager/metrics/views.py index 0165109..c38b28d 100644 --- a/apimanager/metrics/views.py +++ b/apimanager/metrics/views.py @@ -9,11 +9,13 @@ import operator from datetime import datetime, timedelta from django.conf import settings -from apimanager.settings import API_HOST, EXCLUDE_APPS, EXCLUDE_FUNCTIONS, EXCLUDE_URL_PATTERN, API_EXPLORER_APP_NAME, API_DATEFORMAT,CACHE_DATEFORMAT,CACHE_TIME,CACHE_TIME_SHORT +from apimanager.settings import API_HOST, EXCLUDE_APPS, EXCLUDE_FUNCTIONS, EXCLUDE_URL_PATTERN, API_EXPLORER_APP_NAME, API_DATEFORMAT from django.contrib import messages from django.contrib.auth.mixins import LoginRequiredMixin from django.views.generic import TemplateView -from obp.api import API, APIError +from obp.api import API, APIError, LOGGER +from utils.ErrorHandler import error_once_only +from utils.utils import get_cache_key_for_current_call from .forms import APIMetricsForm, ConnectorMetricsForm, MetricsSummaryForm, CustomSummaryForm from pylab import * from django.core.cache import cache @@ -29,19 +31,6 @@ import base64 import matplotlib.pyplot as plt import statistics - -def error_once_only(request, err): - """ - Just add the error once - :param request: - :param err: - :return: - """ - storage = messages.get_messages(request) - if str(err) not in [str(m.message) for m in storage]: - messages.error(request, err) - - def get_random_color(to_hash): hashed = str(int(hashlib.md5(to_hash.encode('utf-8')).hexdigest(), 16)) r = int(hashed[0:3]) % 255 @@ -148,7 +137,7 @@ class MetricsView(LoginRequiredMixin, TemplateView): except KeyError as err: error_once_only(self.request, metrics['message']) except Exception as err: - error_once_only(self.request, 'Unknown Error. {}'.format(type(err).__name__)) + error_once_only(self.request, 'Unknown Error. {}'.format(err)) return metrics def get_context_data(self, **kwargs): @@ -246,87 +235,54 @@ class MetricsSummaryView(LoginRequiredMixin, TemplateView): return params - def get_aggregate_metrics(self, cleaned_data, from_date, to_date): + def get_aggregate_metrics(self, from_date, to_date, is_included_obp_apps, only_show_api_explorer_metrics = False): """ - Gets the metrics from the API, using given cleaned form data. + Gets the metrics from the API, using given parameters, + There are different use cases, so we accept different parameters. + only_show_api_explorer_metrics has the default value False, because it is just used for app = API_Explorer. """ - form = self.get_form() - metrics = [] api_calls_total = 0 average_response_time = 0 - - # If Include OBP Apps is selected - if cleaned_data.get('include_obp_apps'): - urlpath = '/management/aggregate-metrics?from_date={}&to_date={}'.format(from_date, to_date) - api = API(self.request.session.get('obp')) - try: - metrics = api.get(urlpath) - if metrics is not None and 'code' in metrics and metrics['code']==403: - error_once_only(self.request, metrics['message']) - # metrics = self.to_django(metrics) - else: - api_calls_total = metrics[0]["count"] - average_response_time = metrics[0]["average_response_time"] - except APIError as err: - error_once_only(self.request, err) - except Exception as err: - error_once_only(self.request, 'Unknown Error. {}'.format(type(err).__name__)) - + urlpath = '/management/aggregate-metrics' + if only_show_api_explorer_metrics: + urlpath = urlpath + '?from_date={}&to_date={}&app_name={}'.format(from_date, to_date, API_EXPLORER_APP_NAME) + elif ((not only_show_api_explorer_metrics) and (not is_included_obp_apps)): + urlpath = urlpath + '?from_date={}&to_date={}&exclude_app_names={}&exclude_implemented_by_partial_functions={}&exclude_url_pattern={}'.format( + from_date, to_date, ",".join(EXCLUDE_APPS),",".join(EXCLUDE_FUNCTIONS), ",".join(EXCLUDE_URL_PATTERN)) else: - urlpath = '/management/aggregate-metrics?from_date={}&to_date={}&exclude_app_names={}&exclude_implemented_by_partial_functions={}&exclude_url_pattern={}'.format( - from_date, to_date, ",".join(EXCLUDE_APPS), - ",".join(EXCLUDE_FUNCTIONS), ",".join(EXCLUDE_URL_PATTERN)) - api = API(self.request.session.get('obp')) + urlpath = urlpath + '?from_date={}&to_date={}'.format(from_date, to_date) + cache_key = get_cache_key_for_current_call(self.request, urlpath) + try: + apicaches = None try: - metrics = api.get(urlpath) - if metrics is not None and 'code' in metrics and metrics['code']==403: - error_once_only(self.request, metrics['message']) - # metrics = self.to_django(metrics) - else: - api_calls_total = metrics[0]["count"] - average_response_time = metrics[0]["average_response_time"] - except APIError as err: - error_once_only(self.request, err) + apicaches = cache.get(cache_key) except Exception as err: - error_once_only(self.request, 'Unknown Error. {}'.format(type(err).__name__)) + apicaches = None + if not apicaches is None: + api_calls_total = apicaches[0]["count"] + average_response_time = apicaches[0]["average_response_time"] + else: + api = API(self.request.session.get('obp')) + metrics = api.get(urlpath) + api_calls_total = metrics[0]["count"] + average_response_time = metrics[0]["average_response_time"] + apicaches = cache.set(cache_key, metrics) + LOGGER.warning('The cache is setting, url is: {}'.format(urlpath)) + LOGGER.warning('The cache is setting key is: {}'.format(cache_key)) + to_date = datetime.datetime.strptime(to_date, API_DATEFORMAT) + from_date = datetime.datetime.strptime(from_date, API_DATEFORMAT) + number_of_days = abs((to_date - from_date).days) + average_calls_per_day = api_calls_total / number_of_days + except APIError as err: + error_once_only(self.request, err) + except Exception as err: + error_once_only(self.request, 'Unknown Error. {}'.format(err)) - to_date = datetime.datetime.strptime(to_date, API_DATEFORMAT) - from_date = datetime.datetime.strptime(from_date, API_DATEFORMAT) - number_of_days = abs((to_date - from_date).days) - average_calls_per_day = api_calls_total / number_of_days - return api_calls_total, average_response_time, int(average_calls_per_day) def get_aggregate_metrics_api_explorer(self, from_date, to_date): - """ - Gets the metrics from the API, using given cleaned form data. - """ - form = self.get_form() - metrics = [] - api_calls_total = 0 - average_response_time = 0 - - urlpath = '/management/aggregate-metrics?from_date={}&to_date={}&app_name={}'.format(from_date, to_date, API_EXPLORER_APP_NAME) - api = API(self.request.session.get('obp')) - try: - metrics = api.get(urlpath) - if metrics is not None and 'code' in metrics and metrics['code']==403: - error_once_only(self.request, metrics['message']) - else: - api_calls_total = metrics[0]["count"] - average_response_time = metrics[0]["average_response_time"] - except APIError as err: - error_once_only(self.request, err) - except Exception as err: - error_once_only(self.request, "Unknown Error. {}".format(type(err).__name__)) - - to_date = datetime.datetime.strptime(to_date, API_DATEFORMAT) - from_date = datetime.datetime.strptime(from_date, API_DATEFORMAT) - number_of_days = abs((to_date - from_date).days) - average_calls_per_day = api_calls_total / number_of_days - - return api_calls_total, average_response_time, int(average_calls_per_day) + return self.get_aggregate_metrics(from_date, to_date, True, True) def get_active_apps(self, cleaned_data, from_date, to_date): apps = [] @@ -344,7 +300,7 @@ class MetricsSummaryView(LoginRequiredMixin, TemplateView): except APIError as err: error_once_only(self.request, err) except Exception as err: - error_once_only(self.request, 'Unknown Error. {}'.format(type(err).__name__)) + error_once_only(self.request, 'Unknown Error. {}'.format(err)) else: urlpath = '/management/metrics/top-consumers?from_date={}&to_date={}&exclude_app_names={}&exclude_implemented_by_partial_functions={}&exclude_url_pattern={}'.format( from_date, to_date, ",".join(EXCLUDE_APPS), ",".join(EXCLUDE_FUNCTIONS), ",".join(EXCLUDE_URL_PATTERN)) @@ -358,7 +314,7 @@ class MetricsSummaryView(LoginRequiredMixin, TemplateView): except APIError as err: error_once_only(self.request, err) except Exception as err: - error_once_only(self.request, 'Unknown Error. {}'.format(type(err).__name__)) + error_once_only(self.request, 'Unknown Error. {}'.format(err)) return active_apps_list @@ -368,11 +324,12 @@ class MetricsSummaryView(LoginRequiredMixin, TemplateView): apps_list = [] from_date = datetime.datetime.strptime(from_date, API_DATEFORMAT) to_date = datetime.datetime.strptime(to_date, API_DATEFORMAT) - urlpath='/management/consumers' + urlpath = '/management/consumers' api = API(self.request.session.get('obp')) + cache_key = get_cache_key_for_current_call(self.request, urlpath) apicaches=None try: - apicaches=cache.get('consumers,{}'.format(self.request.session.get('obp')['authenticator_kwargs']['token'])) + apicaches=cache.get(cache_key) except Exception as err: apicaches=None if not apicaches is None: @@ -384,11 +341,13 @@ class MetricsSummaryView(LoginRequiredMixin, TemplateView): error_once_only(self.request, apps['message']) else: apps_list = apps["consumers"] - cache.set('consumers,{}'.format(self.request.session.get('obp')['authenticator_kwargs']['token']),apps_list) + cache.set(cache_key, apps_list) + LOGGER.warning('The cache is setting, url is: {}'.format(urlpath)) + LOGGER.warning('The cache is setting key is: {}'.format(cache_key)) except APIError as err: error_once_only(self.request, err) except Exception as err: - error_once_only(self.request, 'Unknown Error. {}'.format(type(err).__name__)) + error_once_only(self.request, 'Unknown Error. {}'.format(err)) for app in apps_list: @@ -426,117 +385,55 @@ class MetricsSummaryView(LoginRequiredMixin, TemplateView): return unique_app_names, number_of_apps_with_unique_app_name, number_of_apps_with_unique_developer_email - def calls_per_delta(self, cleaned_data, from_date, to_date, **delta ): + def calls_per_delta(self, is_included_obp_apps, from_date_string, to_date_string, **delta ): """ - Prints how many calls were made in total per given delta + return how many calls were made in total per given delta. + Here we need to convert date_string to datetime object, and calculate the dates. """ - form = self.get_form() - to_date = datetime.datetime.strptime(to_date, API_DATEFORMAT) + + # we need to convert string to datetime object, then we can calculate the date + from_datetime_object = datetime.datetime.strptime(from_date_string, API_DATEFORMAT) + to_datetime_object = datetime.datetime.strptime(to_date_string , API_DATEFORMAT) + time_delta_in_loop = from_datetime_object + timedelta(**delta) - from_date = datetime.datetime.strptime(from_date, API_DATEFORMAT) - - date_from = from_date - date_to = from_date + timedelta(**delta) - - sum = 0 - metrics = [] result_list = [] result_list_pure = [] date_list = [] + while time_delta_in_loop <= to_datetime_object: + try: + # here we need to first convert datetime object to String + form_date= from_datetime_object.strftime(API_DATEFORMAT) + to_date= time_delta_in_loop.strftime(API_DATEFORMAT) + aggregate_metrics = self.get_aggregate_metrics(form_date, to_date, is_included_obp_apps) + result = aggregate_metrics[0] + result_list_pure.append(result) + result_list.append('{} - {} # {}'.format(from_datetime_object, time_delta_in_loop, result)) + date_list.append(from_datetime_object) + except Exception as err: + break + error_once_only(self.request, 'Unknown Error. {}'.format(err)) + + from_datetime_object = time_delta_in_loop + time_delta_in_loop = time_delta_in_loop + timedelta(**delta) - # If include_obp_apps is selected - if cleaned_data.get('include_obp_apps'): - while date_to <= to_date: - urlpath = '/management/aggregate-metrics?from_date={}&to_date={}'.format( - date_from.strftime(API_DATEFORMAT), date_to.strftime(API_DATEFORMAT)) - apicaches=None - try: - apicaches=cache.get('aggregate-metrics,{},{},{}'.format(self.request.session.get('obp')['authenticator_kwargs']['token'],date_from.strftime(CACHE_DATEFORMAT), date_to.strftime(CACHE_DATEFORMAT))) - except Exception as err: - apicaches=None - if not apicaches is None: - result = apicaches - result_list_pure.append(result) - result_list.append('{} - {} # {}'.format(date_from, date_to, result)) - sum += result - else: - api = API(self.request.session.get('obp')) - try: - metrics = api.get(urlpath) - if metrics is not None and 'code' in metrics and metrics['code'] == 403: - error_once_only(self.request, metrics['message']) - if(metrics['message'].startswith('OBP-20006')): - break - else: - result = metrics[0]["count"] - result_list_pure.append(result) - result_list.append('{} - {} # {}'.format(date_from, date_to, result)) - sum += result - - cache.set('aggregate-metrics,{},{},{}'.format(self.request.session.get('obp')['authenticator_kwargs']['token'],date_from.strftime(CACHE_DATEFORMAT), date_to.strftime(CACHE_DATEFORMAT)),result,CACHE_TIME if date_from.minute==0 and date_from.second==0 and date_to.minute==0 and date_to.second==0 else CACHE_TIME_SHORT) - except APIError as err: - error_once_only(self.request, err) - except Exception as err: - error_once_only(self.request, 'Unknown Error. {}'.format(type(err).__name__)) + return (result_list, result_list_pure, date_list) + - date_list.append(date_from) - date_from = date_to - date_to = date_to + timedelta(**delta) - else: - while date_to <= to_date: - urlpath = '/management/aggregate-metrics?from_date={}&to_date={}&exclude_app_names={}'.format( - date_from.strftime(API_DATEFORMAT), date_to.strftime(API_DATEFORMAT), ",".join(EXCLUDE_APPS)) - apicaches=None - try: - apicaches=cache.get('aggregate-metrics,{},{},{},{}'.format(self.request.session.get('obp')['authenticator_kwargs']['token'],date_from.strftime(CACHE_DATEFORMAT), date_to.strftime(CACHE_DATEFORMAT),",".join(EXCLUDE_APPS))) - except Exception as err: - apicaches=None - if not apicaches is None: - result = apicaches - result_list_pure.append(result) - result_list.append('{} - {} # {}'.format(date_from, date_to, result)) - sum += result - else: - api = API(self.request.session.get('obp')) - try: - metrics = api.get(urlpath) - if metrics is not None and 'code' in metrics and metrics['code'] == 403: - error_once_only(self.request, metrics['message']) - if(metrics['message'].startswith('OBP-20006')): - break - else: - result = metrics[0]["count"] - result_list_pure.append(result) - result_list.append('{} - {} # {}'.format(date_from, date_to, result)) - sum += result - - cache.set('aggregate-metrics,{},{},{},{}'.format(self.request.session.get('obp')['authenticator_kwargs']['token'],date_from.strftime(CACHE_DATEFORMAT), date_to.strftime(CACHE_DATEFORMAT),",".join(EXCLUDE_APPS)),result,CACHE_TIME if date_from.minute==0 and date_from.second==0 and date_to.minute==0 and date_to.second==0 else CACHE_TIME_SHORT) - except APIError as err: - error_once_only(self.request, err) - except Exception as err: - error_once_only(self.request, 'Unknown Error. {}'.format(type(err).__name__)) - - date_list.append(date_from) - date_from = date_to - date_to = date_to + timedelta(**delta) - return result_list, result_list_pure, date_list - - - def calls_per_month(self, cleaned_data, from_date, to_date): + def calls_per_month(self, is_included_obp_apps, from_date, to_date): """ Convenience function to print number of calls per month It is actually 30 days, not a month """ - calls_per_month_list, calls_per_month, month_list = self.calls_per_delta(cleaned_data, from_date, to_date, days=30) + calls_per_month_list, calls_per_month, month_list = self.calls_per_delta(is_included_obp_apps, from_date, to_date, days=30) return calls_per_month_list, calls_per_month, month_list - def calls_per_day(self, cleaned_data, from_date, to_date): + def calls_per_day(self, is_included_obp_apps, from_date, to_date): """ Convenience function to print number of calls per day """ index = [] - calls_per_day, calls_per_day_pure, date_list = self.calls_per_delta(cleaned_data, from_date, to_date, days=1) + calls_per_day, calls_per_day_pure, date_list = self.calls_per_delta(is_included_obp_apps, from_date, to_date, days=1) if len(calls_per_day) >= 90: calls_per_day = calls_per_day[-90:] @@ -550,18 +447,18 @@ class MetricsSummaryView(LoginRequiredMixin, TemplateView): return calls_per_day, calls_per_day_pure, date_list - def calls_per_half_day(self, cleaned_data, from_date): + def calls_per_half_day(self, is_included_obp_apps, from_date): """ Convenience function to print number of calls per half day """ - return self.calls_per_delta(cleaned_data, from_date, hours=12) + return self.calls_per_delta(is_included_obp_apps, from_date, hours=12) - def calls_per_hour(self, cleaned_data, from_date, to_date): + def calls_per_hour(self, is_included_obp_apps, from_date, to_date): """ Convenience function to print number of calls per hour """ - calls_per_hour_list, calls_per_hour, hour_list = self.calls_per_delta(cleaned_data, from_date, to_date, hours=1) + calls_per_hour_list, calls_per_hour, hour_list = self.calls_per_delta(is_included_obp_apps, from_date, to_date, hours=1) return calls_per_hour_list, calls_per_hour, hour_list def plot_line_chart(self, plot_data, date_month_list, period): @@ -683,7 +580,7 @@ class MetricsSummaryView(LoginRequiredMixin, TemplateView): except APIError as err: error_once_only(self.request, err) except Exception as err: - error_once_only(self.request, 'Unknown Error. {}'.format(type(err).__name__)) + error_once_only(self.request, 'Unknown Error. {}'.format(err)) else: try: @@ -696,7 +593,7 @@ class MetricsSummaryView(LoginRequiredMixin, TemplateView): except KeyError as err: messages.error(self.request, 'KeyError: {}'.format(err)) except Exception as err: - error_once_only(self.request, 'Unknown Error. {}'.format(type(err).__name__)) + error_once_only(self.request, 'Unknown Error. {}'.format(err)) user_email_cansearchwarehouse = dict(zip(users_with_cansearchwarehouse, email_with_cansearchwarehouse)) number_of_users_with_cansearchwarehouse = len(user_email_cansearchwarehouse) @@ -718,7 +615,7 @@ class MetricsSummaryView(LoginRequiredMixin, TemplateView): except APIError as err: error_once_only(self.request, err) except Exception as err: - error_once_only(self.request, 'Unknown Error. {}'.format(type(err).__name__)) + error_once_only(self.request, 'Unknown Error. {}'.format(err)) else: urlpath = '/management/metrics/top-apis?from_date={}&to_date={}&exclude_app_names={}&exclude_implemented_by_partial_functions={}&exclude_url_pattern={}'.format( from_date, to_date, ",".join(EXCLUDE_APPS), ",".join(EXCLUDE_FUNCTIONS), ",".join(EXCLUDE_URL_PATTERN)) @@ -733,7 +630,7 @@ class MetricsSummaryView(LoginRequiredMixin, TemplateView): except APIError as err: error_once_only(self.request, err) except Exception as err: - error_once_only(self.request, 'Unknown Error. {}'.format(type(err).__name__)) + error_once_only(self.request, 'Unknown Error. {}'.format(err)) for api in top_apis: if api['Implemented_by_partial_function'] == "": @@ -762,7 +659,7 @@ class MetricsSummaryView(LoginRequiredMixin, TemplateView): except APIError as err: error_once_only(self.request, err) except Exception as err: - error_once_only(self.request, 'Unknown Error. {}'.format(type(err).__name__)) + error_once_only(self.request, 'Unknown Error. {}'.format(err)) else: urlpath = '/management/metrics/top-consumers?from_date={}&to_date={}&exclude_app_names={}&exclude_implemented_by_partial_functions={}&exclude_url_pattern={}'.format( from_date, to_date, ",".join(EXCLUDE_APPS), ",".join(EXCLUDE_FUNCTIONS), ",".join(EXCLUDE_URL_PATTERN)) @@ -777,7 +674,7 @@ class MetricsSummaryView(LoginRequiredMixin, TemplateView): except APIError as err: error_once_only(self.request, err) except Exception as err: - error_once_only(self.request, 'Unknown Error. {}'.format(type(err).__name__)) + error_once_only(self.request, 'Unknown Error. {}'.format(err)) for consumer in top_consumers: if consumer['app_name'] == "": top_consumers.remove(consumer) @@ -797,7 +694,7 @@ class MetricsSummaryView(LoginRequiredMixin, TemplateView): except APIError as err: error_once_only(self.request, err) except Exception as err: - error_once_only(self.request, 'Unknown Error. {}'.format(type(err).__name__)) + error_once_only(self.request, 'Unknown Error. {}'.format(err)) return top_warehouse_calls def get_top_apps_using_warehouse(self, from_date, to_date): @@ -817,7 +714,7 @@ class MetricsSummaryView(LoginRequiredMixin, TemplateView): except APIError as err: error_once_only(self.request, err) except Exception as err: - error_once_only(self.request, 'Unknown Error. {}'.format(type(err).__name__)) + error_once_only(self.request, 'Unknown Error. {}'.format(err)) return top_apps_using_warehouse @@ -830,9 +727,10 @@ class MetricsSummaryView(LoginRequiredMixin, TemplateView): apps_list = [] urlpath_consumers = '/management/consumers' + cache_key = get_cache_key_for_current_call(self.request, urlpath_consumers) apicaches=None try: - apicaches=cache.get('consumers,{}'.format(self.request.session.get('obp')['authenticator_kwargs']['token'])) + apicaches=cache.get(cache_key) except Exception as err: apicaches=None if not apicaches is None: @@ -845,11 +743,13 @@ class MetricsSummaryView(LoginRequiredMixin, TemplateView): error_once_only(self.request, apps['message']) else: apps_list = apps["consumers"] - cache.set('consumers,{}'.format(self.request.session.get('obp')['authenticator_kwargs']['token']),apps_list) + cache.set(cache_key, apps_list) + LOGGER.warning('The cache is setting, url is: {}'.format(urlpath_consumers)) + LOGGER.warning('The cache is setting key is: {}'.format(cache_key)) except APIError as err: error_once_only(self.request, err) except Exception as err: - error_once_only(self.request, 'Unknown Error. {}'.format(type(err).__name__)) + error_once_only(self.request, 'Unknown Error. {}'.format(err)) for app in apps_list: created_date = datetime.datetime.strptime(app['created'], '%Y-%m-%dT%H:%M:%SZ') @@ -865,11 +765,12 @@ class MetricsSummaryView(LoginRequiredMixin, TemplateView): for app in new_apps_list: urlpath_metrics = '/management/metrics?from_date={}&to_date={}&consumer_id={}&sort_by={}&direction={}&limit={}'.format( from_date, to_date, app['consumer_id'], 'date', 'asc', '1') + cache_key = get_cache_key_for_current_call(self.request, urlpath_metrics) api = API(self.request.session.get('obp')) try: apicaches=None try: - apicaches=cache.get('metrics,{},{},{},{}'.format(self.request.session.get('obp')['authenticator_kwargs']['token'],app['consumer_id'],strfrom_date.strftime(CACHE_DATEFORMAT), strto_date.strftime(CACHE_DATEFORMAT))) + apicaches=cache.get(cache_key) except Exception as err: apicaches=None metrics=[] @@ -885,7 +786,9 @@ class MetricsSummaryView(LoginRequiredMixin, TemplateView): metrics = [] else: metrics = list(metrics['metrics']) - cache.set('metrics,{},{},{},{}'.format(self.request.session.get('obp')['authenticator_kwargs']['token'],app['consumer_id'],strfrom_date.strftime(CACHE_DATEFORMAT), strto_date.strftime(CACHE_DATEFORMAT)),metrics,CACHE_TIME if strfrom_date.minute==0 and strfrom_date.second==0 and strto_date.minute==0 and strto_date.second==0 else CACHE_TIME_SHORT) + cache.set(cache_key, metrics) + LOGGER.warning('The cache is setting, url is: {}'.format(urlpath_metrics)) + LOGGER.warning('The cache is setting key is: {}'.format(cache_key)) if metrics: time_difference = datetime.datetime.strptime(metrics[0]['date'], '%Y-%m-%dT%H:%M:%S.%fZ') - datetime.datetime.strptime(app['created'], '%Y-%m-%dT%H:%M:%SZ') times_to_first_call.append(time_difference.total_seconds()) @@ -894,7 +797,7 @@ class MetricsSummaryView(LoginRequiredMixin, TemplateView): except APIError as err: error_once_only(self.request, err) except Exception as err: - error_once_only(self.request, 'Unknown Error. {}'.format(type(err).__name__)) + error_once_only(self.request, 'Unknown Error. {}'.format(err)) if times_to_first_call: median = statistics.median(times_to_first_call) @@ -918,15 +821,16 @@ class MetricsSummaryView(LoginRequiredMixin, TemplateView): user_email_cansearchwarehouse, number_of_users_with_cansearchwarehouse = self.get_users_cansearchwarehouse() median_time_to_first_api_call = self.median_time_to_first_api_call(from_date, to_date) if form.is_valid(): + is_included_obp_apps = form.cleaned_data.get('include_obp_apps') top_apis = self.get_top_apis(form.cleaned_data, from_date, to_date) top_apis_bar_chart = self.plot_bar_chart(top_apis) top_consumers=self.get_top_consumers(form.cleaned_data, from_date, to_date) top_consumers_bar_chart = self.plot_topconsumer_bar_chart(top_consumers) top_warehouse_calls = self.get_top_warehouse_calls(form.cleaned_data, from_date, to_date) - api_calls, average_response_time, average_calls_per_day = self.get_aggregate_metrics(form.cleaned_data, from_date, to_date) + api_calls, average_response_time, average_calls_per_day = self.get_aggregate_metrics(from_date, to_date, is_included_obp_apps) calls_by_api_explorer, average_response_time_api_explorer, average_calls_per_day_api_explorer = self.get_aggregate_metrics_api_explorer(from_date, to_date) #calls_per_month_list, calls_per_month, date_list = self.calls_per_month(form.cleaned_data, from_date, to_date) - calls_per_day_list, calls_per_day, date_list = self.calls_per_day(form.cleaned_data, from_date, to_date) + calls_per_day_list, calls_per_day, date_list = self.calls_per_day(is_included_obp_apps, from_date, to_date) per_day_chart = self.plot_line_chart(calls_per_day, date_list, "day") unique_app_names, number_of_apps_with_unique_app_name, number_of_apps_with_unique_developer_email = self.get_total_number_of_apps(form.cleaned_data, from_date, to_date) active_apps_list = self.get_active_apps(form.cleaned_data, from_date, to_date) @@ -963,7 +867,7 @@ class YearlySummaryView(MetricsSummaryView): def get_context_data(self, **kwargs): form = self.get_form() - to_date = datetime.datetime.strptime(form.data['to_date'], '%Y-%m-%d %H:%M:%S') + to_date = datetime.datetime.strptime(form.data['to_date'], '%Y-%m-%d') to_date = to_date.strftime(API_DATEFORMAT) from_date = (datetime.datetime.strptime(to_date, API_DATEFORMAT) - timedelta(365)).strftime(API_DATEFORMAT) @@ -974,9 +878,10 @@ class YearlySummaryView(MetricsSummaryView): median_time_to_first_api_call = self.median_time_to_first_api_call(from_date, to_date) if form.is_valid(): - api_calls, average_response_time, average_calls_per_day = self.get_aggregate_metrics(form.cleaned_data, from_date, to_date) + is_included_obp_apps = form.cleaned_data.get('include_obp_apps') + api_calls, average_response_time, average_calls_per_day = self.get_aggregate_metrics(from_date, to_date, is_included_obp_apps) calls_by_api_explorer, average_response_time_api_explorer, average_calls_per_day_api_explorer = self.get_aggregate_metrics_api_explorer(from_date, to_date) - calls_per_month_list, calls_per_month, month_list = self.calls_per_month(form.cleaned_data, from_date, to_date) + calls_per_month_list, calls_per_month, month_list = self.calls_per_month(is_included_obp_apps, from_date, to_date) per_month_chart = self.plot_line_chart(calls_per_month, month_list, "month") unique_app_names, number_of_apps_with_unique_app_name, number_of_apps_with_unique_developer_email = self.get_total_number_of_apps(form.cleaned_data, from_date, to_date) active_apps_names = self.get_active_apps(form.cleaned_data, from_date, to_date) @@ -1019,7 +924,7 @@ class QuarterlySummaryView(MetricsSummaryView): def get_context_data(self, **kwargs): form = self.get_form() - to_date = datetime.datetime.strptime(form.data['to_date'], '%Y-%m-%d %H:%M:%S') + to_date = datetime.datetime.strptime(form.data['to_date'], '%Y-%m-%d') to_date = to_date.strftime(API_DATEFORMAT) from_date = (datetime.datetime.strptime(to_date, API_DATEFORMAT) - timedelta(90)).strftime(API_DATEFORMAT) @@ -1030,10 +935,11 @@ class QuarterlySummaryView(MetricsSummaryView): median_time_to_first_api_call = self.median_time_to_first_api_call(from_date, to_date) if form.is_valid(): - api_calls, average_response_time, average_calls_per_day = self.get_aggregate_metrics(form.cleaned_data, from_date, to_date) + is_included_obp_apps = form.cleaned_data.get('include_obp_apps') + api_calls, average_response_time, average_calls_per_day = self.get_aggregate_metrics(from_date, to_date, is_included_obp_apps) calls_by_api_explorer, average_response_time_api_explorer, average_calls_per_day_api_explorer = self.get_aggregate_metrics_api_explorer(from_date, to_date) - calls_per_month_list, calls_per_month, month_list = self.calls_per_month(form.cleaned_data, from_date, to_date) - calls_per_day_list, calls_per_day, date_list = self.calls_per_day(form.cleaned_data, from_date, to_date) + calls_per_month_list, calls_per_month, month_list = self.calls_per_month(is_included_obp_apps, from_date, to_date) + calls_per_day_list, calls_per_day, date_list = self.calls_per_day(is_included_obp_apps, from_date, to_date) per_month_chart = self.plot_line_chart(calls_per_month, month_list, 'month') per_day_chart = self.plot_line_chart(calls_per_day, date_list, 'day') unique_app_names, number_of_apps_with_unique_app_name, number_of_apps_with_unique_developer_email = self.get_total_number_of_apps(form.cleaned_data, from_date, to_date) @@ -1079,7 +985,7 @@ class WeeklySummaryView(MetricsSummaryView): def get_context_data(self, **kwargs): form = self.get_form() - to_date = datetime.datetime.strptime(form.data['to_date'], '%Y-%m-%d %H:%M:%S') + to_date = datetime.datetime.strptime(form.data['to_date'], '%Y-%m-%d') to_date = to_date.strftime(API_DATEFORMAT) from_date = (datetime.datetime.strptime(to_date, API_DATEFORMAT) - timedelta(7)).strftime(API_DATEFORMAT) @@ -1091,9 +997,10 @@ class WeeklySummaryView(MetricsSummaryView): median_time_to_first_api_call = self.median_time_to_first_api_call(from_date, to_date) if form.is_valid(): - api_calls, average_response_time, average_calls_per_day = self.get_aggregate_metrics(form.cleaned_data, from_date, to_date) + is_included_obp_apps = form.cleaned_data.get('include_obp_apps') + api_calls, average_response_time, average_calls_per_day = self.get_aggregate_metrics(from_date, to_date, is_included_obp_apps) calls_by_api_explorer, average_response_time_api_explorer, average_calls_per_day_api_explorer = self.get_aggregate_metrics_api_explorer(from_date, to_date) - calls_per_day_list, calls_per_day, date_list = self.calls_per_day(form.cleaned_data, from_date, to_date) + calls_per_day_list, calls_per_day, date_list = self.calls_per_day(is_included_obp_apps, from_date, to_date) per_day_chart = self.plot_line_chart(calls_per_day, date_list, 'day') unique_app_names, number_of_apps_with_unique_app_name, number_of_apps_with_unique_developer_email = self.get_total_number_of_apps(form.cleaned_data, from_date, to_date) active_apps_names = self.get_active_apps(form.cleaned_data, from_date, to_date) @@ -1137,7 +1044,7 @@ class DailySummaryView(MetricsSummaryView): def get_context_data(self, **kwargs): form = self.get_form() - to_date = datetime.datetime.strptime(form.data['to_date'], '%Y-%m-%d %H:%M:%S') + to_date = datetime.datetime.strptime(form.data['to_date'], '%Y-%m-%d') to_date = to_date.strftime(API_DATEFORMAT) from_date = (datetime.datetime.strptime(to_date, API_DATEFORMAT) - timedelta(1)).strftime(API_DATEFORMAT) @@ -1148,9 +1055,10 @@ class DailySummaryView(MetricsSummaryView): median_time_to_first_api_call = self.median_time_to_first_api_call(from_date, to_date) if form.is_valid(): - api_calls, average_response_time, average_calls_per_day = self.get_aggregate_metrics(form.cleaned_data, from_date, to_date) + is_included_obp_apps = form.cleaned_data.get('include_obp_apps') + api_calls, average_response_time, average_calls_per_day = self.get_aggregate_metrics(from_date, to_date, is_included_obp_apps) calls_by_api_explorer, average_response_time_api_explorer, average_calls_per_day_api_explorer = self.get_aggregate_metrics_api_explorer(from_date, to_date) - calls_per_hour_list, calls_per_hour, hour_list = self.calls_per_hour(form.cleaned_data, from_date, to_date) + calls_per_hour_list, calls_per_hour, hour_list = self.calls_per_hour(is_included_obp_apps, from_date, to_date) per_hour_chart = self.plot_line_chart(calls_per_hour, hour_list, 'hour') # calls_per_hour_list, calls_per_hour = self.calls_per_hour(form.cleaned_data, from_date) # per_hour_chart = self.get_per_hour_chart(form.cleaned_data, from_date) @@ -1200,10 +1108,10 @@ class CustomSummaryView(MetricsSummaryView): def get_context_data(self, **kwargs): form = self.get_form() - to_date = datetime.datetime.strptime(form.data['to_date'], '%Y-%m-%d %H:%M:%S') + to_date = datetime.datetime.strptime(form.data['to_date'], '%Y-%m-%d') to_date = to_date.strftime(API_DATEFORMAT) - from_date = datetime.datetime.strptime(form.data['from_date_custom'], '%Y-%m-%d %H:%M:%S') + from_date = datetime.datetime.strptime(form.data['from_date_custom'], '%Y-%m-%d') from_date = from_date.strftime(API_DATEFORMAT) context = super(MetricsSummaryView, self).get_context_data(**kwargs) api_host_name = API_HOST @@ -1214,9 +1122,10 @@ class CustomSummaryView(MetricsSummaryView): median_time_to_first_api_call = self.median_time_to_first_api_call(from_date, to_date) if form.is_valid(): - api_calls, average_response_time, average_calls_per_day = self.get_aggregate_metrics(form.cleaned_data, from_date, to_date) + is_included_obp_apps = form.cleaned_data.get('include_obp_apps') + api_calls, average_response_time, average_calls_per_day = self.get_aggregate_metrics(from_date, to_date, is_included_obp_apps) calls_by_api_explorer, average_response_time_api_explorer, average_calls_per_day_api_explorer = self.get_aggregate_metrics_api_explorer(from_date, to_date) - calls_per_day_list, calls_per_day, date_list = self.calls_per_day(form.cleaned_data, from_date, to_date) + calls_per_day_list, calls_per_day, date_list = self.calls_per_day(is_included_obp_apps, from_date, to_date) per_day_chart = self.plot_line_chart(calls_per_day, date_list, 'day') unique_app_names, number_of_apps_with_unique_app_name, number_of_apps_with_unique_developer_email = self.get_total_number_of_apps(form.cleaned_data, from_date, to_date) active_apps_names = self.get_active_apps(form.cleaned_data, from_date, to_date) From ef428790d127a4f06b17ef749fe8e8f57ea21f3d Mon Sep 17 00:00:00 2001 From: Hongwei Date: Fri, 23 Oct 2020 10:59:09 +0200 Subject: [PATCH 11/15] feature/fixed get_aggregate_metrics method for dayly,monthly,yearly,custom htmls --- apimanager/apimanager/settings.py | 3 + apimanager/base/utils.py | 73 +++ apimanager/dynamicendpoints/views.py | 2 +- apimanager/methodrouting/views.py | 3 +- apimanager/metrics/forms.py | 4 +- .../{summary.html => monthly_summary.html} | 0 .../templates/metrics/quarterly_summary.html | 8 +- apimanager/metrics/urls.py | 6 +- apimanager/metrics/views.py | 539 ++++++------------ apimanager/utils/ErrorHandler.py | 35 -- apimanager/utils/__init__.py | 0 apimanager/utils/utils.py | 23 - apimanager/webui/views.py | 2 +- 13 files changed, 255 insertions(+), 443 deletions(-) create mode 100644 apimanager/base/utils.py rename apimanager/metrics/templates/metrics/{summary.html => monthly_summary.html} (100%) delete mode 100644 apimanager/utils/ErrorHandler.py delete mode 100644 apimanager/utils/__init__.py delete mode 100644 apimanager/utils/utils.py diff --git a/apimanager/apimanager/settings.py b/apimanager/apimanager/settings.py index c618e76..cff2bd0 100644 --- a/apimanager/apimanager/settings.py +++ b/apimanager/apimanager/settings.py @@ -214,6 +214,9 @@ API_DATETIMEFORMAT = '%Y-%m-%dT%H:%M:%SZ' #Map Java: yyyy-MM-dd'T'HH:mm:ss.SSS'Z' API_DATEFORMAT = '%Y-%m-%dT%H:%M:%S.%fZ' +# the API_Manager the web form date format, eg: 2020-10-11 +API_MANAGER_DATE_FORMAT= '%Y-%m-%d' + API_HOST = 'http://127.0.0.1:8080' API_BASE_PATH = '/obp/v' diff --git a/apimanager/base/utils.py b/apimanager/base/utils.py new file mode 100644 index 0000000..54df439 --- /dev/null +++ b/apimanager/base/utils.py @@ -0,0 +1,73 @@ +# -*- coding: utf-8 -*- +""" +Base utilities +""" +from django.contrib.humanize.templatetags.humanize import naturaltime +from datetime import datetime, timedelta +from apimanager.settings import API_DATEFORMAT, API_MANAGER_DATE_FORMAT +from base import context_processors +from django.contrib import messages +import functools +from obp.api import APIError, LOGGER +from django.http import JsonResponse +import traceback + +def json_serial(obj): + """JSON serializer for objects not serializable by default json code""" + + if isinstance(obj, datetime): + serial = naturaltime(obj) + return serial + raise TypeError('Type not serializable') + +def get_cache_key_for_current_call(request, urlpath): + """we will generate the cache key by login username+urlpath + url path may contain lots of special characters, here we use the hash method first. + """ + return context_processors.api_username(request).get('API_USERNAME') + str(hash(urlpath)) + + +def error_once_only(request, err): + """ + Just add the error once + :param request: + :param err: + :return: + """ + LOGGER.exception('error_once_only - Error Message: {}'.format(err)) + storage = messages.get_messages(request) + if str(err) not in [str(m.message) for m in storage]: + messages.error(request, err) + +def exception_handle(fn): + @functools.wraps(fn) + def wrapper(request, *args, **kwargs): + try: + result = fn(request, *args, **kwargs) + if isinstance(result,dict) and 'code' in result and result['code'] >= 400: + error_once_only(request, result['message']) + else: + msg = 'Submitted!' + messages.success(request, msg) + except APIError as err: + error_once_only(request, APIError(Exception("OBP-API server is not running or do not response properly. " + "Please check OBP-API server. Details: " + str(err)))) + except Exception as err: + error_once_only(request, "Unknown Error. Details: " + str(err)) + return JsonResponse({'state': True}) + return wrapper + +def convert_form_date_to_obpapi_datetime_format(form_to_date_string): + """ + convert the String 2020-10-22 to 2020-10-22T00:00:00.000000Z + """ + return datetime.strptime(form_to_date_string, API_MANAGER_DATE_FORMAT).strftime(API_DATEFORMAT) + +def return_to_days_ago(date, days): + """ + eg: + date 2020-10-22T00:00:00.000000Z + days =1 + return 2020-10-21T00:00:00.000000Z + """ + return (datetime.strptime(date, API_DATEFORMAT) - timedelta(days)).strftime(API_DATEFORMAT) \ No newline at end of file diff --git a/apimanager/dynamicendpoints/views.py b/apimanager/dynamicendpoints/views.py index a0495e0..6e2fe4b 100644 --- a/apimanager/dynamicendpoints/views.py +++ b/apimanager/dynamicendpoints/views.py @@ -7,7 +7,7 @@ import json from django.contrib.auth.mixins import LoginRequiredMixin from django.views.generic import FormView from obp.api import API, APIError -from utils.ErrorHandler import exception_handle, error_once_only +from base.utils import exception_handle, error_once_only from .forms import DynamicEndpointsForm from django.urls import reverse_lazy from django.views.decorators.csrf import csrf_exempt diff --git a/apimanager/methodrouting/views.py b/apimanager/methodrouting/views.py index b9ad1ca..6ee3432 100644 --- a/apimanager/methodrouting/views.py +++ b/apimanager/methodrouting/views.py @@ -10,8 +10,9 @@ from django.http import JsonResponse from django.contrib import messages from django.contrib.auth.mixins import LoginRequiredMixin from django.views.generic import FormView + +from base.utils import error_once_only, exception_handle from obp.api import API, APIError -from utils.ErrorHandler import exception_handle, error_once_only from .forms import MethodRoutingForm from django.urls import reverse_lazy from django.views.decorators.csrf import csrf_exempt diff --git a/apimanager/metrics/forms.py b/apimanager/metrics/forms.py index 3d53e94..0cc79a2 100644 --- a/apimanager/metrics/forms.py +++ b/apimanager/metrics/forms.py @@ -246,7 +246,7 @@ class CustomSummaryForm(forms.Form): super(CustomSummaryForm, self).__init__(*args, **kwargs) -class MetricsSummaryForm(forms.Form): +class MonthlyMetricsSummaryForm(forms.Form): to_date = forms.DateField( label='To Date', # input_formats=[settings.API_DATEFORMAT], @@ -266,4 +266,4 @@ class MetricsSummaryForm(forms.Form): def __init__(self, *args, **kwargs): kwargs.setdefault('label_suffix', '') - super(MetricsSummaryForm, self).__init__(*args, **kwargs) + super(MonthlyMetricsSummaryForm, self).__init__(*args, **kwargs) diff --git a/apimanager/metrics/templates/metrics/summary.html b/apimanager/metrics/templates/metrics/monthly_summary.html similarity index 100% rename from apimanager/metrics/templates/metrics/summary.html rename to apimanager/metrics/templates/metrics/monthly_summary.html diff --git a/apimanager/metrics/templates/metrics/quarterly_summary.html b/apimanager/metrics/templates/metrics/quarterly_summary.html index 9808dfd..9d67aff 100644 --- a/apimanager/metrics/templates/metrics/quarterly_summary.html +++ b/apimanager/metrics/templates/metrics/quarterly_summary.html @@ -81,10 +81,10 @@ Calls per month: somealt - - Calls per day: - somealt - +{# #} +{# Calls per day:#} +{# somealt#} +{# #} Average number of calls per day: {{ average_calls_per_day }} diff --git a/apimanager/metrics/urls.py b/apimanager/metrics/urls.py index 249d83d..3803ef7 100644 --- a/apimanager/metrics/urls.py +++ b/apimanager/metrics/urls.py @@ -9,7 +9,7 @@ from .views import ( APIMetricsView, APISummaryPartialFunctionView, ConnectorMetricsView, - MetricsSummaryView, + MonthlyMetricsSummaryView, YearlySummaryView, QuarterlySummaryView, WeeklySummaryView, @@ -28,8 +28,8 @@ urlpatterns = [ url(r'^connector/$', ConnectorMetricsView.as_view(), name='connector-metrics'), - url(r'^summary/$', - MetricsSummaryView.as_view(), + url(r'^monthly-summary/$', + MonthlyMetricsSummaryView.as_view(), name='metrics-summary'), url(r'^yearly-summary/$', YearlySummaryView.as_view(), diff --git a/apimanager/metrics/views.py b/apimanager/metrics/views.py index c38b28d..5499c44 100644 --- a/apimanager/metrics/views.py +++ b/apimanager/metrics/views.py @@ -5,18 +5,18 @@ Views of metrics app import json import hashlib -import operator from datetime import datetime, timedelta +from enum import Enum from django.conf import settings from apimanager.settings import API_HOST, EXCLUDE_APPS, EXCLUDE_FUNCTIONS, EXCLUDE_URL_PATTERN, API_EXPLORER_APP_NAME, API_DATEFORMAT from django.contrib import messages from django.contrib.auth.mixins import LoginRequiredMixin from django.views.generic import TemplateView +from base.utils import error_once_only, get_cache_key_for_current_call, convert_form_date_to_obpapi_datetime_format, \ + return_to_days_ago from obp.api import API, APIError, LOGGER -from utils.ErrorHandler import error_once_only -from utils.utils import get_cache_key_for_current_call -from .forms import APIMetricsForm, ConnectorMetricsForm, MetricsSummaryForm, CustomSummaryForm +from .forms import APIMetricsForm, ConnectorMetricsForm, MonthlyMetricsSummaryForm, CustomSummaryForm from pylab import * from django.core.cache import cache import traceback @@ -69,6 +69,14 @@ def get_barchart_data(metrics, fieldname): return data +class SummaryType(Enum): + YEARLY = 1 + QUARTERLY = 2 + MONTHLY = 3 + WEEKLY = 4 + DAYLY = 5 + CUSTOM = 6 + class MetricsView(LoginRequiredMixin, TemplateView): """View for metrics (sort of abstract base class)""" form_class = None @@ -181,10 +189,10 @@ class ConnectorMetricsView(MetricsView): template_name = 'metrics/connector.html' api_urlpath = '/management/connector/metrics' -class MetricsSummaryView(LoginRequiredMixin, TemplateView): +class MonthlyMetricsSummaryView(LoginRequiredMixin, TemplateView): """View for metrics summary (sort of abstract base class)""" - form_class = MetricsSummaryForm - template_name = 'metrics/summary.html' + form_class = MonthlyMetricsSummaryForm + template_name = 'metrics/monthly_summary.html' api_urlpath = None def get_form(self): @@ -241,54 +249,66 @@ class MetricsSummaryView(LoginRequiredMixin, TemplateView): There are different use cases, so we accept different parameters. only_show_api_explorer_metrics has the default value False, because it is just used for app = API_Explorer. """ - api_calls_total = 0 - average_response_time = 0 - urlpath = '/management/aggregate-metrics' - if only_show_api_explorer_metrics: - urlpath = urlpath + '?from_date={}&to_date={}&app_name={}'.format(from_date, to_date, API_EXPLORER_APP_NAME) - elif ((not only_show_api_explorer_metrics) and (not is_included_obp_apps)): - urlpath = urlpath + '?from_date={}&to_date={}&exclude_app_names={}&exclude_implemented_by_partial_functions={}&exclude_url_pattern={}'.format( - from_date, to_date, ",".join(EXCLUDE_APPS),",".join(EXCLUDE_FUNCTIONS), ",".join(EXCLUDE_URL_PATTERN)) - else: - urlpath = urlpath + '?from_date={}&to_date={}'.format(from_date, to_date) - cache_key = get_cache_key_for_current_call(self.request, urlpath) try: + api_calls_total = 0 + average_response_time = 0 + urlpath = '/management/aggregate-metrics' + if only_show_api_explorer_metrics: + urlpath = urlpath + '?from_date={}&to_date={}&app_name={}'.format(from_date, to_date, API_EXPLORER_APP_NAME) + elif ((not only_show_api_explorer_metrics) and (not is_included_obp_apps)): + urlpath = urlpath + '?from_date={}&to_date={}&exclude_app_names={}&exclude_implemented_by_partial_functions={}&exclude_url_pattern={}'.format( + from_date, to_date, ",".join(EXCLUDE_APPS),",".join(EXCLUDE_FUNCTIONS), ",".join(EXCLUDE_URL_PATTERN)) + else: + urlpath = urlpath + '?from_date={}&to_date={}'.format(from_date, to_date) + cache_key = get_cache_key_for_current_call(self.request, urlpath) apicaches = None try: apicaches = cache.get(cache_key) except Exception as err: apicaches = None if not apicaches is None: - api_calls_total = apicaches[0]["count"] - average_response_time = apicaches[0]["average_response_time"] + metrics = apicaches else: api = API(self.request.session.get('obp')) metrics = api.get(urlpath) - api_calls_total = metrics[0]["count"] - average_response_time = metrics[0]["average_response_time"] apicaches = cache.set(cache_key, metrics) LOGGER.warning('The cache is setting, url is: {}'.format(urlpath)) LOGGER.warning('The cache is setting key is: {}'.format(cache_key)) - to_date = datetime.datetime.strptime(to_date, API_DATEFORMAT) - from_date = datetime.datetime.strptime(from_date, API_DATEFORMAT) - number_of_days = abs((to_date - from_date).days) - average_calls_per_day = api_calls_total / number_of_days + + api_calls_total, average_calls_per_day, average_response_time = self.get_internal_api_call_metrics( + api_calls_total, average_response_time, cache_key, from_date, metrics, to_date, urlpath) + return api_calls_total, average_response_time, int(average_calls_per_day) except APIError as err: error_once_only(self.request, err) except Exception as err: error_once_only(self.request, 'Unknown Error. {}'.format(err)) + - return api_calls_total, average_response_time, int(average_calls_per_day) + def get_internal_api_call_metrics(self, api_calls_total, average_response_time, cache_key, from_date, metrics, + to_date, urlpath): + api_calls_total = metrics[0]["count"] + average_response_time = metrics[0]["average_response_time"] + to_date = datetime.datetime.strptime(to_date, API_DATEFORMAT) + from_date = datetime.datetime.strptime(from_date, API_DATEFORMAT) + number_of_days = abs((to_date - from_date).days) + # if number_of_days= 0, then it means calls_per_hour + average_calls_per_day = api_calls_total if (number_of_days == 0) else api_calls_total / number_of_days + return api_calls_total, average_calls_per_day, average_response_time def get_aggregate_metrics_api_explorer(self, from_date, to_date): return self.get_aggregate_metrics(from_date, to_date, True, True) - def get_active_apps(self, cleaned_data, from_date, to_date): + def get_active_apps(self, is_included_obp_apps, from_date, to_date): + """ + Gets the metrics from the API, using given parameters, + There are different use cases, so we accept different parameters. + only_show_api_explorer_metrics has the default value False, because it is just used for app = API_Explorer. + """ apps = [] form = self.get_form() active_apps_list = [] - if cleaned_data.get('include_obp_apps'): + if is_included_obp_apps: urlpath = '/management/metrics/top-consumers?from_date={}&to_date={}'.format(from_date, to_date) api = API(self.request.session.get('obp')) try: @@ -307,10 +327,7 @@ class MetricsSummaryView(LoginRequiredMixin, TemplateView): api = API(self.request.session.get('obp')) try: apps = api.get(urlpath) - if apps is not None and 'code' in apps and apps['code']==403: - error_once_only(self.request, apps['message']) - else: - active_apps_list = list(apps) + active_apps_list = list(apps['top_consumers']) except APIError as err: error_once_only(self.request, err) except Exception as err: @@ -807,359 +824,135 @@ class MetricsSummaryView(LoginRequiredMixin, TemplateView): return delta - def get_context_data(self, **kwargs): - form = self.get_form() + def get_context_data(self, **kwargs): return self.prepare_general_context(SummaryType.MONTHLY) + + def prepare_general_context(self, web_page_type, **kwargs): + try: + form = self.get_form() + per_day_chart=[] + calls_per_month_list=[] + per_month_chart=[] + calls_per_hour_list=[] + per_hour_chart=[] + if form.is_valid(): + is_included_obp_apps = form.cleaned_data.get('include_obp_apps') + form_to_date_string = form.data['to_date'] + to_date = convert_form_date_to_obpapi_datetime_format(form_to_date_string) - to_date = datetime.datetime.strptime(form.data['to_date'], '%Y-%m-%d %H:%M:%S') - to_date = to_date.strftime(API_DATEFORMAT) + if (web_page_type == SummaryType.DAYLY): + # for one day, the from_date is 1 day ago. + from_date = return_to_days_ago(to_date, 1) + calls_per_hour_list, calls_per_hour, hour_list = self.calls_per_hour(is_included_obp_apps, from_date, to_date) + per_hour_chart = self.plot_line_chart(calls_per_hour, hour_list, 'hour') + + if (web_page_type == SummaryType.WEEKLY): + # for one month, the from_date is 7 days ago. + from_date = return_to_days_ago(to_date, 7) + calls_per_day_list, calls_per_day, date_list = self.calls_per_day(is_included_obp_apps, from_date, to_date) + per_day_chart = self.plot_line_chart(calls_per_day, date_list, "day") + + if (web_page_type == SummaryType.MONTHLY): + # for one month, the from_date is 30 days ago. + from_date = return_to_days_ago(to_date, 30) + calls_per_day_list, calls_per_day, date_list = self.calls_per_day(is_included_obp_apps, from_date, to_date) + per_day_chart = self.plot_line_chart(calls_per_day, date_list, "day") - from_date = (datetime.datetime.strptime(to_date, API_DATEFORMAT) - timedelta(30)).strftime(API_DATEFORMAT) - - context = super(MetricsSummaryView, self).get_context_data(**kwargs) - api_host_name = API_HOST - top_apps_using_warehouse = self.get_top_apps_using_warehouse(from_date, to_date) - user_email_cansearchwarehouse, number_of_users_with_cansearchwarehouse = self.get_users_cansearchwarehouse() - median_time_to_first_api_call = self.median_time_to_first_api_call(from_date, to_date) - if form.is_valid(): - is_included_obp_apps = form.cleaned_data.get('include_obp_apps') - top_apis = self.get_top_apis(form.cleaned_data, from_date, to_date) - top_apis_bar_chart = self.plot_bar_chart(top_apis) - top_consumers=self.get_top_consumers(form.cleaned_data, from_date, to_date) - top_consumers_bar_chart = self.plot_topconsumer_bar_chart(top_consumers) - top_warehouse_calls = self.get_top_warehouse_calls(form.cleaned_data, from_date, to_date) - api_calls, average_response_time, average_calls_per_day = self.get_aggregate_metrics(from_date, to_date, is_included_obp_apps) - calls_by_api_explorer, average_response_time_api_explorer, average_calls_per_day_api_explorer = self.get_aggregate_metrics_api_explorer(from_date, to_date) - #calls_per_month_list, calls_per_month, date_list = self.calls_per_month(form.cleaned_data, from_date, to_date) - calls_per_day_list, calls_per_day, date_list = self.calls_per_day(is_included_obp_apps, from_date, to_date) - per_day_chart = self.plot_line_chart(calls_per_day, date_list, "day") - unique_app_names, number_of_apps_with_unique_app_name, number_of_apps_with_unique_developer_email = self.get_total_number_of_apps(form.cleaned_data, from_date, to_date) - active_apps_list = self.get_active_apps(form.cleaned_data, from_date, to_date) + if (web_page_type == SummaryType.QUARTERLY): + # for one quarter, the from_date is 90 days ago. + from_date = (datetime.datetime.strptime(to_date, API_DATEFORMAT) - timedelta(90)).strftime(API_DATEFORMAT) + calls_per_month_list, calls_per_month, month_list = self.calls_per_month(is_included_obp_apps, from_date, to_date) + per_month_chart = self.plot_line_chart(calls_per_month, month_list, 'month') - context.update({ - 'api_calls': api_calls, - 'calls_by_api_explorer': calls_by_api_explorer, - #'calls_per_month_list': calls_per_month_list, - 'per_day_chart': per_day_chart, - 'number_of_apps_with_unique_app_name': number_of_apps_with_unique_app_name, - 'number_of_apps_with_unique_developer_email': number_of_apps_with_unique_developer_email, - 'active_apps_list': active_apps_list, - 'average_calls_per_day': average_calls_per_day, - 'average_response_time': average_response_time, - 'top_warehouse_calls': top_warehouse_calls, - 'top_apps_using_warehouse': top_apps_using_warehouse, - 'user_email_cansearchwarehouse': user_email_cansearchwarehouse, - 'number_of_users_with_cansearchwarehouse': number_of_users_with_cansearchwarehouse, - 'api_host_name': api_host_name, - 'from_date': (datetime.datetime.strptime(from_date, API_DATEFORMAT)).strftime('%Y-%m-%d'), - 'to_date': (datetime.datetime.strptime(to_date, API_DATEFORMAT)).strftime('%Y-%m-%d'), - 'top_apis_bar_chart': top_apis_bar_chart, - 'top_consumers_bar_chart':top_consumers_bar_chart, - 'median_time_to_first_api_call': median_time_to_first_api_call, - 'form': form, - 'excluded_apps':EXCLUDE_APPS, - 'excluded_functions':EXCLUDE_FUNCTIONS, - 'excluded_url_pattern':EXCLUDE_URL_PATTERN, - }) - return context + if (web_page_type == SummaryType.YEARLY): + from_date = return_to_days_ago(to_date, 365) + calls_per_month_list, calls_per_month, month_list = self.calls_per_month(is_included_obp_apps, from_date, to_date) + per_month_chart = self.plot_line_chart(calls_per_month, month_list, "month") -class YearlySummaryView(MetricsSummaryView): + if (web_page_type == SummaryType.CUSTOM): + # for one month, the from_date is x day ago. + form_from_date_string = form.data['from_date_custom'] + from_date = convert_form_date_to_obpapi_datetime_format(form_from_date_string) + calls_per_day_list, calls_per_day, date_list = self.calls_per_day(is_included_obp_apps, from_date, to_date) + per_day_chart = self.plot_line_chart(calls_per_day, date_list, "day") + + api_host_name = API_HOST + top_apps_using_warehouse = self.get_top_apps_using_warehouse(from_date, to_date) + user_email_cansearchwarehouse, number_of_users_with_cansearchwarehouse = self.get_users_cansearchwarehouse() + median_time_to_first_api_call = self.median_time_to_first_api_call(from_date, to_date) + + top_apis = self.get_top_apis(form.cleaned_data, from_date, to_date) + top_apis_bar_chart = self.plot_bar_chart(top_apis) + top_consumers = self.get_top_consumers(form.cleaned_data, from_date, to_date) + top_consumers_bar_chart = self.plot_topconsumer_bar_chart(top_consumers) + top_warehouse_calls = self.get_top_warehouse_calls(form.cleaned_data, from_date, to_date) + api_calls, average_response_time, average_calls_per_day = self.get_aggregate_metrics(from_date, to_date,is_included_obp_apps) + calls_by_api_explorer, average_response_time_api_explorer, average_calls_per_day_api_explorer = self.get_aggregate_metrics_api_explorer( + from_date, to_date) + + unique_app_names, number_of_apps_with_unique_app_name, number_of_apps_with_unique_developer_email = self.get_total_number_of_apps( + form.cleaned_data, from_date, to_date) + active_apps_list = self.get_active_apps(is_included_obp_apps, from_date, to_date) + + context = super(MonthlyMetricsSummaryView, self).get_context_data(**kwargs) + + context.update({ + 'form': form, + 'api_calls': api_calls, + 'calls_by_api_explorer': calls_by_api_explorer, + 'calls_per_month_list': calls_per_month_list, + 'per_month_chart': per_month_chart, + 'per_day_chart': per_day_chart, + 'calls_per_hour_list': calls_per_hour_list, + 'per_hour_chart': per_hour_chart, + 'number_of_apps_with_unique_app_name': number_of_apps_with_unique_app_name, + 'number_of_apps_with_unique_developer_email': number_of_apps_with_unique_developer_email, + 'active_apps_list': active_apps_list, + 'average_calls_per_day': average_calls_per_day, + 'average_response_time': average_response_time, + 'top_warehouse_calls': top_warehouse_calls, + 'top_apps_using_warehouse': top_apps_using_warehouse, + 'user_email_cansearchwarehouse': user_email_cansearchwarehouse, + 'number_of_users_with_cansearchwarehouse': number_of_users_with_cansearchwarehouse, + 'api_host_name': api_host_name, + 'from_date': (datetime.datetime.strptime(from_date, API_DATEFORMAT)).strftime('%Y-%m-%d'), + 'to_date': (datetime.datetime.strptime(to_date, API_DATEFORMAT)).strftime('%Y-%m-%d'), + 'top_apis': top_apis, + 'top_apis_bar_chart': top_apis_bar_chart, + 'top_consumers_bar_chart': top_consumers_bar_chart, + 'median_time_to_first_api_call': median_time_to_first_api_call, + 'excluded_apps': EXCLUDE_APPS, + 'excluded_functions': EXCLUDE_FUNCTIONS, + 'excluded_url_pattern': EXCLUDE_URL_PATTERN, + }) + return context + else: + error_once_only(self.request, form.errors) + except Exception as err: + error_once_only(self.request, 'Unknown Error. {}'.format(err)) + + +class YearlySummaryView(MonthlyMetricsSummaryView): template_name = 'metrics/yearly_summary.html' - - def get_context_data(self, **kwargs): - form = self.get_form() - to_date = datetime.datetime.strptime(form.data['to_date'], '%Y-%m-%d') - to_date = to_date.strftime(API_DATEFORMAT) - from_date = (datetime.datetime.strptime(to_date, API_DATEFORMAT) - timedelta(365)).strftime(API_DATEFORMAT) - - context = super(MetricsSummaryView, self).get_context_data(**kwargs) - api_host_name = API_HOST - top_apps_using_warehouse = self.get_top_apps_using_warehouse(from_date, to_date) - user_email_cansearchwarehouse, number_of_users_with_cansearchwarehouse = self.get_users_cansearchwarehouse() - median_time_to_first_api_call = self.median_time_to_first_api_call(from_date, to_date) - - if form.is_valid(): - is_included_obp_apps = form.cleaned_data.get('include_obp_apps') - api_calls, average_response_time, average_calls_per_day = self.get_aggregate_metrics(from_date, to_date, is_included_obp_apps) - calls_by_api_explorer, average_response_time_api_explorer, average_calls_per_day_api_explorer = self.get_aggregate_metrics_api_explorer(from_date, to_date) - calls_per_month_list, calls_per_month, month_list = self.calls_per_month(is_included_obp_apps, from_date, to_date) - per_month_chart = self.plot_line_chart(calls_per_month, month_list, "month") - unique_app_names, number_of_apps_with_unique_app_name, number_of_apps_with_unique_developer_email = self.get_total_number_of_apps(form.cleaned_data, from_date, to_date) - active_apps_names = self.get_active_apps(form.cleaned_data, from_date, to_date) - top_apis = self.get_top_apis(form.cleaned_data, from_date, to_date) - top_apis_bar_chart = self.plot_bar_chart(top_apis) - top_consumers=self.get_top_consumers(form.cleaned_data, from_date, to_date) - top_consumers_bar_chart = self.plot_topconsumer_bar_chart(top_consumers) - top_warehouse_calls = self.get_top_warehouse_calls(form.cleaned_data, from_date, to_date) - - context.update({ - 'api_calls': api_calls, - 'calls_by_api_explorer': calls_by_api_explorer, - 'calls_per_month_list': calls_per_month_list, - 'per_month_chart': per_month_chart, - 'number_of_apps_with_unique_app_name': number_of_apps_with_unique_app_name, - 'number_of_apps_with_unique_developer_email': number_of_apps_with_unique_developer_email, - 'active_apps_names': active_apps_names, - 'average_calls_per_day': average_calls_per_day, - 'average_response_time': average_response_time, - 'top_apis': top_apis, - 'top_warehouse_calls': top_warehouse_calls, - 'top_apps_using_warehouse': top_apps_using_warehouse, - 'user_email_cansearchwarehouse': user_email_cansearchwarehouse, - 'number_of_users_with_cansearchwarehouse': number_of_users_with_cansearchwarehouse, - 'api_host_name': api_host_name, - 'from_date': (datetime.datetime.strptime(from_date, API_DATEFORMAT)).strftime('%Y-%m-%d'), - 'to_date': (datetime.datetime.strptime(to_date, API_DATEFORMAT)).strftime('%Y-%m-%d'), - 'top_apis_bar_chart': top_apis_bar_chart, - 'top_consumers_bar_chart':top_consumers_bar_chart, - 'median_time_to_first_api_call': median_time_to_first_api_call, - 'form': form, - 'excluded_apps':EXCLUDE_APPS, - 'excluded_functions':EXCLUDE_FUNCTIONS, - 'excluded_url_pattern':EXCLUDE_URL_PATTERN, - }) - return context - -class QuarterlySummaryView(MetricsSummaryView): + def get_context_data(self, **kwargs): return self.prepare_general_context(SummaryType.YEARLY, **kwargs) + +class QuarterlySummaryView(MonthlyMetricsSummaryView): template_name = 'metrics/quarterly_summary.html' + def get_context_data(self, **kwargs): return self.prepare_general_context(SummaryType.QUARTERLY, **kwargs) + - def get_context_data(self, **kwargs): - form = self.get_form() - to_date = datetime.datetime.strptime(form.data['to_date'], '%Y-%m-%d') - to_date = to_date.strftime(API_DATEFORMAT) - from_date = (datetime.datetime.strptime(to_date, API_DATEFORMAT) - timedelta(90)).strftime(API_DATEFORMAT) - - context = super(MetricsSummaryView, self).get_context_data(**kwargs) - api_host_name = API_HOST - top_apps_using_warehouse = self.get_top_apps_using_warehouse(from_date, to_date) - user_email_cansearchwarehouse, number_of_users_with_cansearchwarehouse = self.get_users_cansearchwarehouse() - median_time_to_first_api_call = self.median_time_to_first_api_call(from_date, to_date) - - if form.is_valid(): - is_included_obp_apps = form.cleaned_data.get('include_obp_apps') - api_calls, average_response_time, average_calls_per_day = self.get_aggregate_metrics(from_date, to_date, is_included_obp_apps) - calls_by_api_explorer, average_response_time_api_explorer, average_calls_per_day_api_explorer = self.get_aggregate_metrics_api_explorer(from_date, to_date) - calls_per_month_list, calls_per_month, month_list = self.calls_per_month(is_included_obp_apps, from_date, to_date) - calls_per_day_list, calls_per_day, date_list = self.calls_per_day(is_included_obp_apps, from_date, to_date) - per_month_chart = self.plot_line_chart(calls_per_month, month_list, 'month') - per_day_chart = self.plot_line_chart(calls_per_day, date_list, 'day') - unique_app_names, number_of_apps_with_unique_app_name, number_of_apps_with_unique_developer_email = self.get_total_number_of_apps(form.cleaned_data, from_date, to_date) - active_apps_names = self.get_active_apps(form.cleaned_data, from_date, to_date) - top_apis = self.get_top_apis(form.cleaned_data, from_date, to_date) - top_apis_bar_chart = self.plot_bar_chart(top_apis) - top_consumers=self.get_top_consumers(form.cleaned_data, from_date, to_date) - top_consumers_bar_chart = self.plot_topconsumer_bar_chart(top_consumers) - top_warehouse_calls = self.get_top_warehouse_calls(form.cleaned_data, from_date, to_date) - - context.update({ - 'api_calls': api_calls, - 'calls_by_api_explorer': calls_by_api_explorer, - 'calls_per_month_list': calls_per_month_list, - 'per_month_chart': per_month_chart, - 'per_day_chart': per_day_chart, - 'number_of_apps_with_unique_app_name': number_of_apps_with_unique_app_name, - 'number_of_apps_with_unique_developer_email': number_of_apps_with_unique_developer_email, - 'active_apps_names': active_apps_names, - 'average_calls_per_day': average_calls_per_day, - 'average_response_time': average_response_time, - 'top_apis': top_apis, - 'top_warehouse_calls': top_warehouse_calls, - 'top_apps_using_warehouse': top_apps_using_warehouse, - 'user_email_cansearchwarehouse': user_email_cansearchwarehouse, - 'number_of_users_with_cansearchwarehouse': number_of_users_with_cansearchwarehouse, - 'api_host_name': api_host_name, - #'from_date': from_date.strftime('%Y-%m-%d'), - 'from_date': (datetime.datetime.strptime(from_date, API_DATEFORMAT)).strftime('%Y-%m-%d'), - 'to_date': (datetime.datetime.strptime(to_date, API_DATEFORMAT)).strftime('%Y-%m-%d'), - 'top_apis_bar_chart': top_apis_bar_chart, - 'top_consumers_bar_chart':top_consumers_bar_chart, - 'median_time_to_first_api_call': median_time_to_first_api_call, - 'form': form, - 'excluded_apps':EXCLUDE_APPS, - 'excluded_functions':EXCLUDE_FUNCTIONS, - 'excluded_url_pattern':EXCLUDE_URL_PATTERN, - }) - return context - -class WeeklySummaryView(MetricsSummaryView): +class WeeklySummaryView(MonthlyMetricsSummaryView): template_name = 'metrics/weekly_summary.html' + def get_context_data(self, **kwargs): return self.prepare_general_context(SummaryType.WEEKLY, **kwargs) - def get_context_data(self, **kwargs): - form = self.get_form() - to_date = datetime.datetime.strptime(form.data['to_date'], '%Y-%m-%d') - to_date = to_date.strftime(API_DATEFORMAT) - from_date = (datetime.datetime.strptime(to_date, API_DATEFORMAT) - timedelta(7)).strftime(API_DATEFORMAT) - - context = super(MetricsSummaryView, self).get_context_data(**kwargs) - api_host_name = API_HOST - top_apps_using_warehouse = self.get_top_apps_using_warehouse(from_date, to_date) - user_email_cansearchwarehouse, number_of_users_with_cansearchwarehouse = self.get_users_cansearchwarehouse() - # calls_per_half_day = self.calls_per_half_day() - median_time_to_first_api_call = self.median_time_to_first_api_call(from_date, to_date) - - if form.is_valid(): - is_included_obp_apps = form.cleaned_data.get('include_obp_apps') - api_calls, average_response_time, average_calls_per_day = self.get_aggregate_metrics(from_date, to_date, is_included_obp_apps) - calls_by_api_explorer, average_response_time_api_explorer, average_calls_per_day_api_explorer = self.get_aggregate_metrics_api_explorer(from_date, to_date) - calls_per_day_list, calls_per_day, date_list = self.calls_per_day(is_included_obp_apps, from_date, to_date) - per_day_chart = self.plot_line_chart(calls_per_day, date_list, 'day') - unique_app_names, number_of_apps_with_unique_app_name, number_of_apps_with_unique_developer_email = self.get_total_number_of_apps(form.cleaned_data, from_date, to_date) - active_apps_names = self.get_active_apps(form.cleaned_data, from_date, to_date) - top_apis = self.get_top_apis(form.cleaned_data, from_date, to_date) - top_apis_bar_chart = self.plot_bar_chart(top_apis) - top_consumers=self.get_top_consumers(form.cleaned_data, from_date, to_date) - top_consumers_bar_chart = self.plot_topconsumer_bar_chart(top_consumers) - top_warehouse_calls = self.get_top_warehouse_calls(form.cleaned_data, from_date, to_date) - - context.update({ - 'api_calls': api_calls, - 'calls_by_api_explorer': calls_by_api_explorer, - 'per_day_chart': per_day_chart, - 'number_of_apps_with_unique_app_name': number_of_apps_with_unique_app_name, - 'number_of_apps_with_unique_developer_email': number_of_apps_with_unique_developer_email, - 'active_apps_names': active_apps_names, - 'average_calls_per_day': average_calls_per_day, - 'average_response_time': average_response_time, - 'top_apis': top_apis, - 'top_warehouse_calls': top_warehouse_calls, - 'top_apps_using_warehouse': top_apps_using_warehouse, - 'user_email_cansearchwarehouse': user_email_cansearchwarehouse, - 'number_of_users_with_cansearchwarehouse': number_of_users_with_cansearchwarehouse, - 'api_host_name': api_host_name, - 'from_date': (datetime.datetime.strptime(from_date, API_DATEFORMAT)).strftime('%Y-%m-%d'), - 'to_date': (datetime.datetime.strptime(to_date, API_DATEFORMAT)).strftime('%Y-%m-%d'), - 'top_apis_bar_chart': top_apis_bar_chart, - 'top_consumers_bar_chart':top_consumers_bar_chart, - # ##'calls_per_half_day': calls_per_half_day, - 'median_time_to_first_api_call': median_time_to_first_api_call, - 'form': form, - 'excluded_apps':EXCLUDE_APPS, - 'excluded_functions':EXCLUDE_FUNCTIONS, - 'excluded_url_pattern':EXCLUDE_URL_PATTERN, - }) - return context - - -class DailySummaryView(MetricsSummaryView): +class DailySummaryView(MonthlyMetricsSummaryView): template_name = 'metrics/daily_summary.html' + def get_context_data(self, **kwargs): return self.prepare_general_context(SummaryType.DAYLY, **kwargs) - def get_context_data(self, **kwargs): - form = self.get_form() - to_date = datetime.datetime.strptime(form.data['to_date'], '%Y-%m-%d') - to_date = to_date.strftime(API_DATEFORMAT) - from_date = (datetime.datetime.strptime(to_date, API_DATEFORMAT) - timedelta(1)).strftime(API_DATEFORMAT) - - context = super(DailySummaryView, self).get_context_data(**kwargs) - api_host_name = API_HOST - top_apps_using_warehouse = self.get_top_apps_using_warehouse(from_date, to_date) - user_email_cansearchwarehouse, number_of_users_with_cansearchwarehouse = self.get_users_cansearchwarehouse() - median_time_to_first_api_call = self.median_time_to_first_api_call(from_date, to_date) - - if form.is_valid(): - is_included_obp_apps = form.cleaned_data.get('include_obp_apps') - api_calls, average_response_time, average_calls_per_day = self.get_aggregate_metrics(from_date, to_date, is_included_obp_apps) - calls_by_api_explorer, average_response_time_api_explorer, average_calls_per_day_api_explorer = self.get_aggregate_metrics_api_explorer(from_date, to_date) - calls_per_hour_list, calls_per_hour, hour_list = self.calls_per_hour(is_included_obp_apps, from_date, to_date) - per_hour_chart = self.plot_line_chart(calls_per_hour, hour_list, 'hour') - # calls_per_hour_list, calls_per_hour = self.calls_per_hour(form.cleaned_data, from_date) - # per_hour_chart = self.get_per_hour_chart(form.cleaned_data, from_date) - unique_app_names, number_of_apps_with_unique_app_name, number_of_apps_with_unique_developer_email = self.get_total_number_of_apps(form.cleaned_data, from_date, to_date) - active_apps_names = self.get_active_apps(form.cleaned_data, from_date, to_date) - top_apis = self.get_top_apis(form.cleaned_data, from_date, to_date) - top_apis_bar_chart = self.plot_bar_chart(top_apis) - top_consumers=self.get_top_consumers(form.cleaned_data, from_date, to_date) - top_consumers_bar_chart = self.plot_topconsumer_bar_chart(top_consumers) - top_warehouse_calls = self.get_top_warehouse_calls(form.cleaned_data, from_date, to_date) - - context.update({ - 'api_calls': api_calls, - 'calls_by_api_explorer': calls_by_api_explorer, - 'calls_per_hour_list': calls_per_hour_list, - 'per_hour_chart': per_hour_chart, - 'number_of_apps_with_unique_app_name': number_of_apps_with_unique_app_name, - 'number_of_apps_with_unique_developer_email': number_of_apps_with_unique_developer_email, - 'active_apps_names': active_apps_names, - 'average_calls_per_day': average_calls_per_day, - 'average_response_time': average_response_time, - 'top_apis': top_apis, - 'top_warehouse_calls': top_warehouse_calls, - 'top_apps_using_warehouse': top_apps_using_warehouse, - 'user_email_cansearchwarehouse': user_email_cansearchwarehouse, - 'number_of_users_with_cansearchwarehouse': number_of_users_with_cansearchwarehouse, - 'api_host_name': api_host_name, - 'from_date': (datetime.datetime.strptime(from_date, API_DATEFORMAT)).strftime('%Y-%m-%d'), - 'to_date': (datetime.datetime.strptime(to_date, API_DATEFORMAT)).strftime('%Y-%m-%d'), - 'top_apis_bar_chart': top_apis_bar_chart, - 'top_consumers_bar_chart':top_consumers_bar_chart, - 'median_time_to_first_api_call': median_time_to_first_api_call, - 'form': form, - 'excluded_apps':EXCLUDE_APPS, - 'excluded_functions':EXCLUDE_FUNCTIONS, - 'excluded_url_pattern':EXCLUDE_URL_PATTERN, - }) - return context - -class HourlySummaryView(MetricsSummaryView): +class HourlySummaryView(MonthlyMetricsSummaryView): template_name = 'metrics/hourly_summary.html' -class CustomSummaryView(MetricsSummaryView): +class CustomSummaryView(MonthlyMetricsSummaryView): form_class = CustomSummaryForm template_name = 'metrics/custom_summary.html' - def get_context_data(self, **kwargs): - form = self.get_form() - - to_date = datetime.datetime.strptime(form.data['to_date'], '%Y-%m-%d') - to_date = to_date.strftime(API_DATEFORMAT) - - from_date = datetime.datetime.strptime(form.data['from_date_custom'], '%Y-%m-%d') - from_date = from_date.strftime(API_DATEFORMAT) - context = super(MetricsSummaryView, self).get_context_data(**kwargs) - api_host_name = API_HOST - top_apps_using_warehouse = self.get_top_apps_using_warehouse(from_date, to_date) - user_email_cansearchwarehouse, number_of_users_with_cansearchwarehouse = self.get_users_cansearchwarehouse() - # calls_per_day = self.calls_per_day(from_date) - # calls_per_half_day = self.calls_per_half_day() - median_time_to_first_api_call = self.median_time_to_first_api_call(from_date, to_date) - - if form.is_valid(): - is_included_obp_apps = form.cleaned_data.get('include_obp_apps') - api_calls, average_response_time, average_calls_per_day = self.get_aggregate_metrics(from_date, to_date, is_included_obp_apps) - calls_by_api_explorer, average_response_time_api_explorer, average_calls_per_day_api_explorer = self.get_aggregate_metrics_api_explorer(from_date, to_date) - calls_per_day_list, calls_per_day, date_list = self.calls_per_day(is_included_obp_apps, from_date, to_date) - per_day_chart = self.plot_line_chart(calls_per_day, date_list, 'day') - unique_app_names, number_of_apps_with_unique_app_name, number_of_apps_with_unique_developer_email = self.get_total_number_of_apps(form.cleaned_data, from_date, to_date) - active_apps_names = self.get_active_apps(form.cleaned_data, from_date, to_date) - top_apis = self.get_top_apis(form.cleaned_data, from_date, to_date) - top_apis_bar_chart = self.plot_bar_chart(top_apis) - top_consumers=self.get_top_consumers(form.cleaned_data, from_date, to_date) - top_consumers_bar_chart = self.plot_topconsumer_bar_chart(top_consumers) - top_warehouse_calls = self.get_top_warehouse_calls(form.cleaned_data, from_date, to_date) - - context.update({ - 'api_calls': api_calls, - 'calls_by_api_explorer': calls_by_api_explorer, - 'per_day_chart': per_day_chart, - 'number_of_apps_with_unique_app_name': number_of_apps_with_unique_app_name, - 'number_of_apps_with_unique_developer_email': number_of_apps_with_unique_developer_email, - 'active_apps_names': active_apps_names, - 'average_calls_per_day': average_calls_per_day, - 'average_response_time': average_response_time, - 'top_apis': top_apis, - 'top_warehouse_calls': top_warehouse_calls, - 'top_apps_using_warehouse': top_apps_using_warehouse, - 'user_email_cansearchwarehouse': user_email_cansearchwarehouse, - 'number_of_users_with_cansearchwarehouse': number_of_users_with_cansearchwarehouse, - 'api_host_name': api_host_name, - 'from_date': (datetime.datetime.strptime(from_date, API_DATEFORMAT)).strftime('%Y-%m-%d'), - 'to_date': (datetime.datetime.strptime(to_date, API_DATEFORMAT)).strftime('%Y-%m-%d'), - 'top_apis_bar_chart': top_apis_bar_chart, - 'top_consumers_bar_chart':top_consumers_bar_chart, - 'median_time_to_first_api_call': median_time_to_first_api_call, - # ##'calls_per_day': calls_per_day, - # ##'calls_per_half_day': calls_per_half_day, - 'form': form, - 'excluded_apps':EXCLUDE_APPS, - 'excluded_functions':EXCLUDE_FUNCTIONS, - 'excluded_url_pattern':EXCLUDE_URL_PATTERN, - }) - return context \ No newline at end of file + def get_context_data(self, **kwargs): return self.prepare_general_context(SummaryType.CUSTOM, **kwargs) \ No newline at end of file diff --git a/apimanager/utils/ErrorHandler.py b/apimanager/utils/ErrorHandler.py deleted file mode 100644 index 7b58f58..0000000 --- a/apimanager/utils/ErrorHandler.py +++ /dev/null @@ -1,35 +0,0 @@ -from django.contrib import messages -import functools -from obp.api import API, APIError, LOGGER -from django.http import JsonResponse -import traceback - -def error_once_only(request, err): - """ - Just add the error once - :param request: - :param err: - :return: - """ - LOGGER.exception('error_once_only - Error Message: {}'.format(err)) - storage = messages.get_messages(request) - if str(err) not in [str(m.message) for m in storage]: - messages.error(request, err) - -def exception_handle(fn): - @functools.wraps(fn) - def wrapper(request, *args, **kwargs): - try: - result = fn(request, *args, **kwargs) - if isinstance(result,dict) and 'code' in result and result['code'] >= 400: - error_once_only(request, result['message']) - else: - msg = 'Submitted!' - messages.success(request, msg) - except APIError as err: - error_once_only(request, APIError(Exception("OBP-API server is not running or do not response properly. " - "Please check OBP-API server. Details: " + str(err)))) - except Exception as err: - error_once_only(request, "Unknown Error. Details: " + str(err)) - return JsonResponse({'state': True}) - return wrapper \ No newline at end of file diff --git a/apimanager/utils/__init__.py b/apimanager/utils/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/apimanager/utils/utils.py b/apimanager/utils/utils.py deleted file mode 100644 index 25c4c21..0000000 --- a/apimanager/utils/utils.py +++ /dev/null @@ -1,23 +0,0 @@ -# -*- coding: utf-8 -*- -""" -Base utilities -""" - -from datetime import datetime -from django.contrib.humanize.templatetags.humanize import naturaltime -from base import context_processors - - -def json_serial(obj): - """JSON serializer for objects not serializable by default json code""" - - if isinstance(obj, datetime): - serial = naturaltime(obj) - return serial - raise TypeError('Type not serializable') - -def get_cache_key_for_current_call(request, urlpath): - """we will generate the cache key by login username+urlpath - url path may contain lots of special characters, here we use the hash method first. - """ - return context_processors.api_username(request).get('API_USERNAME') + str(hash(urlpath)) \ No newline at end of file diff --git a/apimanager/webui/views.py b/apimanager/webui/views.py index 44cfa4c..2f4d1f6 100644 --- a/apimanager/webui/views.py +++ b/apimanager/webui/views.py @@ -13,7 +13,7 @@ from django.http import JsonResponse from .forms import WebuiForm from django.urls import reverse_lazy from django.views.decorators.csrf import csrf_exempt -from utils.ErrorHandler import exception_handle, error_once_only +from base.utils import exception_handle, error_once_only class IndexView(LoginRequiredMixin, FormView): """Index view for config""" From e139305e94802455397b581a82e1371fd60a9280 Mon Sep 17 00:00:00 2001 From: Hongwei Date: Fri, 23 Oct 2020 11:08:44 +0200 Subject: [PATCH 12/15] feature/tweaked the default cache TIMEOUT setting --- apimanager/apimanager/settings.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apimanager/apimanager/settings.py b/apimanager/apimanager/settings.py index cff2bd0..9577530 100644 --- a/apimanager/apimanager/settings.py +++ b/apimanager/apimanager/settings.py @@ -71,13 +71,13 @@ MIDDLEWARE = [ 'django.middleware.clickjacking.XFrameOptionsMiddleware', ] -# TIMEOUT is 2592000 seconds = 60*60*24*30 (1 month) +# TIMEOUT is 31104000 seconds = 60*60*24*30*12 (1 year) # MAX_ENTRIES is 1000000 entities CACHES = { 'default': { 'BACKEND': 'django.core.cache.backends.locmem.LocMemCache', 'LOCATION': 'unique-snowflake', - 'TIMEOUT': 600000, + 'TIMEOUT': 31104000, 'OPTIONS': { 'MAX_ENTRIES': 10000000 } From e2ed95d09ddd4aa157ac0877ecd78526bce8fd3f Mon Sep 17 00:00:00 2001 From: Hongwei Date: Fri, 23 Oct 2020 12:07:11 +0200 Subject: [PATCH 13/15] feature/extracted new method get_all_consumers method --- apimanager/metrics/views.py | 80 +++++++++++++------------------------ 1 file changed, 27 insertions(+), 53 deletions(-) diff --git a/apimanager/metrics/views.py b/apimanager/metrics/views.py index 5499c44..7d3d8f9 100644 --- a/apimanager/metrics/views.py +++ b/apimanager/metrics/views.py @@ -338,34 +338,9 @@ class MonthlyMetricsSummaryView(LoginRequiredMixin, TemplateView): def get_total_number_of_apps(self, cleaned_data, from_date, to_date): apps = [] - apps_list = [] from_date = datetime.datetime.strptime(from_date, API_DATEFORMAT) to_date = datetime.datetime.strptime(to_date, API_DATEFORMAT) - urlpath = '/management/consumers' - api = API(self.request.session.get('obp')) - cache_key = get_cache_key_for_current_call(self.request, urlpath) - apicaches=None - try: - apicaches=cache.get(cache_key) - except Exception as err: - apicaches=None - if not apicaches is None: - apps_list=apicaches - else: - try: - apps = api.get(urlpath) - if apps is not None and 'code' in apps and apps['code'] == 403: - error_once_only(self.request, apps['message']) - else: - apps_list = apps["consumers"] - cache.set(cache_key, apps_list) - LOGGER.warning('The cache is setting, url is: {}'.format(urlpath)) - LOGGER.warning('The cache is setting key is: {}'.format(cache_key)) - except APIError as err: - error_once_only(self.request, err) - except Exception as err: - error_once_only(self.request, 'Unknown Error. {}'.format(err)) - + apps_list = self.get_all_consumers() for app in apps_list: app_created_date = datetime.datetime.strptime(app["created"], '%Y-%m-%dT%H:%M:%SZ') @@ -402,6 +377,30 @@ class MonthlyMetricsSummaryView(LoginRequiredMixin, TemplateView): return unique_app_names, number_of_apps_with_unique_app_name, number_of_apps_with_unique_developer_email + def get_all_consumers(self): + urlpath = '/management/consumers' + api = API(self.request.session.get('obp')) + cache_key = get_cache_key_for_current_call(self.request, urlpath) + apicaches = None + try: + apicaches = cache.get(cache_key) + except Exception as err: + apicaches = None + if not apicaches is None: + apps_list = apicaches + else: + try: + apps = api.get(urlpath) + apps_list = apps["consumers"] + cache.set(cache_key, apps_list, 60 * 60) # for the consumers we cache for 1 hour, consumers may be increased + LOGGER.warning('The cache is setting, url is: {}'.format(urlpath)) + LOGGER.warning('The cache is setting key is: {}'.format(cache_key)) + except APIError as err: + error_once_only(self.request, err) + except Exception as err: + error_once_only(self.request, 'Unknown Error. {}'.format(err)) + return apps_list + def calls_per_delta(self, is_included_obp_apps, from_date_string, to_date_string, **delta ): """ return how many calls were made in total per given delta. @@ -736,37 +735,12 @@ class MonthlyMetricsSummaryView(LoginRequiredMixin, TemplateView): return top_apps_using_warehouse def median_time_to_first_api_call(self, from_date, to_date): - if 2>1: - return 0 + return 0 #TODO this cost too much time, do not use this at the moment. form = self.get_form() new_apps_list = [] apps = [] - apps_list = [] + apps_list = self.get_all_consumers() - urlpath_consumers = '/management/consumers' - cache_key = get_cache_key_for_current_call(self.request, urlpath_consumers) - apicaches=None - try: - apicaches=cache.get(cache_key) - except Exception as err: - apicaches=None - if not apicaches is None: - apps_list=apicaches - else: - api = API(self.request.session.get('obp')) - try: - apps = api.get(urlpath_consumers) - if apps is not None and 'code' in apps and apps['code']==403: - error_once_only(self.request, apps['message']) - else: - apps_list = apps["consumers"] - cache.set(cache_key, apps_list) - LOGGER.warning('The cache is setting, url is: {}'.format(urlpath_consumers)) - LOGGER.warning('The cache is setting key is: {}'.format(cache_key)) - except APIError as err: - error_once_only(self.request, err) - except Exception as err: - error_once_only(self.request, 'Unknown Error. {}'.format(err)) for app in apps_list: created_date = datetime.datetime.strptime(app['created'], '%Y-%m-%dT%H:%M:%SZ') From 6ff1c852d3d88f1716e8487a54f7b3c6a0cec02c Mon Sep 17 00:00:00 2001 From: Hongwei Date: Fri, 23 Oct 2020 12:13:06 +0200 Subject: [PATCH 14/15] feature/introduced the django session cache --- apimanager/apimanager/settings.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/apimanager/apimanager/settings.py b/apimanager/apimanager/settings.py index 9577530..c303ea7 100644 --- a/apimanager/apimanager/settings.py +++ b/apimanager/apimanager/settings.py @@ -62,6 +62,7 @@ INSTALLED_APPS = [ ] MIDDLEWARE = [ + 'django.middleware.cache.UpdateCacheMiddleware', 'django.middleware.security.SecurityMiddleware', 'django.contrib.sessions.middleware.SessionMiddleware', 'django.middleware.common.CommonMiddleware', @@ -69,8 +70,12 @@ MIDDLEWARE = [ 'django.contrib.auth.middleware.AuthenticationMiddleware', 'django.contrib.messages.middleware.MessageMiddleware', 'django.middleware.clickjacking.XFrameOptionsMiddleware', + 'django.middleware.cache.FetchFromCacheMiddleware', ] +#cache the view page, we set 60s = 1m, +CACHE_MIDDLEWARE_SECONDS = 60 + # TIMEOUT is 31104000 seconds = 60*60*24*30*12 (1 year) # MAX_ENTRIES is 1000000 entities CACHES = { From 525b52603477934af69011be556e52b218963b0b Mon Sep 17 00:00:00 2001 From: Hongwei Date: Fri, 23 Oct 2020 12:41:36 +0200 Subject: [PATCH 15/15] feature/added the TODO for the cache key --- apimanager/base/utils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/apimanager/base/utils.py b/apimanager/base/utils.py index 54df439..e904df6 100644 --- a/apimanager/base/utils.py +++ b/apimanager/base/utils.py @@ -24,6 +24,7 @@ def get_cache_key_for_current_call(request, urlpath): """we will generate the cache key by login username+urlpath url path may contain lots of special characters, here we use the hash method first. """ + #TODO, we need the obp user.provide there. return context_processors.api_username(request).get('API_USERNAME') + str(hash(urlpath))