artin/API-Manager
1
0
Fork 0
mirror of https://github.com/OpenBankProject/API-Manager.git synced 2026-02-06 14:56:47 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #116 from hongwei1/master

Browse Source 
fixed the Metrics KPI performance
This commit is contained in:
Simon Redfern 2020-10-27 12:41:05 +01:00 committed by GitHub
commit ef4ec2dd7c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
15 changed files with 438 additions and 710 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

41
apimanager/apimanager/settings.py
View File

@ -62,6 +62,7 @@ INSTALLED_APPS = [
]
MIDDLEWARE = [
'django.middleware.cache.UpdateCacheMiddleware',
'django.middleware.security.SecurityMiddleware',
'django.contrib.sessions.middleware.SessionMiddleware',
'django.middleware.common.CommonMiddleware',
@ -69,8 +70,25 @@ MIDDLEWARE = [
'django.contrib.auth.middleware.AuthenticationMiddleware',
'django.contrib.messages.middleware.MessageMiddleware',
'django.middleware.clickjacking.XFrameOptionsMiddleware',
'django.middleware.cache.FetchFromCacheMiddleware',
]
#cache the view page, we set 60s = 1m,
CACHE_MIDDLEWARE_SECONDS = 60
# TIMEOUT is 31104000 seconds = 60*60*24*30*12 (1 year)
# MAX_ENTRIES is 1000000 entities
CACHES = {
'default': {
'BACKEND': 'django.core.cache.backends.locmem.LocMemCache',
'LOCATION': 'unique-snowflake',
'TIMEOUT': 31104000,
'OPTIONS': {
'MAX_ENTRIES': 10000000
}
}
}
ROOT_URLCONF = 'apimanager.urls'
TEMPLATES = [
@ -201,6 +219,9 @@ API_DATETIMEFORMAT = '%Y-%m-%dT%H:%M:%SZ'
#Map Java: yyyy-MM-dd'T'HH:mm:ss.SSS'Z'
API_DATEFORMAT = '%Y-%m-%dT%H:%M:%S.%fZ'
# the API_Manager the web form date format, eg: 2020-10-11
API_MANAGER_DATE_FORMAT= '%Y-%m-%d'
API_HOST = 'http://127.0.0.1:8080'
API_BASE_PATH = '/obp/v'
@ -228,14 +249,6 @@ DIRECTLOGIN_PATH = '/my/logins/direct'
GATEWAYLOGIN_HAS_CBS = False
CACHES = {
'default': {
'BACKEND': 'django.core.cache.backends.memcached.MemcachedCache',
'LOCATION': 'unix:/tmp/memcached.sock',
}
}
# Use BOOTSTRAP3 if you are using Bootstrap 3
BOOTSTRAP4 = {
'include_jquery': True,
@ -248,17 +261,7 @@ EXCLUDE_FUNCTIONS = []
# Url Patterns to exclude when reqeust to OBP-API's api
EXCLUDE_URL_PATTERN = []
# App Name to aggregate metrics
API_EXPLORER_APP_NAME = 'xxx'
CACHES = {
'default': {
'BACKEND': 'django.core.cache.backends.locmem.LocMemCache',
'LOCATION': 'unique-snowflake',
}
}
CACHE_DATEFORMAT = '%Y%m%d%H%M%S'
CACHE_TIME = 3600
CACHE_TIME_SHORT = 300
API_EXPLORER_APP_NAME = 'API Explorer'
# Local settings can override anything in here
try:

90
apimanager/base/context_processors.py
View File

@ -6,8 +6,8 @@ Context processors for base app
from django.conf import settings
from django.contrib import messages
from obp.api import API, APIError
from obp.api import API, APIError, LOGGER
from django.core.cache import cache
def api_root(request):
"""Returns the configured API_ROOT"""
@ -17,39 +17,69 @@ def api_root(request):
def api_username(request):
"""Returns the API username/email of the logged-in user"""
nametodisplay = 'not authenticated'
if request.user.is_authenticated:
try:
api = API(request.session.get('obp'))
data = api.get('/users/current')
username = data['username']
email = data['email']
provider = data['provider']
if "google" in provider:
nametodisplay = email
elif "yahoo" in provider:
nametodisplay = email
else:
nametodisplay = username
except APIError as err:
messages.error(request, err)
except Exception as err:
messages.error(request, err)
return {'API_USERNAME': nametodisplay}
get_current_user_api_url = '/users/current'
#Here we can not get the user from obp-api side, so we use the django auth user id here.
cache_key_django_user_id = request.session._session.get('_auth_user_id')
cache_key = '{},{},{}'.format('api_username',get_current_user_api_url, cache_key_django_user_id)
apicaches=None
try:
apicaches=cache.get(cache_key)
except Exception as err:
apicaches=None
if not apicaches is None:
return apicaches
else:
if request.user.is_authenticated:
try:
api = API(request.session.get('obp'))
data = api.get(get_current_user_api_url)
username = data['username']
email = data['email']
provider = data['provider']
if "google" in provider:
nametodisplay = email
elif "yahoo" in provider:
nametodisplay = email
else:
nametodisplay = username
apicaches=cache.set(cache_key, {'API_USERNAME': nametodisplay})
LOGGER.warning('The cache is setting try to api_user_name:')
LOGGER.warning('The cache is setting key is: {}'.format(cache_key))
except APIError as err:
messages.error(request, err)
except Exception as err:
messages.error(request, err)
return {'API_USERNAME': nametodisplay}
def api_user_id(request):
"""Returns the API user id of the logged-in user"""
user_id = 'not authenticated'
if request.user.is_authenticated:
try:
api = API(request.session.get('obp'))
data = api.get('/users/current')
user_id = data['user_id']
except APIError as err:
messages.error(request, err)
except Exception as err:
messages.error(request, err)
return {'API_USER_ID': user_id}
get_current_user_api_url = '/users/current'
#Here we can not get the user from obp-api side, so we use the django auth user id here.
cache_key_django_user_id = request.session._session.get('_auth_user_id')
cache_key = '{},{},{}'.format('api_user_id',get_current_user_api_url, cache_key_django_user_id)
apicaches=None
try:
apicaches=cache.get(cache_key)
except Exception as err:
apicaches=None
if not apicaches is None:
return apicaches
else:
if request.user.is_authenticated:
try:
api = API(request.session.get('obp'))
data = api.get('/users/current')
user_id = data['user_id']
apicaches=cache.set(cache_key, {'API_USER_ID': user_id})
LOGGER.warning('The cache is setting try to api_user_id:')
LOGGER.warning('The cache is setting key is: {}'.format(cache_key))
except APIError as err:
messages.error(request, err)
except Exception as err:
messages.error(request, err)
return {'API_USER_ID': user_id}
def api_tester_url(request):

64
apimanager/base/utils.py
View File

@ -2,10 +2,15 @@
"""
Base utilities
"""
from datetime import datetime
from django.contrib.humanize.templatetags.humanize import naturaltime
from datetime import datetime, timedelta
from apimanager.settings import API_DATEFORMAT, API_MANAGER_DATE_FORMAT
from base import context_processors
from django.contrib import messages
import functools
from obp.api import APIError, LOGGER
from django.http import JsonResponse
import traceback
def json_serial(obj):
"""JSON serializer for objects not serializable by default json code"""
@ -14,3 +19,56 @@ def json_serial(obj):
serial = naturaltime(obj)
return serial
raise TypeError('Type not serializable')
def get_cache_key_for_current_call(request, urlpath):
"""we will generate the cache key by login username+urlpath
url path may contain lots of special characters, here we use the hash method first.
"""
#TODO, we need the obp user.provide there.
return context_processors.api_username(request).get('API_USERNAME') + str(hash(urlpath))
def error_once_only(request, err):
"""
Just add the error once
:param request:
:param err:
:return:
"""
LOGGER.exception('error_once_only - Error Message: {}'.format(err))
storage = messages.get_messages(request)
if str(err) not in [str(m.message) for m in storage]:
messages.error(request, err)
def exception_handle(fn):
@functools.wraps(fn)
def wrapper(request, *args, **kwargs):
try:
result = fn(request, *args, **kwargs)
if isinstance(result,dict) and 'code' in result and result['code'] >= 400:
error_once_only(request, result['message'])
else:
msg = 'Submitted!'
messages.success(request, msg)
except APIError as err:
error_once_only(request, APIError(Exception("OBP-API server is not running or do not response properly. "
"Please check OBP-API server. Details: " + str(err))))
except Exception as err:
error_once_only(request, "Unknown Error. Details: " + str(err))
return JsonResponse({'state': True})
return wrapper
def convert_form_date_to_obpapi_datetime_format(form_to_date_string):
"""
convert the String 2020-10-22 to 2020-10-22T00:00:00.000000Z
"""
return datetime.strptime(form_to_date_string, API_MANAGER_DATE_FORMAT).strftime(API_DATEFORMAT)
def return_to_days_ago(date, days):
"""
eg:
date 2020-10-22T00:00:00.000000Z
days =1
return 2020-10-21T00:00:00.000000Z
"""
return (datetime.strptime(date, API_DATEFORMAT) - timedelta(days)).strftime(API_DATEFORMAT)

13
apimanager/branches/views.py
View File

@ -16,19 +16,6 @@ from obp.api import API, APIError
from .forms import CreateBranchForm
def error_once_only(request, err):
"""
Just add the error once
:param request:
:param err:
:return:
"""
storage = messages.get_messages(request)
if str(err) not in [str(m.message) for m in storage]:
messages.error(request, err)
class IndexBranchesView(LoginRequiredMixin, FormView):
"""Index view for branches"""
template_name = "branches/index.html"

2
apimanager/dynamicendpoints/views.py
View File

@ -7,7 +7,7 @@ import json
from django.contrib.auth.mixins import LoginRequiredMixin
from django.views.generic import FormView
from obp.api import API, APIError
from utils.ErrorHandler import exception_handle, error_once_only
from base.utils import exception_handle, error_once_only
from .forms import DynamicEndpointsForm
from django.urls import reverse_lazy
from django.views.decorators.csrf import csrf_exempt

3
apimanager/methodrouting/views.py
View File

@ -10,8 +10,9 @@ from django.http import JsonResponse
from django.contrib import messages
from django.contrib.auth.mixins import LoginRequiredMixin
from django.views.generic import FormView
from base.utils import error_once_only, exception_handle
from obp.api import API, APIError
from utils.ErrorHandler import exception_handle, error_once_only
from .forms import MethodRoutingForm
from django.urls import reverse_lazy
from django.views.decorators.csrf import csrf_exempt

22
apimanager/metrics/forms.py
View File

@ -211,7 +211,7 @@ class ConnectorMetricsForm(MetricsForm):
class CustomSummaryForm(forms.Form):
to_date = forms.DateTimeField(
to_date = forms.DateField(
label='To Date',
# input_formats=[settings.API_DATEFORMAT],
# widget=forms.DateTimeInput(
@ -220,12 +220,12 @@ class CustomSummaryForm(forms.Form):
# 'class': 'form-control',
# }
# ),
widget=DateTimePickerInput(format='%Y-%m-%d %H:%M:%S'),
widget=DatePickerInput(format='%Y-%m-%d'),
required=True,
initial=str(datetime.now().strftime('%Y-%m-%d %H:00:00')),
initial=str(datetime.now().strftime('%Y-%m-%d')),
)
from_date_custom = forms.DateTimeField(
from_date_custom = forms.DateField(
label='From Date',
# input_formats=[settings.API_DATEFORMAT],
# widget=forms.DateTimeInput(
@ -234,9 +234,9 @@ class CustomSummaryForm(forms.Form):
# 'class': 'form-control',
# }
# ),
widget=DateTimePickerInput(format='%Y-%m-%d %H:%M:%S'),
widget=DatePickerInput(format='%Y-%m-%d'),
required=True,
initial=(datetime.now() - timedelta(6)).strftime('%Y-%m-%d %H:00:00'),
initial=(datetime.now() - timedelta(6)).strftime('%Y-%m-%d'),
)
include_obp_apps = forms.BooleanField(required=False)
@ -246,8 +246,8 @@ class CustomSummaryForm(forms.Form):
super(CustomSummaryForm, self).__init__(*args, **kwargs)
class MetricsSummaryForm(forms.Form):
to_date = forms.DateTimeField(
class MonthlyMetricsSummaryForm(forms.Form):
to_date = forms.DateField(
label='To Date',
# input_formats=[settings.API_DATEFORMAT],
# widget=forms.DateTimeInput(
@ -256,14 +256,14 @@ class MetricsSummaryForm(forms.Form):
# 'class': 'form-control',
# }
# ),
widget=DateTimePickerInput(format='%Y-%m-%d %H:%M:%S'),
widget=DatePickerInput(format='%Y-%m-%d'),
required=True,
# initial=str(datetime.now().strftime('%Y-%m-%dT%H:%M:%S.%fZ')),
initial=str(datetime.now().strftime('%Y-%m-%d %H:00:00')),
initial=str(datetime.now().strftime('%Y-%m-%d')),
)
include_obp_apps = forms.BooleanField(required=False)
def __init__(self, *args, **kwargs):
kwargs.setdefault('label_suffix', '')
super(MetricsSummaryForm, self).__init__(*args, **kwargs)
super(MonthlyMetricsSummaryForm, self).__init__(*args, **kwargs)

8
apimanager/metrics/templates/metrics/summary.html → apimanager/metrics/templates/metrics/monthly_summary.html
View File

@ -55,12 +55,16 @@
<div class="col-xs-6 col-sm-3">
<div class="form-group">
{{ form.include_obp_apps }} Include System Calls
{{ excluded_functions }}
{{ excluded_url_pattern }}
</div>
{% if excluded_apps %}
{{ excluded_apps }}
{% endif %}
{% if excluded_functions %}
{{ excluded_functions }}
{% endif %}
{% if excluded_url_pattern %}
{{ excluded_url_pattern }}
{% endif %}
</div>
</div>

8
apimanager/metrics/templates/metrics/quarterly_summary.html
View File

@ -81,10 +81,10 @@
<td>Calls per month:</td>
<td><img src="data:image/png;base64, {{ per_month_chart }}" alt="somealt" /></td>
</tr>
<tr>
<td>Calls per day:</td>
<td><img src="data:image/png;base64, {{ per_day_chart }}" alt="somealt" /></td>
</tr>
{# <tr>#}
{# <td>Calls per day:</td>#}
{# <td><img src="data:image/png;base64, {{ per_day_chart }}" alt="somealt" /></td>#}
{# </tr>#}
<tr>
<td>Average number of calls per day: </td>
<td>{{ average_calls_per_day }}</td>

6
apimanager/metrics/urls.py
View File

@ -9,7 +9,7 @@ from .views import (
APIMetricsView,
APISummaryPartialFunctionView,
ConnectorMetricsView,
MetricsSummaryView,
MonthlyMetricsSummaryView,
YearlySummaryView,
QuarterlySummaryView,
WeeklySummaryView,
@ -28,8 +28,8 @@ urlpatterns = [
url(r'^connector/$',
ConnectorMetricsView.as_view(),
name='connector-metrics'),
url(r'^summary/$',
MetricsSummaryView.as_view(),
url(r'^monthly-summary/$',
MonthlyMetricsSummaryView.as_view(),
name='metrics-summary'),
url(r'^yearly-summary/$',
YearlySummaryView.as_view(),

848
apimanager/metrics/views.py
View File

File diff suppressed because it is too large Load Diff

7
apimanager/obp/api.py
View File

@ -71,8 +71,11 @@ class API(object):
Convenience call which uses API_ROOT from settings
"""
url = settings.API_ROOT + urlpath
response = self.call('GET', url)
return self.handle_response(response)
response = self.handle_response(self.call('GET', url))
if response is not None and 'code' in response:
raise APIError(response['message'])
else:
return response
def delete(self, urlpath):
"""

34
apimanager/utils/ErrorHandler.py
View File

@ -1,34 +0,0 @@
from django.contrib import messages
import functools
from obp.api import API, APIError
from django.http import JsonResponse
import traceback
def error_once_only(request, err):
"""
Just add the error once
:param request:
:param err:
:return:
"""
storage = messages.get_messages(request)
if str(err) not in [str(m.message) for m in storage]:
messages.error(request, err)
def exception_handle(fn):
@functools.wraps(fn)
def wrapper(request, *args, **kwargs):
try:
result = fn(request, *args, **kwargs)
if isinstance(result,dict) and 'code' in result and result['code'] >= 400:
error_once_only(request, result['message'])
else:
msg = 'Submitted!'
messages.success(request, msg)
except APIError as err:
error_once_only(request, APIError(Exception("OBP-API server is not running or do not response properly. "
"Please check OBP-API server. Details: " + str(err))))
except Exception as err:
error_once_only(request, "Unknown Error. Details: " + str(err))
return JsonResponse({'state': True})
return wrapper

0
apimanager/utils/__init__.py
View File

2
apimanager/webui/views.py
View File

@ -13,7 +13,7 @@ from django.http import JsonResponse
from .forms import WebuiForm
from django.urls import reverse_lazy
from django.views.decorators.csrf import csrf_exempt
from utils.ErrorHandler import exception_handle, error_once_only
from base.utils import exception_handle, error_once_only
class IndexView(LoginRequiredMixin, FormView):
"""Index view for config"""