diff --git a/apimanager/apimanager/settings.py b/apimanager/apimanager/settings.py index dd26105..9c918ae 100644 --- a/apimanager/apimanager/settings.py +++ b/apimanager/apimanager/settings.py @@ -27,7 +27,7 @@ BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) SECRET_KEY = None # SECURITY WARNING: don't run with debug turned on in production! -DEBUG = True +DEBUG = False diff --git a/apimanager/metrics/forms.py b/apimanager/metrics/forms.py index 916ac52..fd680c2 100644 --- a/apimanager/metrics/forms.py +++ b/apimanager/metrics/forms.py @@ -212,18 +212,16 @@ class CustomSummaryForm(forms.Form): required=True, initial=(datetime.now() - timedelta(6)).strftime(API_MANAGER_DATE_FORMAT), ) - #exclude_app_names = forms.CharField( - # label=_('Exclude App Names'), - # widget=forms.TextInput( - # attrs={ - # 'class': FORM_CONTROL, - # } - # ), - # required=False, - # initial='API-Manager', - #) - #include_obp_apps = forms.BooleanField(required=False, label=_('Include System Data')) - include_obp_apps = forms.BooleanField(required=False) + include_app_names = forms.CharField( + label=_('Include App Names'), + widget=forms.TextInput( + attrs={ + 'class': FORM_CONTROL, + } + ), + required=False, + ) + def __init__(self, *args, **kwargs): kwargs.setdefault('label_suffix', '') @@ -237,16 +235,15 @@ class MonthlyMetricsSummaryForm(forms.Form): #initial=str(datetime.now().strftime('%Y-%m-%dT%H:%M:%S.%fZ')), initial=str(datetime.now().strftime(API_MANAGER_DATE_FORMAT)), ) - #exclude_app_names = forms.CharField( - # label=_('Exclude App Names'), - # widget=forms.TextInput( - # attrs={ - # 'class': FORM_CONTROL, - # } - # ), - # required=False, - # initial='API-Manager', - #) + include_app_names = forms.CharField( + label=_('Include App Names'), + widget=forms.TextInput( + attrs={ + 'class': FORM_CONTROL, + } + ), + required=False, + ) def __init__(self, *args, **kwargs): kwargs.setdefault('label_suffix', '') diff --git a/apimanager/metrics/templates/metrics/custom_summary.html b/apimanager/metrics/templates/metrics/custom_summary.html index 2733fb2..dce0171 100644 --- a/apimanager/metrics/templates/metrics/custom_summary.html +++ b/apimanager/metrics/templates/metrics/custom_summary.html @@ -50,10 +50,10 @@
- {% if form.exclude_app_names.errors %}
{{ form.exclude_app_names.errors }}
{% endif %} + {% if form.include_app_names.errors %}
{{ form.include_app_names.errors }}
{% endif %}
- {{ form.exclude_app_names.label_tag }} - {{ form.exclude_app_names }} + {{ form.include_app_names.label_tag }} + {{ form.include_app_names }}
diff --git a/apimanager/metrics/templates/metrics/daily_summary.html b/apimanager/metrics/templates/metrics/daily_summary.html index ada32fe..74f8915 100644 --- a/apimanager/metrics/templates/metrics/daily_summary.html +++ b/apimanager/metrics/templates/metrics/daily_summary.html @@ -50,10 +50,10 @@
- {% if form.exclude_app_names.errors %}
{{ form.exclude_app_names.errors }}
{% endif %} + {% if form.include_app_names.errors %}
{{ form.include_app_names.errors }}
{% endif %}
- {{ form.exclude_app_names.label_tag }} - {{ form.exclude_app_names }} + {{ form.include_app_names.label_tag }} + {{ form.include_app_names }}
diff --git a/apimanager/metrics/templates/metrics/hourly_summary.html b/apimanager/metrics/templates/metrics/hourly_summary.html index 74e3661..ea7f147 100644 --- a/apimanager/metrics/templates/metrics/hourly_summary.html +++ b/apimanager/metrics/templates/metrics/hourly_summary.html @@ -49,10 +49,10 @@
- {% if form.exclude_app_names.errors %}
{{ form.exclude_app_names.errors }}
{% endif %} + {% if form.include_app_names.errors %}
{{ form.include_app_names.errors }}
{% endif %}
- {{ form.exclude_app_names.label_tag }} - {{ form.exclude_app_names }} + {{ form.include_app_names.label_tag }} + {{ form.include_app_names }}
diff --git a/apimanager/metrics/templates/metrics/monthly_summary.html b/apimanager/metrics/templates/metrics/monthly_summary.html index c0cee4f..5cc4082 100644 --- a/apimanager/metrics/templates/metrics/monthly_summary.html +++ b/apimanager/metrics/templates/metrics/monthly_summary.html @@ -45,10 +45,10 @@
- {% if form.exclude_app_names.errors %}
{{ form.exclude_app_names.errors }}
{% endif %} + {% if form.include_app_names.errors %}
{{ form.include_app_names.errors }}
{% endif %}
- {{ form.exclude_app_names.label_tag }} - {{ form.exclude_app_names }} + {{ form.include_app_names.label_tag }} + {{ form.include_app_names }}
diff --git a/apimanager/metrics/templates/metrics/quarterly_summary.html b/apimanager/metrics/templates/metrics/quarterly_summary.html index 3490cb3..dd48a2d 100644 --- a/apimanager/metrics/templates/metrics/quarterly_summary.html +++ b/apimanager/metrics/templates/metrics/quarterly_summary.html @@ -49,10 +49,10 @@
- {% if form.exclude_app_names.errors %}
{{ form.exclude_app_names.errors }}
{% endif %} + {% if form.include_app_names.errors %}
{{ form.include_app_names.errors }}
{% endif %}
- {{ form.exclude_app_names.label_tag }} - {{ form.exclude_app_names }} + {{ form.include_app_names.label_tag }} + {{ form.include_app_names }}
diff --git a/apimanager/metrics/templates/metrics/weekly_summary.html b/apimanager/metrics/templates/metrics/weekly_summary.html index 18a8257..76c9e27 100644 --- a/apimanager/metrics/templates/metrics/weekly_summary.html +++ b/apimanager/metrics/templates/metrics/weekly_summary.html @@ -51,10 +51,10 @@
- {% if form.exclude_app_names.errors %}
{{ form.exclude_app_names.errors }}
{% endif %} + {% if form.include_app_names.errors %}
{{ form.include_app_names.errors }}
{% endif %}
- {{ form.exclude_app_names.label_tag }} - {{ form.exclude_app_names }} + {{ form.include_app_names.label_tag }} + {{ form.include_app_names }}
diff --git a/apimanager/metrics/templates/metrics/yearly_summary.html b/apimanager/metrics/templates/metrics/yearly_summary.html index 505c8dd..28ce3eb 100644 --- a/apimanager/metrics/templates/metrics/yearly_summary.html +++ b/apimanager/metrics/templates/metrics/yearly_summary.html @@ -51,10 +51,10 @@
- {% if form.exclude_app_names.errors %}
{{ form.exclude_app_names.errors }}
{% endif %} + {% if form.include_app_names.errors %}
{{ form.include_app_names.errors }}
{% endif %}
- {{ form.exclude_app_names.label_tag }} - {{ form.exclude_app_names }} + {{ form.include_app_names.label_tag }} + {{ form.include_app_names }}
diff --git a/apimanager/metrics/views.py b/apimanager/metrics/views.py index 6cde57d..c18a8ea 100644 --- a/apimanager/metrics/views.py +++ b/apimanager/metrics/views.py @@ -11,7 +11,7 @@ from enum import Enum from django.conf import settings from apimanager import local_settings -from apimanager.settings import API_HOST, EXCLUDE_APPS, EXCLUDE_FUNCTIONS, EXCLUDE_URL_PATTERN, API_EXPLORER_APP_NAME, API_DATE_FORMAT, API_DATE_TIME_FORMAT +from apimanager.settings import API_HOST, EXCLUDE_APPS, EXCLUDE_FUNCTIONS, EXCLUDE_URL_PATTERN, API_EXPLORER_APP_NAME, API_DATE_FORMAT, API_DATE_TIME_FORMAT, DEBUG from django.contrib import messages from django.contrib.auth.mixins import LoginRequiredMixin from django.views.generic import TemplateView @@ -33,6 +33,8 @@ import matplotlib matplotlib.use('Agg') import matplotlib.pyplot as plt import statistics +import urllib.parse + CACHE_SETTING_URL_MSG = "The cache setting url is" CACHE_SETTING_KEY_MSG = "The cache setting key is" @@ -146,10 +148,16 @@ class MetricsView(LoginRequiredMixin, TemplateView): metrics = api.get(urlpath) metrics = self.to_django(metrics['metrics']) except APIError as err: + if DEBUG: + raise(err) error_once_only(self.request, err) except KeyError as err: + if DEBUG: + raise(err) error_once_only(self.request, metrics['message']) except Exception as err: + if DEBUG: + raise(err) error_once_only(self.request, err) return metrics @@ -246,14 +254,45 @@ class MonthlyMetricsSummaryView(LoginRequiredMixin, TemplateView): params = '&'.join(params) return params - - def get_aggregate_metrics(self, from_date, to_date): + def get_app_name_parameters(self, include_app_names): + #if len(app_names) == 1: + # return + #1. Parse include_app_names create a list using a commo (,) separator + #2. Trim each word (remove space), + #3. Then is one word, &app_name = thing + #4. IF IT IS MORE than one word then return number of app without space + #5. return either &app_name=thing + #6. Or + #7. return &include_app_names=thing1,thing2, + #8. url encode + #app_names = [] + #input_string = "simon says, foo, bar , App 2 " + input_string = include_app_names.strip() + result = "" + if input_string != "": + input_list = input_string.strip().split(",") + #print("input_list is:", input_list) + cleaned_list = [item.strip() for item in input_list] + #print("cleaned_list is: ", cleaned_list) + cleaned_string=', '.join([str(item) for item in cleaned_list]) + #print("cleaned_string is:", cleaned_string) + url_encoded_string = urllib.parse.quote(cleaned_string) + #print("url_encoded_string is:", url_encoded_string) + if len(cleaned_list) == 0: + result = "" + elif len(cleaned_list) == 1: + result = "&app_name={}".format(url_encoded_string) + else: + result = "&include_app_names={}".format(url_encoded_string) + return result + def get_aggregate_metrics(self, from_date, to_date, include_app_names): """ Gets the metrics from the API, using given parameters, There are different use cases, so we accept different parameters. only_show_api_explorer_metrics has the default value False, because it is just used for app = API_Explorer. """ try: + print("get_app_name_parameters is: ", self.get_app_name_parameters(include_app_names)) api_calls_total = 0 average_response_time = 0 url_path = '/management/aggregate-metrics' @@ -264,7 +303,9 @@ class MonthlyMetricsSummaryView(LoginRequiredMixin, TemplateView): # from_date, to_date, ",".join(local_settings.EXCLUDE_FUNCTIONS), ",".join(local_settings.EXCLUDE_URL_PATTERN)) # #else: - url_path = url_path + '?from_date={}&to_date={}'.format(from_date, to_date) + url_path = url_path + '?from_date={}&to_date={}{}'.format(from_date, to_date, self.get_app_name_parameters(include_app_names)) + #print("get_app_name_parameters(include_app_names) is:", self.get_app_name_parameters(include_app_names)) + #print("url_path is: ", url_path) cache_key = get_cache_key_for_current_call(self.request, url_path) api_cache = None try: @@ -284,8 +325,12 @@ class MonthlyMetricsSummaryView(LoginRequiredMixin, TemplateView): api_calls_total, average_response_time, cache_key, from_date, metrics, to_date, url_path) return api_calls_total, average_response_time, int(average_calls_per_day) except APIError as err: + if DEBUG: + raise(err) error_once_only(self.request, err) except Exception as err: + if DEBUG: + raise(err) error_once_only(self.request, err) def get_internal_api_call_metrics(self, api_calls_total, average_response_time, cache_key, from_date, metrics, @@ -317,6 +362,8 @@ class MonthlyMetricsSummaryView(LoginRequiredMixin, TemplateView): except APIError as err: error_once_only(self.request, err) except Exception as err: + if DEBUG: + raise(err) error_once_only(self.request, err) else: urlpath = '/management/metrics/top-consumers?from_date={}&to_date={}&exclude_implemented_by_partial_functions={}&exclude_url_pattern={}'.format( @@ -326,8 +373,12 @@ class MonthlyMetricsSummaryView(LoginRequiredMixin, TemplateView): apps = api.get(urlpath) active_apps_list = list(apps['top_consumers']) except APIError as err: + if DEBUG: + raise(err) error_once_only(self.request, err) except Exception as err: + if DEBUG: + raise(err) error_once_only(self.request, err) return active_apps_list @@ -391,12 +442,16 @@ class MonthlyMetricsSummaryView(LoginRequiredMixin, TemplateView): LOGGER.warning('{0}: {1}'.format(CACHE_SETTING_URL_MSG, urlpath)) LOGGER.warning('{0}: {1}'.format(CACHE_SETTING_KEY_MSG, cache_key)) except APIError as err: + if DEBUG: + raise(err) error_once_only(self.request, err) except Exception as err: + if DEBUG: + raise(err) error_once_only(self.request, err) return apps_list - def calls_per_delta(self, from_date_string, to_date_string, **delta ): + def calls_per_delta(self, from_date_string, to_date_string, include_app_names, **delta ): """ return how many calls were made in total per given delta. Here we need to convert date_string to datetime object, and calculate the dates. @@ -415,12 +470,14 @@ class MonthlyMetricsSummaryView(LoginRequiredMixin, TemplateView): # here we need to first convert datetime object to String form_date= from_datetime_object.strftime(API_DATE_FORMAT) to_date= time_delta_in_loop.strftime(API_DATE_FORMAT) - aggregate_metrics = self.get_aggregate_metrics(form_date, to_date, ) + aggregate_metrics = self.get_aggregate_metrics(form_date, to_date, include_app_names) result = aggregate_metrics[0] result_list_pure.append(result) result_list.append('{} - {} # {}'.format(from_datetime_object, time_delta_in_loop, result)) date_list.append(from_datetime_object) except Exception as err: + if DEBUG: + raise(err) error_once_only(self.request, err) break @@ -430,20 +487,20 @@ class MonthlyMetricsSummaryView(LoginRequiredMixin, TemplateView): return (result_list, result_list_pure, date_list) - def calls_per_month(self,from_date, to_date): + def calls_per_month(self,from_date, to_date, include_app_names): """ Convenience function to print number of calls per month It is actually 30 days, not a month """ - calls_per_month_list, calls_per_month, month_list = self.calls_per_delta(from_date, to_date, days=30) + calls_per_month_list, calls_per_month, month_list = self.calls_per_delta(from_date, to_date, include_app_names, days=30) return calls_per_month_list, calls_per_month, month_list - def calls_per_day(self,from_date, to_date): + def calls_per_day(self,from_date, to_date, include_app_names): """ Convenience function to print number of calls per day """ - calls_per_day, calls_per_day_pure, date_list = self.calls_per_delta(from_date, to_date, days=1) + calls_per_day, calls_per_day_pure, date_list = self.calls_per_delta(from_date, to_date, include_app_names, days=1) if len(calls_per_day) >= 90: calls_per_day = calls_per_day[-90:] @@ -457,18 +514,18 @@ class MonthlyMetricsSummaryView(LoginRequiredMixin, TemplateView): return calls_per_day, calls_per_day_pure, date_list - def calls_per_half_day(self,from_date): + def calls_per_half_day(self,from_date, include_app_names): """ Convenience function to print number of calls per half day """ return self.calls_per_delta(from_date, hours=12) - def calls_per_hour(self,from_date, to_date): + def calls_per_hour(self,from_date, to_date, include_app_names): """ Convenience function to print number of calls per hour """ - calls_per_hour_list, calls_per_hour, hour_list = self.calls_per_delta(from_date, to_date, hours=1) + calls_per_hour_list, calls_per_hour, hour_list = self.calls_per_delta(from_date, to_date, include_app_names, hours=1) return calls_per_hour_list, calls_per_hour, hour_list def plot_line_chart(self, plot_data, date_month_list, period): @@ -602,6 +659,8 @@ class MonthlyMetricsSummaryView(LoginRequiredMixin, TemplateView): except KeyError as err: messages.error(self.request, 'KeyError: {}'.format(err)) except Exception as err: + if DEBUG: + raise(err) error_once_only(self.request, err) user_email_cansearchwarehouse = dict(zip(users_with_cansearchwarehouse, email_with_cansearchwarehouse)) @@ -626,8 +685,12 @@ class MonthlyMetricsSummaryView(LoginRequiredMixin, TemplateView): else: data = data[data_key] except APIError as err: + if DEBUG: + raise(err) error_once_only(self.request, err) except Exception as err: + if DEBUG: + raise(err) error_once_only(self.request, err) return data @@ -680,8 +743,12 @@ class MonthlyMetricsSummaryView(LoginRequiredMixin, TemplateView): if "elasticSearchWarehouse" in api['Implemented_by_partial_function']: top_warehouse_calls.append(api) except APIError as err: + if DEBUG: + raise(err) error_once_only(self.request, err) except Exception as err: + if DEBUG: + raise(err) error_once_only(self.request, err) return top_warehouse_calls @@ -699,8 +766,12 @@ class MonthlyMetricsSummaryView(LoginRequiredMixin, TemplateView): else: top_apps_using_warehouse = top_apps_using_warehouse["top_consumers"][:2] except APIError as err: + if DEBUG: + raise(err) error_once_only(self.request, err) except Exception as err: + if DEBUG: + raise(err) error_once_only(self.request, err) return top_apps_using_warehouse @@ -757,8 +828,12 @@ class MonthlyMetricsSummaryView(LoginRequiredMixin, TemplateView): except APIError as err: + if DEBUG: + raise(err) error_once_only(self.request, err) except Exception as err: + if DEBUG: + raise(err) error_once_only(self.request, 'Unknown Error. {}'.format(err)) if times_to_first_call: @@ -781,7 +856,7 @@ class MonthlyMetricsSummaryView(LoginRequiredMixin, TemplateView): per_hour_chart=[] if form.is_valid(): # = form.cleaned_data.get('include_obp_apps') - #exclude_app_names = form.cleaned_data.get("exclude_app_names") + include_app_names = form.cleaned_data.get("include_app_names") #if exclude_app_names not in local_settings.EXCLUDE_APPS: # error_once_only(self.request, "Invalid Exclude App Name, Please select" + str(local_settings.EXCLUDE_APPS) + "Anyone of these") form_to_date_string = form.data['to_date'] @@ -790,31 +865,31 @@ class MonthlyMetricsSummaryView(LoginRequiredMixin, TemplateView): if (web_page_type == SummaryType.DAILY): # for one day, the from_date is 1 day ago. from_date = return_to_days_ago(to_date, 1) - calls_per_hour_list, calls_per_hour, hour_list = self.calls_per_hour(from_date, to_date) + calls_per_hour_list, calls_per_hour, hour_list = self.calls_per_hour(from_date, to_date, include_app_names) per_hour_chart = self.plot_line_chart(calls_per_hour, hour_list, 'hour') if (web_page_type == SummaryType.WEEKLY): # for one month, the from_date is 7 days ago. from_date = return_to_days_ago(to_date, 7) - calls_per_day_list, calls_per_day, date_list = self.calls_per_day(from_date, to_date) + calls_per_day_list, calls_per_day, date_list = self.calls_per_day(from_date, to_date, include_app_names) per_day_chart = self.plot_line_chart(calls_per_day, date_list, "day") if (web_page_type == SummaryType.MONTHLY): # for one month, the from_date is 30 days ago. from_date = return_to_days_ago(to_date, 30) - calls_per_day_list, calls_per_day, date_list = self.calls_per_day(from_date, to_date) + calls_per_day_list, calls_per_day, date_list = self.calls_per_day(from_date, to_date, include_app_names) per_day_chart = self.plot_line_chart(calls_per_day, date_list, "day") if (web_page_type == SummaryType.QUARTERLY): # for one quarter, the from_date is 90 days ago. from_date = (datetime.datetime.strptime(to_date, API_DATE_FORMAT) - timedelta(90)).strftime(API_DATE_FORMAT) - calls_per_month_list, calls_per_month, month_list = self.calls_per_month(from_date, to_date) + calls_per_month_list, calls_per_month, month_list = self.calls_per_month(from_date, to_date, include_app_names) per_month_chart = self.plot_line_chart(calls_per_month, month_list, 'month') if (web_page_type == SummaryType.YEARLY): from_date = return_to_days_ago(to_date, 365) #calls_per_month_list, calls_per_month, month_list = self.calls_per_month(, from_date, to_date) - calls_per_month_list, calls_per_month, month_list = self.calls_per_month(from_date, to_date) + calls_per_month_list, calls_per_month, month_list = self.calls_per_month(from_date, to_date, include_app_names) per_month_chart = self.plot_line_chart(calls_per_month, month_list, "month") if (web_page_type == SummaryType.CUSTOM): @@ -822,7 +897,7 @@ class MonthlyMetricsSummaryView(LoginRequiredMixin, TemplateView): form_from_date_string = form.data['from_date_custom'] from_date = convert_form_date_to_obpapi_datetime_format(form_from_date_string) #calls_per_day_list, calls_per_day, date_list = self.calls_per_day(, from_date, to_date) - calls_per_day_list, calls_per_day, date_list = self.calls_per_day(from_date, to_date) + calls_per_day_list, calls_per_day, date_list = self.calls_per_day(from_date, to_date,include_app_names) per_day_chart = self.plot_line_chart(calls_per_day, date_list, "day") api_host_name = API_HOST @@ -835,7 +910,7 @@ class MonthlyMetricsSummaryView(LoginRequiredMixin, TemplateView): top_consumers = self.get_top_consumers(form.cleaned_data, from_date, to_date) top_consumers_bar_chart = self.plot_topconsumer_bar_chart(top_consumers) top_warehouse_calls = self.get_top_warehouse_calls(form.cleaned_data, from_date, to_date) - api_calls, average_response_time, average_calls_per_day = self.get_aggregate_metrics(from_date, to_date) + api_calls, average_response_time, average_calls_per_day = self.get_aggregate_metrics(from_date, to_date, include_app_names) unique_app_names, number_of_apps_with_unique_app_name, number_of_apps_with_unique_developer_email = self.get_total_number_of_apps( form.cleaned_data, from_date, to_date) active_apps_list = self.get_active_apps(from_date, to_date) @@ -871,6 +946,8 @@ class MonthlyMetricsSummaryView(LoginRequiredMixin, TemplateView): else: error_once_only(self.request, str(form.errors)) except Exception as err: + if DEBUG: + raise(err) error_once_only(self.request, err) def _daily_and_weekly(self, web_page_type,to_date, per_hour_chart, per_day_chart, from_date): if (web_page_type == SummaryType.DAILY): @@ -897,7 +974,7 @@ class MonthlyMetricsSummaryView(LoginRequiredMixin, TemplateView): if (web_page_type == SummaryType.QUARTERLY): # for one quarter, the from_date is 90 days ago. from_date = (datetime.datetime.strptime(to_date, API_DATE_FORMAT) - timedelta(90)).strftime(API_DATE_FORMAT) - calls_per_month_list, calls_per_month, month_list = self.calls_per_month(from_date, to_date) + calls_per_month_list, calls_per_month, month_list = self.calls_per_month(from_date, to_date, include_app_names) per_month_chart = self.plot_line_chart(calls_per_month, month_list, 'month') return (from_date, per_day_chart, per_month_chart) @@ -905,7 +982,7 @@ class MonthlyMetricsSummaryView(LoginRequiredMixin, TemplateView): def _yearly_and_custom(self, web_page_type,to_date, per_month_chart, per_day_chart, from_date): if (web_page_type == SummaryType.YEARLY): from_date = return_to_days_ago(to_date, 365) - calls_per_month_list, calls_per_month, month_list = self.calls_per_month(from_date, to_date) + calls_per_month_list, calls_per_month, month_list = self.calls_per_month(from_date, to_date, include_app_names) per_month_chart = self.plot_line_chart(calls_per_month, month_list, "month") if (web_page_type == SummaryType.CUSTOM):