Source code for perfsim.helpers.plotter

#  Copyright (C) 2020 Michel Gokan Khan
#  This program is free software; you can redistribute it and/or modify
#  it under the terms of the GNU General Public License as published by
#  the Free Software Foundation; either version 2 of the License, or
#  (at your option) any later version.
#
#  This program is distributed in the hope that it will be useful,
#  but WITHOUT ANY WARRANTY; without even the implied warranty of
#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
#  GNU General Public License for more details.
#
#  You should have received a copy of the GNU General Public License along
#  with this program; if not, write to the Free Software Foundation, Inc.,
#  51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
#
#  This file is a part of the PerfSim project, which is now open source and available under the GPLv2.
#  Written by Michel Gokan Khan, February 2020


from __future__ import annotations

import json
import os
import random
import webbrowser
from typing import TYPE_CHECKING

import matplotlib.pyplot as plt
import networkx as nx
import numpy as np
import pandas as pd
import plotly.figure_factory as pff
# from networkx.drawing.tests.test_pylab import plt
from networkx.drawing.nx_agraph import graphviz_layout
from pandas import DataFrame
from sortedcontainers import SortedList

from perfsim import Utils, Transmission, SimulationScenarioResultDict

if TYPE_CHECKING:
    from perfsim import LoadGenerator


[docs] class Plotter: """ This class is responsible for plotting the simulation results. """
[docs] @staticmethod def draw_topology(cluster): """ Draw the topology of the cluster. """ from networkx.drawing.tests.test_pylab import plt nx.draw(cluster.topology) plt.show()
[docs] @staticmethod def draw_timeline_graph(results: SimulationScenarioResultDict): """ Draw the timeline graph. """ lst = [] colors = set() ticks = set() colors_dict = {} l = SortedList() for sfc_name, sfc_result_dict in results["service_chains"].items(): for iteration_id, lats_dict in sfc_result_dict["latencies"]['iterations'].items(): for index, lats in lats_dict.items(): lst.append({"Task": index, "Start": sfc_result_dict["arrival_times"]['iterations'][iteration_id][index], "Finish": sfc_result_dict["completion_times"]['iterations'][iteration_id][index], "SFC": sfc_name }) ticks.add(sfc_result_dict["arrival_times"]['iterations'][iteration_id][index]) l.add(sfc_result_dict["arrival_times"]['iterations'][iteration_id][index]) while True: rand_color = ["#" + ''.join([random.choice('ABCDEF0123456789') for i in range(6)])][0] if rand_color not in colors: colors.add(rand_color) break colors_dict[sfc_name] = rand_color df = pd.DataFrame(lst) # fig = px.timeline(df, x_start="Start", x_end="Finish", y="Request ID", color="SFC", index_col="Request ID") fig = pff.create_gantt(df, colors=colors_dict, show_colorbar=True, showgrid_x=True, showgrid_y=True, group_tasks=True, index_col="SFC") fig.update_layout(xaxis_type='linear') fig['layout']['xaxis']['tickformat'] = '%L' return fig
[docs] @staticmethod def draw_figures(load_generator: LoadGenerator, scenario_name: str, show_events: bool = True, path_to_save_results: str = os.getcwd() + '/results/') -> None: """ Draw the figures for the simulation results. :param load_generator: :param scenario_name: :param show_events: :param path_to_save_results: """ __base_path = path_to_save_results + scenario_name Utils.mkdir_p(__base_path) Utils.mkdir_p(__base_path + "/hosts") import seaborn as sns palette = sns.color_palette("hls", len(load_generator.threads) + 1) threads_color_code = {} for (key, _t) in enumerate(load_generator.threads): threads_color_code[_t.id] = int(key) threads_color_code["idle"] = len(load_generator.threads) for host in load_generator.sim.cluster.cluster_scheduler.hosts_dict.values(): Utils.mkdir_p(__base_path + "/hosts/" + host.name) count_of_cores = len(host.cpu.cores) # Initialize the figure plt.style.use('seaborn-darkgrid') plot_margin = 1000 # create a color palette _palette = plt.get_cmap('Set1') fig, axes = plt.subplots(count_of_cores, 1, figsize=(10, count_of_cores * 3 / 2)) axes[0].ticklabel_format(useOffset=False, style="plain") axes[1].ticklabel_format(useOffset=False, style="plain") _top = 0.98 if count_of_cores == 8 else 0.9 fig.subplots_adjust(top=_top, bottom=0.12, left=0.055, right=0.965) # Same limits for everybody! plt.xlim(0, 1000) plt.ylim(0, 100) # multiple line plot num = 0 counter = 0 for _core in np.arange(0, count_of_cores): num += 1 df = pd.DataFrame.from_dict(host.cpu.cores[_core].runqueue.threads_total_time).sort_index().fillna(0) _counter = len(df) - 1 try: df.loc[0]["idle"] = 1 except KeyError: continue while _counter > 0: _current_index = df.index[_counter] _previous_index = df.index[_counter - 1] _current_values = df.loc[_current_index] _previous_values = df.loc[_previous_index] if _previous_index + 1 < _current_index: _time_delta = _current_index - _previous_index _value_for_a_nanoseconds = 1 - ((_time_delta - _current_values) / _time_delta) df.loc[int(_previous_index + 1)] = _value_for_a_nanoseconds df.loc[_current_index] = df.loc[_current_index] - _value_for_a_nanoseconds df = df.sort_index().fillna(0) _counter -= 1 _last_index = df.index[-1] + 1 df.loc[_last_index] = 0 df.loc[_last_index]["idle"] = 1 df = df.sort_index().fillna(0) df = df.loc[:, (df != 0).any(axis=0)] # df = df[(df.T != 0).any()] # Find the right spot on the plot ax = plt.gca() ax.get_xaxis().get_major_formatter().set_useOffset(False) ax.get_yaxis().get_major_formatter().set_useOffset(False) ax.get_xaxis().get_major_formatter().set_scientific(False) ax.get_yaxis().get_major_formatter().set_scientific(False) plt.subplot(count_of_cores, 1, num) if len(df) > 0: _len = len(df) df_split = np.array_split(df, _len) _df = DataFrame() for _index, sub_df in enumerate(df_split): time_delta = sub_df.index[0] - (df_split[_index - 1].index[0] if _index != 0 else -1) sum_of_time_slices = sub_df.agg('sum') time_proportion = sum_of_time_slices / time_delta _df[sub_df.index[0]] = time_proportion _df = _df.transpose() # _df.loc[0] = 0 _df = _df.sort_index() # plot every group, but discreet for v in _df: a, = plt.plot(_df.index.tolist(), _df[v], marker='', color=palette[threads_color_code[v]], linewidth=2, alpha=0.7, label=_core) # plt.legend(df.columns.tolist()) counter += 1 counter = 0 # Plot the lineplot # plt.plot(df['x'], df[column], marker='', color=palette(num + counter), linewidth=2.4, alpha=0.9, label=column) # Not ticks everywhere if num in range(7): plt.tick_params(labelbottom='off') if num not in [1, 4, 7]: plt.tick_params(labelleft='off') # Add title plt.title(_core, loc='left', fontsize=12, fontweight=0, color=palette[num]) # plt.subplots_adjust(left=0, bottom=0, right=1, top=1, wspace=0, hspace=0) ax = plt.gca() ax.get_xaxis().get_major_formatter().set_useOffset(False) ax.get_yaxis().get_major_formatter().set_useOffset(False) ax.get_xaxis().get_major_formatter().set_scientific(False) ax.get_yaxis().get_major_formatter().set_scientific(False) # general title plt.suptitle("CPU Usages " + str(host.name), fontsize=13, fontweight=0, color='black', y=0.99 if count_of_cores == 8 else 0.96, style='italic' ) fig.savefig(__base_path + "/hosts/" + host.name + "/cpu.png", dpi=fig.dpi) if show_events: levels = [] # Choose some nice levels for i in np.arange(1, 20): levels.append(-1 * i) levels.append(i) levels = np.tile(levels, int(np.ceil(len(host.timeline_time) / 6)))[:len(host.timeline_time)] fig2, ax2 = plt.subplots(figsize=(20, 10), constrained_layout=True) # Create figure and plot a stem plot with the date ax2.set(title="Simulation events") markerline, stemline, baseline = ax2.stem(host.timeline_time, levels, linefmt="C3-", basefmt="k-", use_line_collection=True) plt.setp(markerline, mec="k", mfc="w", zorder=3) # Shift the markers to the baseline by replacing the y-data by zeros. markerline.set_ydata(np.zeros(len(host.timeline_time))) # annotate lines vert = np.array(['top', 'bottom'])[(levels > 0).astype(int)] for d, l, r, va in zip(host.timeline_time, levels, host.timeline_event, vert): ax2.annotate(r, xy=(d, l), xytext=(-3, np.sign(l) * 3), textcoords="offset points", va=va, ha="right") plt.setp(ax2.get_xticklabels(), rotation=30, ha="right") # remove y axis and spines ax2.get_yaxis().set_visible(False) for spine in ["left", "top", "right"]: ax2.spines[spine].set_visible(False) ax2.margins(y=0.1) try: ax = plt.gca() ax.get_xaxis().get_major_formatter().set_useOffset(False) ax.get_yaxis().get_major_formatter().set_useOffset(False) ax.get_xaxis().get_major_formatter().set_scientific(False) ax.get_yaxis().get_major_formatter().set_scientific(False) except: pass fig2.savefig(__base_path + "/hosts/" + host.name + "/events.png", dpi=fig.dpi) plt.show()
[docs] @staticmethod def draw_graph(G, name, save_dir: str, output_type: str = "html", with_labels: bool = False, relabel: bool = False, node_labels_map=None, show: bool = False): """ Draw a graph. :param G: :param name: :param save_dir: :param output_type: :param with_labels: :param relabel: :param node_labels_map: :param show: :return: """ pos = nx.spring_layout(G) node_labels = {} edge_labels = {} color_map = [] router_color = 'green' host_color = '#6390AF' replica_color = '#FFE5B4' for edge in G.edges(): try: edge_data = G.get_edge_data(u=edge[0], v=edge[1])[0] except KeyError: edge_data = G.get_edge_data(u=edge[0], v=edge[1]) if type(edge[0]).__name__ in ['Host', 'Router', 'MicroserviceReplica']: bandwidth = Transmission.get_bandwidth_on_link(source=edge[0], destination=edge[1]) link_name = str(edge_data["name"]) edge_labels[edge] = link_name + " bw:" + "{:.2e}".format(bandwidth) elif type(edge[0]).__name__ == 'MicroserviceEndpointFunction': edge_labels[edge] = str(list(edge_data.values())[0]["payload"]) + "B" elif type(edge[0]).__name__ == 'tuple' and type(edge[0][1]).__name__ == 'MicroserviceEndpointFunction': edge_labels[edge] = str(edge_data["payload"]) + "B" if output_type == "html": cytoscape_json = Plotter.convert_networkx_graph_to_cytoscape_json(G, edge_labels=edge_labels) cytoscape_js = Plotter.get_cytoscape_template(ready_script=f'var cy = cytoscape({cytoscape_json});') if save_dir is not None: file_path = os.path.join(save_dir, f"{name}.html") Utils.mkdir_p(save_dir) Utils.save_file(file_path=file_path, content=cytoscape_js) else: file_path = os.path.join(".", f"{name}.html") if show: webbrowser.open(f"file://{file_path}") if save_dir is None: os.remove(file_path) return cytoscape_js else: fig = plt.figure(1, figsize=(12, 12)) ax = plt.gca() plt.title('draw_networkx') pos = graphviz_layout(G, prog='dot', args='-Grankdir=LR') nx.draw(G, pos, with_labels=with_labels, arrows=True) if save_dir is not None and save_dir is not False: Utils.mkdir_p(dir_path=save_dir) plt.savefig(save_dir + "alt_graph_" + name + ".pdf") if show: plt.show()
# Credits: Partially from cytoscape.py (cytoscape_data function)
[docs] @staticmethod def convert_networkx_graph_to_cytoscape_json(G, edge_labels: dict = None, node_style: dict = None, edge_style: dict = None, layout="euler") -> str: """ Convert a NetworkX graph to a Cytoscape JSON object. :param G: :param edge_labels: :param node_style: :param edge_style: :param layout: :return: """ # load all nodes into nodes array final = {} final["directed"] = G.is_directed() final["multigraph"] = G.is_multigraph() # final["elements"] = {"nodes": [], "edges": []} final["elements"] = [] added_nodes = {} if node_style is None: node_style = { 'label': 'data(label)', 'width': '60px', 'height': '60px', 'color': 'blue', 'background-fit': 'contain', 'background-clip': 'none' } if edge_style is None: edge_style = { 'label': 'data(label)', 'text-background-color': 'yellow', 'text-background-opacity': 0.4, 'width': '6px', 'curve-style': 'bezier', 'target-arrow-shape': 'triangle', 'control-point-step-size': '140px' } for n in G.nodes(): nparent = None ntype = type(n).__name__ if (ntype == "MicroserviceEndpointFunction" or (ntype == 'tuple' and type(n[1]).__name__ == 'MicroserviceEndpointFunction')): _n = n.microservice if ntype != "tuple" else n[1].microservice _n_id = str(_n) if _n_id not in added_nodes.keys(): nparent = {"group": "nodes", "data": {}, "classes": type(_n).__name__} nparent["data"]["id"] = _n_id nparent["data"]["label"] = _n_id added_nodes[_n_id] = nparent final["elements"].append(nparent) else: nparent = added_nodes[_n_id] nx = {"group": "nodes", "data": {}, "classes": ntype} nx_id = str(n) nx["data"]["id"] = nx_id nx["data"]["label"] = nx_id if ntype != "tuple" else str(n[1]) if nparent is not None: nx["data"]["parent"] = str(n.microservice) if ntype != "tuple" else str(n[1].microservice) added_nodes[nx_id] = nx # final["elements"]["nodes"].append(nx.copy()) final["elements"].append(nx) for e in G.edges(): nx = {"group": "edges", "data": {}} nx["data"]["id"] = str(e[0]) + "_" + str(e[1]) nx["data"]["source"] = str(e[0]) nx["data"]["target"] = str(e[1]) nx["data"]["label"] = str(edge_labels[e]) if edge_labels else str(e[0]) + "->" + str(e[1]) # final["elements"]["edges"].append(nx) final["elements"].append(nx) final["container"] = "----" final["layout"] = {} final["layout"]["name"] = layout final["layout"]["animate"] = "true" final["layout"]["randomize"] = "true" final["layout"]["gravity"] = "-300" final["style"] = [{ "selector": 'node', "style": node_style }, { "selector": 'edge', "style": edge_style }, { "selector": '.Host', "style": { # "border-color": "red", # 'border-width': 3, "shape": "rectangle", "background-image": "https://raw.githubusercontent.com/michelgokan/perfsim-assets/main/images/host.png", "background-color": "white" } }, { "selector": '.Router', "style": { # "border-color": "red", # 'border-width': 3, "shape": "rectangle", "background-image": "https://raw.githubusercontent.com/michelgokan/perfsim-assets/main/images/router.png", "background-color": "white" } }, { "selector": '.MicroserviceReplica', "style": { # "border-color": "red", # 'border-width': 3, "shape": "rectangle", "background-image": "https://raw.githubusercontent.com/michelgokan/perfsim-assets/main/images/replica.png", "background-color": "white" } }] final = json.dumps(final).replace('\"----\"', "document.getElementById('cy')") return final
[docs] @staticmethod def get_cytoscape_template(ready_script: str) -> str: """ Get the template for the Cytoscape graph. :param ready_script: :return: """ return f"""<!DOCTYPE> <html style=\"height: 100%;\"> <head> <script src=\"https://code.jquery.com/jquery-3.6.0.min.js\"></script> <script src=\"https://cdnjs.cloudflare.com/ajax/libs/cytoscape/3.21.1/cytoscape.min.js\"></script> <script src=\"https://cytoscape.org/cytoscape.js-euler/cytoscape-euler.js\"></script> <script src=\"https://unpkg.com/webcola@3.4.0/WebCola/cola.min.js\"></script> <script src=\"https://cytoscape.org/cytoscape.js-cola/cytoscape-cola.js\"></script> <style> #cy {{ width: 100%; height: 100%; }} body {{ width: 100%; height: 100%; }} </style> </head> <body> <div id=\"cy\"></div> <script> $(document).ready(function () {{ {ready_script} }}); </script> </body> </html> """
[docs] @staticmethod def figures_to_html(figs, filename="dashboard.html"): """ Save a list of figures to an HTML file. :param figs: :param filename: :return: """ with open(filename, 'w') as dashboard: dashboard.write("<html><head></head><body><table style='width: 100%; height: 100%'>" + "\n") for fig_and_names in figs: # inner_html = fig_and_names[1].to_html() dashboard.write("<tr><td><iframe id='FileFrame' style='width: 100%; height: 100%'" " src='" + fig_and_names[0] + "'></iframe></td></tr>") dashboard.write("</table></body></html>" + "\n")