286 lines
12 KiB
Python
286 lines
12 KiB
Python
#!/usr/bin/env python3
|
|
#
|
|
# Copyright 2020 MongoDB Inc.
|
|
#
|
|
# Permission is hereby granted, free of charge, to any person obtaining
|
|
# a copy of this software and associated documentation files (the
|
|
# "Software"), to deal in the Software without restriction, including
|
|
# without limitation the rights to use, copy, modify, merge, publish,
|
|
# distribute, sublicense, and/or sell copies of the Software, and to
|
|
# permit persons to whom the Software is furnished to do so, subject to
|
|
# the following conditions:
|
|
#
|
|
# The above copyright notice and this permission notice shall be included
|
|
# in all copies or substantial portions of the Software.
|
|
#
|
|
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
|
|
# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
|
|
# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
|
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
|
# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
|
# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
|
# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
|
#
|
|
"""
|
|
Flask backend web server.
|
|
|
|
The backend interacts with the graph_analyzer to perform queries on various libdeps graphs.
|
|
"""
|
|
|
|
from pathlib import Path
|
|
from collections import namedtuple, OrderedDict
|
|
|
|
import flask
|
|
import networkx
|
|
|
|
from flask_socketio import SocketIO, emit
|
|
from flask_cors import CORS
|
|
from flask_session import Session
|
|
from lxml import etree
|
|
|
|
import libdeps.graph
|
|
import libdeps.analyzer
|
|
|
|
|
|
class BackendServer:
|
|
"""Create small class for storing variables and state of the backend."""
|
|
|
|
# pylint: disable=too-many-instance-attributes
|
|
def __init__(self, graphml_dir, frontend_url):
|
|
"""Create and setup the state variables."""
|
|
self.app = flask.Flask(__name__)
|
|
self.socketio = SocketIO(self.app, cors_allowed_origins=frontend_url)
|
|
self.app.config['CORS_HEADERS'] = 'Content-Type'
|
|
CORS(self.app, resources={r"/*": {"origins": frontend_url}})
|
|
|
|
self.app.add_url_rule("/graph_files", "return_graph_files", self.return_graph_files)
|
|
self.socketio.on_event('git_hash_selected', self.git_hash_selected)
|
|
self.socketio.on_event('row_selected', self.row_selected)
|
|
|
|
self.loaded_graphs = {}
|
|
self.current_selected_rows = {}
|
|
self.graphml_dir = Path(graphml_dir)
|
|
self.frontend_url = frontend_url
|
|
|
|
self.graph_file_tuple = namedtuple('GraphFile', ['version', 'git_hash', 'graph_file'])
|
|
self.graph_files = self.get_graphml_files()
|
|
|
|
try:
|
|
default_selected_graph = list(self.graph_files.items())[0][1].graph_file
|
|
self.load_graph_from_file(default_selected_graph)
|
|
self._dependents_graph = networkx.reverse_view(self._dependency_graph)
|
|
except (IndexError, AttributeError) as ex:
|
|
print(ex)
|
|
print(
|
|
f"Failed to load read a graph file from {list(self.graph_files.items())} for graphml_dir '{self.graphml_dir}'"
|
|
)
|
|
exit(1)
|
|
|
|
def load_graph_from_file(self, file_path):
|
|
"""Load a graph file from disk and handle version."""
|
|
|
|
graph = libdeps.graph.LibdepsGraph(networkx.read_graphml(file_path))
|
|
if graph.graph['graph_schema_version'] == 1:
|
|
self._dependents_graph = graph
|
|
self._dependency_graph = networkx.reverse_view(self._dependents_graph)
|
|
else:
|
|
self._dependency_graph = graph
|
|
self._dependents_graph = networkx.reverse_view(self._dependency_graph)
|
|
|
|
def get_app(self):
|
|
"""Return the app and socketio instances."""
|
|
|
|
return self.app, self.socketio
|
|
|
|
def get_graph_build_data(self, graph_file):
|
|
"""Fast method for extracting basic build data from the graph file."""
|
|
|
|
version = ''
|
|
git_hash = ''
|
|
# pylint: disable=c-extension-no-member
|
|
for _, element in etree.iterparse(
|
|
str(graph_file), tag="{http://graphml.graphdrawing.org/xmlns}data"):
|
|
if element.get('key') == 'graph_schema_version':
|
|
version = element.text
|
|
if element.get('key') == 'git_hash':
|
|
git_hash = element.text
|
|
element.clear()
|
|
if version and git_hash:
|
|
break
|
|
return self.graph_file_tuple(version, git_hash, graph_file)
|
|
|
|
def get_graphml_files(self):
|
|
"""Find all graphml files in the target graphml dir."""
|
|
|
|
graph_files = OrderedDict()
|
|
for graph_file in self.graphml_dir.glob("**/*.graphml"):
|
|
graph_file_tuple = self.get_graph_build_data(graph_file)
|
|
graph_files[graph_file_tuple.git_hash[:7]] = graph_file_tuple
|
|
return graph_files
|
|
|
|
def return_graph_files(self):
|
|
"""Prepare the list of graph files for the frontend."""
|
|
|
|
data = {'graph_files': []}
|
|
for i, (_, graph_file_data) in enumerate(self.graph_files.items(), start=1):
|
|
data['graph_files'].append({
|
|
'id': i, 'version': graph_file_data.version, 'git': graph_file_data.git_hash[:7],
|
|
'selected': False
|
|
})
|
|
return data
|
|
|
|
def send_node_infos(self):
|
|
"""Search through the selected rows and find information about the selected rows."""
|
|
|
|
with self.app.test_request_context():
|
|
|
|
nodeinfo_data = {'nodeInfos': []}
|
|
|
|
for node, _ in self.current_selected_rows.items():
|
|
|
|
nodeinfo_data['nodeInfos'].append({
|
|
'id':
|
|
len(nodeinfo_data['nodeInfos']),
|
|
'node':
|
|
str(node),
|
|
'name':
|
|
node.name,
|
|
'attribs': [{
|
|
'name': key, 'value': value
|
|
} for key, value in self._dependents_graph.nodes(data=True)[str(node)].items()],
|
|
'dependers': [{
|
|
'node':
|
|
depender, 'symbols':
|
|
self._dependents_graph[str(node)][depender].get('symbols',
|
|
'').split(' ')
|
|
} for depender in self._dependents_graph[str(node)]],
|
|
'dependencies': [{
|
|
'node':
|
|
dependency, 'symbols':
|
|
self._dependents_graph[dependency][str(node)].get('symbols',
|
|
'').split(' ')
|
|
} for dependency in self._dependency_graph[str(node)]],
|
|
})
|
|
|
|
self.socketio.emit("node_infos", nodeinfo_data)
|
|
|
|
def send_graph_data(self):
|
|
"""Convert the current selected rows into a format for D3."""
|
|
|
|
with self.app.test_request_context():
|
|
|
|
nodes = set()
|
|
links = set()
|
|
|
|
for node, _ in self.current_selected_rows.items():
|
|
nodes.add(
|
|
tuple({
|
|
'id': str(node), 'name': node.name, 'type': self._dependents_graph.nodes()
|
|
[str(node)]['bin_type']
|
|
}.items()))
|
|
|
|
for depender in self._dependency_graph[str(node)]:
|
|
|
|
depender_path = Path(depender)
|
|
if self._dependents_graph[depender][str(node)].get('direct'):
|
|
nodes.add(
|
|
tuple({
|
|
'id':
|
|
str(depender_path), 'name':
|
|
depender_path.name, 'type':
|
|
self._dependents_graph.nodes()[str(depender_path)]
|
|
['bin_type']
|
|
}.items()))
|
|
links.add(
|
|
tuple({'source': str(node), 'target': str(depender_path)}.items()))
|
|
|
|
node_data = {
|
|
'graphData': {
|
|
'nodes': [dict(node) for node in nodes],
|
|
'links': [dict(link) for link in links],
|
|
}, 'selectedNodes': [str(node) for node in list(self.current_selected_rows.keys())]
|
|
}
|
|
self.socketio.emit("graph_data", node_data)
|
|
|
|
def row_selected(self, message):
|
|
"""Construct the new graphData nodeInfo when a cell is selected."""
|
|
|
|
print(f"Got row {message}!")
|
|
|
|
if message['isSelected'] == 'flip':
|
|
if message['data']['node'] in self.current_selected_rows:
|
|
self.current_selected_rows.pop(message['data']['node'])
|
|
else:
|
|
self.current_selected_rows[Path(message['data']['node'])] = message['data']
|
|
else:
|
|
if message['isSelected'] and message:
|
|
self.current_selected_rows[Path(message['data']['node'])] = message['data']
|
|
else:
|
|
self.current_selected_rows.pop(message['data']['node'])
|
|
|
|
self.socketio.start_background_task(self.send_graph_data)
|
|
self.socketio.start_background_task(self.send_node_infos)
|
|
|
|
def analyze_counts(self):
|
|
"""Perform count analysis and send the results back to frontend."""
|
|
|
|
with self.app.test_request_context():
|
|
|
|
analysis = libdeps.analyzer.counter_factory(
|
|
self._dependents_graph,
|
|
[name[0] for name in libdeps.analyzer.CountTypes.__members__.items()])
|
|
ga = libdeps.analyzer.LibdepsGraphAnalysis(analysis)
|
|
results = ga.get_results()
|
|
|
|
graph_data = []
|
|
for i, data in enumerate(results):
|
|
graph_data.append({'id': i, 'type': data, 'value': results[data]})
|
|
self.socketio.emit("graph_results", graph_data)
|
|
|
|
def send_node_list(self):
|
|
"""Gather all the nodes in the graph for the node list."""
|
|
|
|
with self.app.test_request_context():
|
|
node_data = {
|
|
'graphData': {'nodes': [], 'links': []},
|
|
"selectedNodes": [str(node) for node in list(self.current_selected_rows.keys())]
|
|
}
|
|
|
|
for node in self._dependents_graph.nodes():
|
|
node_path = Path(node)
|
|
node_data['graphData']['nodes'].append(
|
|
{'id': str(node_path), 'name': node_path.name})
|
|
self.socketio.emit("graph_nodes", node_data)
|
|
|
|
def load_graph(self, message):
|
|
"""Load the graph into application memory and kick off threads for analysis on new graph."""
|
|
|
|
with self.app.test_request_context():
|
|
|
|
current_hash = self._dependents_graph.graph.get('git_hash', 'NO_HASH')[:7]
|
|
if current_hash != message['hash']:
|
|
self.current_selected_rows = {}
|
|
if message['hash'] in self.loaded_graphs:
|
|
self._dependents_graph = self.loaded_graphs[message['hash']]
|
|
self._dependents_graph = networkx.reverse_view(self._dependency_graph)
|
|
else:
|
|
print(
|
|
f'loading new graph {current_hash} because different than {message["hash"]}'
|
|
)
|
|
|
|
self.load_graph_from_file(self.graph_files[message['hash']].graph_file)
|
|
self.loaded_graphs[message['hash']] = self._dependents_graph
|
|
|
|
self.socketio.start_background_task(self.analyze_counts)
|
|
self.socketio.start_background_task(self.send_node_list)
|
|
self.socketio.emit("graph_data", {'graphData': {'nodes': [], 'links': []}})
|
|
|
|
def git_hash_selected(self, message):
|
|
"""Load the new graph and perform queries on it."""
|
|
|
|
print(f"Got requests2 {message}!")
|
|
|
|
emit("other_hash_selected", message, broadcast=True)
|
|
|
|
self.socketio.start_background_task(self.load_graph, message)
|