313 lines
13 KiB
Python
313 lines
13 KiB
Python
#!/usr/bin/env python3
|
|
#
|
|
# Copyright 2020 MongoDB Inc.
|
|
#
|
|
# Permission is hereby granted, free of charge, to any person obtaining
|
|
# a copy of this software and associated documentation files (the
|
|
# "Software"), to deal in the Software without restriction, including
|
|
# without limitation the rights to use, copy, modify, merge, publish,
|
|
# distribute, sublicense, and/or sell copies of the Software, and to
|
|
# permit persons to whom the Software is furnished to do so, subject to
|
|
# the following conditions:
|
|
#
|
|
# The above copyright notice and this permission notice shall be included
|
|
# in all copies or substantial portions of the Software.
|
|
#
|
|
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
|
|
# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
|
|
# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
|
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
|
# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
|
# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
|
# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
|
#
|
|
"""
|
|
Flask backend web server.
|
|
|
|
The backend interacts with the graph_analyzer to perform queries on various libdeps graphs.
|
|
"""
|
|
|
|
from pathlib import Path
|
|
from collections import namedtuple, OrderedDict
|
|
|
|
import flask
|
|
import networkx
|
|
|
|
from flask_cors import CORS
|
|
from flask_session import Session
|
|
from lxml import etree
|
|
from flask import request
|
|
|
|
import libdeps.graph
|
|
import libdeps.analyzer
|
|
|
|
|
|
class BackendServer:
|
|
"""Create small class for storing variables and state of the backend."""
|
|
|
|
# pylint: disable=too-many-instance-attributes
|
|
def __init__(self, graphml_dir, frontend_url):
|
|
"""Create and setup the state variables."""
|
|
self.app = flask.Flask(__name__)
|
|
self.app.config['CORS_HEADERS'] = 'Content-Type'
|
|
CORS(self.app, resources={r"/*": {"origins": frontend_url}})
|
|
|
|
self.app.add_url_rule("/api/graphs", "return_graph_files", self.return_graph_files)
|
|
self.app.add_url_rule("/api/graphs/<git_hash>/nodes", "return_node_list",
|
|
self.return_node_list)
|
|
self.app.add_url_rule("/api/graphs/<git_hash>/analysis", "return_analyze_counts",
|
|
self.return_analyze_counts)
|
|
self.app.add_url_rule("/api/graphs/<git_hash>/d3", "return_d3", self.return_d3,
|
|
methods=['POST'])
|
|
self.app.add_url_rule("/api/graphs/<git_hash>/nodes/details", "return_node_infos",
|
|
self.return_node_infos, methods=['POST'])
|
|
self.app.add_url_rule("/api/graphs/<git_hash>/paths", "return_paths_between",
|
|
self.return_paths_between, methods=['POST'])
|
|
|
|
self.loaded_graphs = {}
|
|
self.graphml_dir = Path(graphml_dir)
|
|
self.frontend_url = frontend_url
|
|
|
|
self.graph_file_tuple = namedtuple('GraphFile', ['version', 'git_hash', 'graph_file'])
|
|
self.graph_files = self.get_graphml_files()
|
|
|
|
@staticmethod
|
|
def get_dependency_graph(graph):
|
|
"""Returns the dependency graph of a given graph."""
|
|
|
|
if graph.graph['graph_schema_version'] == 1:
|
|
return networkx.reverse_view(graph)
|
|
else:
|
|
return graph
|
|
|
|
@staticmethod
|
|
def get_dependents_graph(graph):
|
|
"""Returns the dependents graph of a given graph."""
|
|
|
|
if graph.graph['graph_schema_version'] == 1:
|
|
return graph
|
|
else:
|
|
return networkx.reverse_view(graph)
|
|
|
|
def get_app(self):
|
|
"""Return the app instance."""
|
|
|
|
return self.app
|
|
|
|
def get_graph_build_data(self, graph_file):
|
|
"""Fast method for extracting basic build data from the graph file."""
|
|
|
|
version = ''
|
|
git_hash = ''
|
|
# pylint: disable=c-extension-no-member
|
|
for _, element in etree.iterparse(
|
|
str(graph_file), tag="{http://graphml.graphdrawing.org/xmlns}data"):
|
|
if element.get('key') == 'graph_schema_version':
|
|
version = element.text
|
|
if element.get('key') == 'git_hash':
|
|
git_hash = element.text
|
|
element.clear()
|
|
if version and git_hash:
|
|
break
|
|
return self.graph_file_tuple(version, git_hash, graph_file)
|
|
|
|
def get_graphml_files(self):
|
|
"""Find all graphml files in the target graphml dir."""
|
|
|
|
graph_files = OrderedDict()
|
|
for graph_file in self.graphml_dir.glob("**/*.graphml"):
|
|
graph_file_tuple = self.get_graph_build_data(graph_file)
|
|
graph_files[graph_file_tuple.git_hash[:7]] = graph_file_tuple
|
|
return graph_files
|
|
|
|
def return_graph_files(self):
|
|
"""Prepare the list of graph files for the frontend."""
|
|
|
|
data = {'graph_files': []}
|
|
for i, (_, graph_file_data) in enumerate(self.graph_files.items(), start=1):
|
|
data['graph_files'].append({
|
|
'id': i, 'version': graph_file_data.version, 'git': graph_file_data.git_hash[:7],
|
|
'selected': False
|
|
})
|
|
return data
|
|
|
|
def return_node_infos(self, git_hash):
|
|
"""Returns details about a set of selected nodes."""
|
|
|
|
req_body = request.get_json()
|
|
if "selected_nodes" in req_body.keys():
|
|
selected_nodes = req_body["selected_nodes"]
|
|
|
|
if graph := self.load_graph(git_hash):
|
|
dependents_graph = self.get_dependents_graph(graph)
|
|
dependency_graph = self.get_dependency_graph(graph)
|
|
|
|
nodeinfo_data = {'nodeInfos': []}
|
|
|
|
for node in selected_nodes:
|
|
|
|
nodeinfo_data['nodeInfos'].append({
|
|
'id':
|
|
len(nodeinfo_data['nodeInfos']),
|
|
'node':
|
|
str(node),
|
|
'name':
|
|
Path(node).name,
|
|
'attribs': [{
|
|
'name': key, 'value': value
|
|
} for key, value in dependents_graph.nodes(data=True)[str(node)].items()],
|
|
'dependers': [{
|
|
'node':
|
|
depender, 'symbols':
|
|
dependents_graph[str(node)][depender].get('symbols',
|
|
'').split(' ')
|
|
} for depender in dependents_graph[str(node)]],
|
|
'dependencies': [{
|
|
'node':
|
|
dependency, 'symbols':
|
|
dependents_graph[dependency][str(node)].get('symbols',
|
|
'').split(' ')
|
|
} for dependency in dependency_graph[str(node)]],
|
|
})
|
|
|
|
return nodeinfo_data, 200
|
|
return {
|
|
'error': 'Git commit hash (' + git_hash + ') does not have a matching graph file.'
|
|
}, 400
|
|
return {'error': 'Request body does not contain "selected_nodes" attribute.'}, 400
|
|
|
|
def return_d3(self, git_hash):
|
|
"""Convert the current selected rows into a format for D3."""
|
|
|
|
req_body = request.get_json()
|
|
if "selected_nodes" in req_body.keys():
|
|
selected_nodes = req_body["selected_nodes"]
|
|
|
|
if graph := self.load_graph(git_hash):
|
|
dependents_graph = self.get_dependents_graph(graph)
|
|
dependency_graph = self.get_dependency_graph(graph)
|
|
|
|
nodes = {}
|
|
links = {}
|
|
|
|
def add_node_to_graph_data(node):
|
|
nodes[str(node)] = {
|
|
'id': str(node), 'name': Path(node).name, 'type': dependents_graph.nodes()
|
|
[str(node)]['bin_type']
|
|
}
|
|
|
|
def add_link_to_graph_data(source, target):
|
|
links[str(source) + str(target)] = {
|
|
'source': str(source), 'target': str(target)
|
|
}
|
|
|
|
for node in selected_nodes:
|
|
add_node_to_graph_data(node)
|
|
|
|
for libdep in dependency_graph[str(node)]:
|
|
if dependents_graph[libdep][str(node)].get('direct'):
|
|
add_node_to_graph_data(libdep)
|
|
add_link_to_graph_data(node, libdep)
|
|
|
|
if "extra_nodes" in req_body.keys():
|
|
extra_nodes = req_body["extra_nodes"]
|
|
for node in extra_nodes:
|
|
add_node_to_graph_data(node)
|
|
|
|
for libdep in dependency_graph.get_direct_nonprivate_graph()[str(node)]:
|
|
add_node_to_graph_data(libdep)
|
|
add_link_to_graph_data(node, libdep)
|
|
|
|
node_data = {
|
|
'graphData': {
|
|
'nodes': [data for node, data in nodes.items()],
|
|
'links': [data for link, data in links.items()],
|
|
}
|
|
}
|
|
return node_data, 200
|
|
return {
|
|
'error': 'Git commit hash (' + git_hash + ') does not have a matching graph file.'
|
|
}, 400
|
|
return {'error': 'Request body does not contain "selected_nodes" attribute.'}, 400
|
|
|
|
def return_analyze_counts(self, git_hash):
|
|
"""Perform count analysis and send the results back to frontend."""
|
|
|
|
with self.app.test_request_context():
|
|
if graph := self.load_graph(git_hash):
|
|
dependency_graph = self.get_dependency_graph(graph)
|
|
|
|
analysis = libdeps.analyzer.counter_factory(
|
|
dependency_graph,
|
|
[name[0] for name in libdeps.analyzer.CountTypes.__members__.items()])
|
|
ga = libdeps.analyzer.LibdepsGraphAnalysis(analysis)
|
|
results = ga.get_results()
|
|
|
|
graph_data = []
|
|
for i, data in enumerate(results):
|
|
graph_data.append({'id': i, 'type': data, 'value': results[data]})
|
|
return {'results': graph_data}, 200
|
|
return {
|
|
'error': 'Git commit hash (' + git_hash + ') does not have a matching graph file.'
|
|
}, 400
|
|
|
|
def return_paths_between(self, git_hash):
|
|
"""Gather all the paths in the graph between a fromNode and toNode."""
|
|
|
|
message = request.get_json()
|
|
if "fromNode" in message.keys() and "toNode" in message.keys():
|
|
if graph := self.load_graph(git_hash):
|
|
dependency_graph = self.get_dependency_graph(graph)
|
|
analysis = [
|
|
libdeps.analyzer.GraphPaths(dependency_graph, message['fromNode'],
|
|
message['toNode'])
|
|
]
|
|
ga = libdeps.analyzer.LibdepsGraphAnalysis(analysis=analysis)
|
|
results = ga.get_results()
|
|
|
|
paths = results[libdeps.analyzer.DependsReportTypes.GRAPH_PATHS.name][(
|
|
message['fromNode'], message['toNode'])]
|
|
paths.sort(key=len)
|
|
nodes = set()
|
|
for path in paths:
|
|
for node in path:
|
|
nodes.add(node)
|
|
|
|
# Need to handle self.send_graph_data(extra_nodes=list(nodes))
|
|
return {
|
|
'fromNode': message['fromNode'], 'toNode': message['toNode'], 'paths': paths,
|
|
'extraNodes': list(nodes)
|
|
}, 200
|
|
return {
|
|
'error': 'Git commit hash (' + git_hash + ') does not have a matching graph file.'
|
|
}, 400
|
|
return {'error': 'Body must contain toNode and fromNode'}, 400
|
|
|
|
def return_node_list(self, git_hash):
|
|
"""Gather all the nodes in the graph for the node list."""
|
|
|
|
with self.app.test_request_context():
|
|
node_data = {'nodes': [], 'links': []}
|
|
if graph := self.load_graph(git_hash):
|
|
for node in sorted(graph.nodes()):
|
|
node_path = Path(node)
|
|
node_data['nodes'].append(str(node_path))
|
|
return node_data, 200
|
|
return {
|
|
'error': 'Git commit hash (' + git_hash + ') does not have a matching graph file.'
|
|
}, 400
|
|
|
|
def load_graph(self, git_hash):
|
|
"""Load the graph into application memory."""
|
|
|
|
with self.app.test_request_context():
|
|
if git_hash in self.loaded_graphs:
|
|
return self.loaded_graphs[git_hash]
|
|
else:
|
|
if git_hash in self.graph_files:
|
|
file_path = self.graph_files[git_hash].graph_file
|
|
graph = libdeps.graph.LibdepsGraph(networkx.read_graphml(file_path))
|
|
self.loaded_graphs[git_hash] = graph
|
|
return graph
|
|
return None
|