#!/usr/bin/env python3 # # Copyright 2020 MongoDB Inc. # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the # "Software"), to deal in the Software without restriction, including # without limitation the rights to use, copy, modify, merge, publish, # distribute, sublicense, and/or sell copies of the Software, and to # permit persons to whom the Software is furnished to do so, subject to # the following conditions: # # The above copyright notice and this permission notice shall be included # in all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY # KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE # WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. # """ Libdeps Graph Analysis Tool. This will perform various metric's gathering and linting on the graph generated from SCons generate-libdeps-graph target. The graph represents the dependency information between all binaries from the build. """ from enum import Enum, auto from pathlib import Path import networkx class CountTypes(Enum): """Enums for the different types of counts to perform on a graph.""" all = auto() node = auto() edge = auto() dir_edge = auto() trans_edge = auto() dir_pub_edge = auto() pub_edge = auto() priv_edge = auto() if_edge = auto() class DependsReportTypes(Enum): """Enums for the different type of depends reports to perform on a graph.""" direct_depends = auto() common_depends = auto() exclude_depends = auto() class EdgeProps(Enum): """Enums for edge properties.""" direct = auto() visibility = auto() class LibdepsGraph(networkx.DiGraph): """Class for analyzing the graph.""" def __init__(self, graph=networkx.DiGraph()): """Load the graph data.""" super().__init__(incoming_graph_data=graph) # Load in the graph and store a reversed version as well for quick look ups # the in directions. self.rgraph = graph.reverse() def number_of_edge_types(self, edge_type, value): """Count the graphs edges based on type.""" return len([edge for edge in self.edges(data=True) if edge[2].get(edge_type) == value]) def node_count(self): """Count the graphs nodes.""" return self.number_of_nodes() def edge_count(self): """Count the graphs edges.""" return self.number_of_edges() def direct_edge_count(self): """Count the graphs direct edges.""" return self.number_of_edge_types(EdgeProps.direct.name, 1) def transitive_edge_count(self): """Count the graphs transitive edges.""" return self.number_of_edge_types(EdgeProps.direct.name, 0) def direct_public_edge_count(self): """Count the graphs direct public edges.""" return len([ edge for edge in self.edges(data=True) if edge[2].get(EdgeProps.direct.name) == 1 and edge[2].get(EdgeProps.visibility.name) == 0 ]) def public_edge_count(self): """Count the graphs public edges.""" return self.number_of_edge_types(EdgeProps.visibility.name, 0) def private_edge_count(self): """Count the graphs private edges.""" return self.number_of_edge_types(EdgeProps.visibility.name, 1) def interface_edge_count(self): """Count the graphs interface edges.""" return self.number_of_edge_types(EdgeProps.visibility.name, 2) def direct_depends(self, node): """For given nodes, report what nodes depend directly on that node.""" return [ depender for depender in self[node] if self[node][depender].get(EdgeProps.direct.name) == 1 ] def common_depends(self, nodes): """For a given set of nodes, report what nodes depend on all nodes from that set.""" neighbor_sets = [set(self[node]) for node in nodes] return list(set.intersection(*neighbor_sets)) def exclude_depends(self, nodes): """Find depends with exclusions. Given a node, and a set of other nodes, find what nodes depend on the given node, but do not depend on the set of nodes. """ valid_depender_nodes = [] for depender_node in set(self[nodes[0]]): if all([ bool(excludes_node not in set(self.rgraph[depender_node])) for excludes_node in nodes[1:] ]): valid_depender_nodes.append(depender_node) return valid_depender_nodes class LibdepsGraphAnalysis: """Runs the given analysis on the input graph.""" def __init__(self, libdeps_graph, build_dir='build/opt', counts='all', depends_reports=None): """Perform analysis based off input args.""" self.build_dir = Path(build_dir) self.libdeps_graph = libdeps_graph self.results = {} self.count_types = { CountTypes.node.name: ("num_nodes", libdeps_graph.node_count), CountTypes.edge.name: ("num_edges", libdeps_graph.edge_count), CountTypes.dir_edge.name: ("num_direct_edges", libdeps_graph.direct_edge_count), CountTypes.trans_edge.name: ("num_trans_edges", libdeps_graph.transitive_edge_count), CountTypes.dir_pub_edge.name: ("num_direct_public_edges", libdeps_graph.direct_public_edge_count), CountTypes.pub_edge.name: ("num_public_edges", libdeps_graph.public_edge_count), CountTypes.priv_edge.name: ("num_private_edges", libdeps_graph.private_edge_count), CountTypes.if_edge.name: ("num_interface_edges", libdeps_graph.interface_edge_count), } for name in DependsReportTypes.__members__.items(): setattr(self, f'{name[0]}_key', name[0]) if counts: self.run_graph_counters(counts) if depends_reports: self.run_depend_reports(depends_reports) def get_results(self): """Return the results fo the analysis.""" return self.results def _strip_build_dir(self, node): """Small util function for making args match the graph paths.""" node = Path(node) if str(node.absolute()).startswith(str(self.build_dir.absolute())): return str(node.relative_to(self.build_dir)) else: raise Exception( f"build path not in node path: node: {node} build_dir: {self.build_dir}") def _strip_build_dirs(self, nodes): """Small util function for making a list of nodes match graph paths.""" for node in nodes: yield self._strip_build_dir(node) def run_graph_counters(self, counts): """Run the various graph counters for nodes and edges.""" for count_type in CountTypes.__members__.items(): if count_type[0] in self.count_types: dict_name, func = self.count_types[count_type[0]] if count_type[0] in counts: self.results[dict_name] = func() def run_depend_reports(self, depends_reports): """Run the various dependency reports.""" if depends_reports.get(self.direct_depends_key): self.results[self.direct_depends_key] = {} for node in depends_reports[self.direct_depends_key]: self.results[self.direct_depends_key][node] = self.libdeps_graph.direct_depends( self._strip_build_dir(node)) if depends_reports.get(self.common_depends_key): self.results[self.common_depends_key] = {} for nodes in depends_reports[self.common_depends_key]: nodes = frozenset(self._strip_build_dirs(nodes)) self.results[self.common_depends_key][nodes] = self.libdeps_graph.common_depends( nodes) if depends_reports.get(self.exclude_depends_key): self.results[self.exclude_depends_key] = {} for nodes in depends_reports[self.exclude_depends_key]: nodes = tuple(self._strip_build_dirs(nodes)) self.results[self.exclude_depends_key][nodes] = self.libdeps_graph.exclude_depends( nodes) class GaPrinter: """Base class for printers of the graph analysis.""" def __init__(self, libdeps_graph_analysis): """Store the graph analysis for use when printing.""" self.libdeps_graph_analysis = libdeps_graph_analysis class GaJsonPrinter(GaPrinter): """Printer for json output.""" def serialize(self, dictionary): """Serialize the k,v pairs in the dictionary.""" new = {} for key, value in dictionary.items(): if isinstance(value, dict): value = self.serialize(value) new[str(key)] = value return new def print(self): """Print the result data.""" import json results = self.libdeps_graph_analysis.get_results() print(json.dumps(self.serialize(results))) class GaPrettyPrinter(GaPrinter): """Printer for pretty console output.""" count_desc = { CountTypes.node.name: ("num_nodes", "Nodes in Graph: {}"), CountTypes.edge.name: ("num_edges", "Edges in Graph: {}"), CountTypes.dir_edge.name: ("num_direct_edges", "Direct Edges in Graph: {}"), CountTypes.trans_edge.name: ("num_trans_edges", "Transitive Edges in Graph: {}"), CountTypes.dir_pub_edge.name: ("num_direct_public_edges", "Direct Public Edges in Graph: {}"), CountTypes.pub_edge.name: ("num_public_edges", "Public Edges in Graph: {}"), CountTypes.priv_edge.name: ("num_private_edges", "Private Edges in Graph: {}"), CountTypes.if_edge.name: ("num_interface_edges", "Interface Edges in Graph: {}"), } @staticmethod def _print_results_node_list(heading, nodes): """Util function for printing a list of nodes for depend reports.""" print(heading) for i, depender in enumerate(nodes, start=1): print(f"\t{i}: {depender}") print("") def print(self): """Print the result data.""" results = self.libdeps_graph_analysis.get_results() for count_type in CountTypes.__members__.items(): if count_type[0] in self.count_desc: dict_name, desc = self.count_desc[count_type[0]] if dict_name in results: print(desc.format(results[dict_name])) if DependsReportTypes.direct_depends.name in results: print("\nNodes that directly depend on:") for node in results[DependsReportTypes.direct_depends.name]: self._print_results_node_list(f'=>depends on {node}:', results[DependsReportTypes.direct_depends.name][node]) if DependsReportTypes.common_depends.name in results: print("\nNodes that commonly depend on:") for nodes in results[DependsReportTypes.common_depends.name]: self._print_results_node_list( f'=>depends on {nodes}:', results[DependsReportTypes.common_depends.name][nodes]) if DependsReportTypes.exclude_depends.name in results: print("\nNodes that depend on a node, but exclude others:") for nodes in results[DependsReportTypes.exclude_depends.name]: self._print_results_node_list( f"=>depends: {nodes[0]}, exclude: {nodes[1:]}:", results[DependsReportTypes.exclude_depends.name][nodes])