diff --git a/.gitignore b/.gitignore index 410de9a..d681a01 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ *.pyc output *~ +.idea/* diff --git a/common.py b/common.py index c26e897..8d4c8f8 100644 --- a/common.py +++ b/common.py @@ -2,53 +2,58 @@ Common and other crappy code used throughout git by a bus. """ + def safe_author_name(author): if author: return author.replace(',', '_').replace(':', '_') else: return author -def safe_int(i): + +def safe_int(i: str | None) -> int | None: if i is None or i == '': return None else: return int(i) -def safe_str(s): + +def safe_str(s) -> str: if s is None: return '' else: return str(s) + def parse_dev_shared(s, num_func): dev_shared = [] if not s: return dev_shared - for ddv in s.split(','): - segs = ddv.split(':') - k = segs[:-1] - v = float(segs[-1]) - dev_shared.append((k, v)) + return [(ddv.split(':')[:-1], float(ddv.split(':')[-1])) for ddv in s.split(',')] - return dev_shared def dev_shared_to_str(dev_shared): return ','.join([':'.join([':'.join(devs), str(shared)]) for devs, shared in dev_shared]) -def parse_dev_exp_str(s, num_func): + +def parse_dev_exp_str(s: str, num_func) -> list[tuple[str, str, str]]: if not s: return [] - return [(dd[0], num_func(dd[1]), num_func(dd[2])) for dd in [d.split(':') for d in s.split(',')]] + return [ + (dd[0], num_func(dd[1]), num_func(dd[2])) + for dd in [d.split(':') for d in s.split(',')] + ] + def dev_exp_to_str(devs): return ','.join([':'.join([str(x) for x in d]) for d in devs]) + def project_name(fname): if not fname: return None return fname.split(':')[0] - + class FileData(object): """ Represents a single line of data about a single file, can encode / parse to / from tsv. @@ -74,7 +79,7 @@ def __init__(self, line): if line is None: line = '' line = line.strip('\n\r') - fields = line.split('\t') + fields: list[str | None] = line.split('\t') n_missing_fields = FileData.num_fields - len(fields) fields.extend(n_missing_fields * [None]) @@ -90,23 +95,27 @@ def __init__(self, line): self.project = project_name(self.fname) def as_line(self): - return '\t'.join(map(safe_str, [self.fname, - self.cnt_lines, - dev_exp_to_str(self.dev_experience), - self.tot_knowledge, - dev_shared_to_str(self.dev_uniq), - dev_shared_to_str(self.dev_risk)])) + return '\t'.join( + [ + safe_str(s) + for s in [ + self.fname, self.cnt_lines, dev_exp_to_str(self.dev_experience), + self.tot_knowledge, dev_shared_to_str(self.dev_uniq), + dev_shared_to_str(self.dev_risk) + ] + ] + ) def __str__(self): - s = ("fname: %s, cnt_lines: %s, dev_experience: %s, tot_knowledge: %s, dev_uniq: %s, " + \ - "risk: %s ") % (self.fname, - str(self.cnt_lines), - dev_exp_to_str(self.dev_experience), - str(self.tot_knowledge), - dev_shared_to_str(self.dev_uniq), - dev_shared_to_str(self.dev_risk)) + s = ( + f"fname: {self.fname}, cnt_lines: {self.cnt_lines}, " + f"dev_experience: {dev_exp_to_str(self.dev_experience)}, " + f"tot_knowledge: {self.tot_knowledge}, dev_uniq: {dev_shared_to_str(self.dev_uniq)}, " + f"risk: {dev_shared_to_str(self.dev_risk)}" + ) return s - + + def is_interesting(f, interesting, not_interesting): if f.strip() == '': return False @@ -115,6 +124,7 @@ def is_interesting(f, interesting, not_interesting): has_my_interest = not any([n.search(f) for n in not_interesting]) return has_my_interest + def parse_departed_devs(dd_file, departed_devs): fil = open(dd_file, 'r') for line in fil: @@ -124,4 +134,3 @@ def parse_departed_devs(dd_file, departed_devs): continue departed_devs.append(line) fil.close() - diff --git a/estimate_file_risk.py b/estimate_file_risk.py index 6b35387..ae45181 100644 --- a/estimate_file_risk.py +++ b/estimate_file_risk.py @@ -10,16 +10,19 @@ """ import sys -import optparse + +from argparse import ArgumentParser from common import FileData, safe_author_name + def get_bus_risk(dev, bus_risks, def_risk): if dev not in bus_risks: return def_risk else: return bus_risks[dev] + def estimate_file_risks(lines, bus_risks, def_bus_risk): """ Estimate the risk in the file as: @@ -41,28 +44,33 @@ def estimate_file_risks(lines, bus_risks, def_bus_risk): fd.dev_risk = dev_risk yield fd.as_line() + def parse_risk_file(risk_file, bus_risks): - risk_f = open(risk_file, 'r') - for line in risk_f: - line = line.strip() - if not line: - continue - dev, risk = line.split('=') - dev = safe_author_name(dev) - bus_risks[dev] = float(risk) - risk_f.close() + with open(risk_file, 'r') as risk_f: + for line in risk_f: + line = line.strip() + if not line: + continue + dev, risk = line.split('=') + dev = safe_author_name(dev) + bus_risks[dev] = float(risk) + if __name__ == '__main__': - parser = optparse.OptionParser() - parser.add_option('-b', '--bus-risk', dest='bus_risk', metavar='FLOAT', default=0.1, - help='The estimated probability that a dev will be hit by a bus in your analysis timeframe') - parser.add_option('-r', '--risk-file', dest='risk_file', metavar='FILE', - help='File of dev=float lines (e.g. ejorgensen=0.4) with dev bus likelihoods') - options, args = parser.parse_args() + parser = ArgumentParser() + parser.add_argument( + "-b", "--bus-risk", dest="bus_risk", metavar="FLOAT", default=0.1, + help="The estimated probability that a dev will be hit by a bus in your analysis timeframe" + ) + parser.add_argument( + "-r", "--risk-file", dest="risk_file", metavar="FILE", + help="File of dev=float lines (e.g. ejorgensen=0.4) with dev bus likelihoods" + ) + args = parser.parse_args() - bus_risks = {} - if options.risk_file: - parse_risk_file(options.risk_file, bus_risks) + bus_risks_options = {} + if args.risk_file: + parse_risk_file(args.risk_file, bus_risks_options) - for line in estimate_file_risks(sys.stdin, bus_risks, float(options.bus_risk)): - print line + for line in estimate_file_risks(sys.stdin, bus_risks_options, float(args.bus_risk)): + print(line) diff --git a/estimate_unique_knowledge.py b/estimate_unique_knowledge.py index b827058..e12bc2e 100644 --- a/estimate_unique_knowledge.py +++ b/estimate_unique_knowledge.py @@ -24,13 +24,12 @@ """ import sys -import math -import copy -from optparse import OptionParser +from argparse import ArgumentParser from common import FileData + def sequential_create_knowledge(dev_uniq, dev, adjustment): """ Create adjustment lines of knowledge in dev's account. @@ -42,6 +41,7 @@ def sequential_create_knowledge(dev_uniq, dev, adjustment): dev_uniq[dev] = 0 dev_uniq[dev] += adjustment + def sequential_destroy_knowledge(adjustment, tot_knowledge, dev_uniq): """ Find the percentage of tot_knowledge the adjustment represents and @@ -57,6 +57,7 @@ def sequential_destroy_knowledge(adjustment, tot_knowledge, dev_uniq): k -= k * pct_to_destroy dev_uniq[devs] = k + def sequential_share_knowledge_group(dev, shared_key_exploded, pct_to_share, dev_uniq): """ Share pct_to_share knowledge from all accounts that dev doesn't @@ -74,6 +75,7 @@ def sequential_share_knowledge_group(dev, shared_key_exploded, pct_to_share, dev dev_uniq[new_shared_key] = 0 dev_uniq[new_shared_key] += amt_to_share + def sequential_distribute_shared_knowledge(dev, shared_knowledge, tot_knowledge, dev_uniq): """ Share the percent of knowledge represented by shared_knowledge of @@ -83,11 +85,13 @@ def sequential_distribute_shared_knowledge(dev, shared_knowledge, tot_knowledge, pct_to_share = 0 if tot_knowledge: pct_to_share = float(shared_knowledge) / float(tot_knowledge) - for shared_key in dev_uniq.keys(): + # list is needed, since dictionary is being modified during the loop + for shared_key in list(dev_uniq.keys()): shared_key_exploded = shared_key.split('\0') if dev not in shared_key_exploded: sequential_share_knowledge_group(dev, shared_key_exploded, pct_to_share, dev_uniq) + def sequential_estimate_uniq(fd, knowledge_churn_constant): """ Estimate the amounts of unique knowledge for each developer who @@ -119,7 +123,8 @@ def sequential_estimate_uniq(fd, knowledge_churn_constant): dev_uniq = [(shared_key.split('\0'), shared) for shared_key, shared in dev_uniq.items()] return dev_uniq, int(tot_knowledge) - + + def sequential(lines, model_args): """ Entry point for the sequential algorithm. @@ -130,22 +135,25 @@ def sequential(lines, model_args): tot_knowledge fields filled in. """ knowledge_churn_constant = float(model_args[0]) - for line in lines: - fd = FileData(line) + for ln in lines: + fd = FileData(ln) dev_uniq, tot_knowledge = sequential_estimate_uniq(fd, knowledge_churn_constant) fd.dev_uniq = dev_uniq fd.tot_knowledge = tot_knowledge yield fd.as_line() -if __name__ == '__main__': - parser = OptionParser() - parser.add_option('--model', dest='model', metavar='MODEL[:MARG1[:MARG2]...]', default="sequential:0.1", - help='Knowledge model to use, with arguments.') - options, args = parser.parse_args() - model = options.model.split(':') +if __name__ == '__main__': + parser = ArgumentParser() + parser.add_argument( + "--model", dest="model", metavar="MODEL[:MARG1[:MARG2]...]", + default="sequential:0.1", help='Knowledge model to use, with arguments.' + ) + args = parser.parse_args() + + model = args.model.split(':') model_func = locals()[model[0]] model_args = model[1:] for line in model_func(sys.stdin, model_args): - print line + print(line) diff --git a/gen_file_stats.py b/gen_file_stats.py index 133799f..d1baf7b 100644 --- a/gen_file_stats.py +++ b/gen_file_stats.py @@ -10,39 +10,51 @@ import os import re -from optparse import OptionParser +from argparse import ArgumentParser import git_file_stats + if __name__ == '__main__': usage = "usage: %prog [options] git_controlled_path[=project_name]" - parser = OptionParser() - parser.add_option('-i', '--interesting', metavar="REGEXP", dest='interesting', action='append', - help='Regular expression to determine which files should be included in calculations. ' + \ - 'May be repeated, any match is sufficient to indicate interest. ' + \ - 'Defaults are \.java$ \.cs$ \.py$ \.c$ \.cpp$ \.h$ \.hpp$ \.pl$ \.rb$ \.sh$', - default=[]) - parser.add_option('-n', '--not-interesting', metavar="REGEXP", dest="not_interesting", action='append', - help="Regular expression to override interesting files. May be repeated, any match is enough to squelch interest.") - parser.add_option('--case-sensitive', dest='case_sensitive', action='store_true', default=False, - help='Use case-sensitive regepxs when finding interesting / uninteresting files (defaults to case-insensitive)') - parser.add_option('--git-exe', dest='git_exe', default='/usr/bin/env git', - help='Path to the git exe (defaults to "/usr/bin/env git")') - parser.add_option('--svn', dest='use_svn', default=False, action='store_true', - help='Use svn intead of git to generate file statistics. This requires you to install pysvn.') - - options, args = parser.parse_args() + parser = ArgumentParser() + parser.add_argument( + '-i', '--interesting', metavar="REGEXP", dest='interesting', action='append', + help='Regular expression to determine which files should be included in calculations. ' + 'May be repeated, any match is sufficient to indicate interest. ' + 'Defaults are \\.java$ \\.cs$ \\.py$ \\.c$ \\.cpp$ \\.h$ \\.hpp$ \\.pl$ \\.rb$ \\.sh$', + default=[] + ) + parser.add_argument( + '-n', '--not-interesting', metavar="REGEXP", dest="not_interesting", + action='append', + help="Regular expression to override interesting files. " + "May be repeated, any match is enough to squelch interest." + ) + parser.add_argument( + '--case-sensitive', dest='case_sensitive', action='store_true', default=False, + help="Use case-sensitive regepxs when finding interesting / uninteresting files " + "(defaults to case-insensitive)" + ) + parser.add_argument( + '--git-exe', dest='git_exe', default='/usr/bin/git', + help='Path to the git exe (defaults to "/usr/bin/git")' + ) + parser.add_argument( + '--svn', dest='use_svn', default=False, action='store_true', + help="Use svn intead of git to generate file statistics. " + "This requires you to install pysvn." + ) + parser.add_argument("git_controlled_path") + args = parser.parse_args() - if len(args) != 1: - parser.error("You must pass a single git controlled path as the argument.") - - path_project = args[0].split('=') + path_project = args.git_controlled_path.split('=') project = None # handle symlinked directories, which git doesn't like. # but don't use them for svn. - if not options.use_svn: + if not args.use_svn: root = os.path.realpath(path_project[0]) else: root = path_project[0] @@ -54,10 +66,13 @@ # the root. project = os.path.split(root)[1] - interesting = options.interesting or r'\.java$ \.cs$ \.py$ \.c$ \.cpp$ \.h$ \.hpp$ \.pl$ \.rb$ \.sh$'.split(' ') - not_interesting = options.not_interesting or [] + interesting = ( + args.interesting or + r'\.java$ \.cs$ \.py$ \.c$ \.cpp$ \.h$ \.hpp$ \.pl$ \.rb$ \.sh$'.split(' ') + ) + not_interesting = args.not_interesting or [] - if options.case_sensitive: + if args.case_sensitive: interesting = [re.compile(i) for i in interesting] not_interesting = [re.compile(n) for n in not_interesting] else: @@ -66,12 +81,12 @@ gen_stats = git_file_stats.gen_stats - if options.use_svn: + if args.use_svn: # only run the import if they actually try to use svn, since # we don't want to import pysvn and fail if we don't have to. import svn_file_stats gen_stats = svn_file_stats.gen_stats - for line in gen_stats(root, project, interesting, not_interesting, options): + for line in gen_stats(root, project, interesting, not_interesting, args): if line.strip(): - print line + print(line) diff --git a/git_by_a_bus.py b/git_by_a_bus.py index d4148ae..05e72e9 100755 --- a/git_by_a_bus.py +++ b/git_by_a_bus.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/python """ Driver for git by a bus. @@ -17,34 +17,39 @@ Run as python git_by_a_bus.py -h for options. """ -import sys import os +import sys +import textwrap -from optparse import OptionParser +from argparse import ArgumentParser from subprocess import Popen from string import Template + SCRIPT_PATH=os.path.dirname(os.path.realpath(__file__)) sys.path.append(SCRIPT_PATH) -def exit_with_error(err): - print >> sys.stderr, "Error: " + err + +def exit_with_error(err: str): + print("Error: ", err, file=sys.stderr) exit(1) + def read_projects_file(fname, paths_projects): try: - fil = open(fname, 'r') - paths_projects.extend([line.strip() for line in fil if line.strip()]) - fil.close() + with open(fname, 'r') as fil: + paths_projects.extend([line.strip() for line in fil if line.strip()]) return True except IOError: return False + def output_fname_for(pyfile, output_dir): if not pyfile: return None return os.path.join(output_dir, os.path.splitext(os.path.basename(pyfile))[0] + '.tsv') + def run_chained(cmd_ts, python_cmd, output_dir, verbose): for cmd_t in cmd_ts: input_pyfile = cmd_t[0] @@ -58,22 +63,20 @@ def run_chained(cmd_ts, python_cmd, output_dir, verbose): output_fname = output_fname_for(output_pyfile, output_dir) # don't re-run if the results exist - if os.path.isfile(output_fname): + if output_fname is not None and os.path.isfile(output_fname): if verbose: - print >> sys.stderr, "%s EXISTS, SKIPPING" % output_fname + print(f"{output_fname} EXISTS, SKIPPING", file=sys.stderr) continue - input_f = None - if input_fname: - input_f = open(input_fname, 'r') - output_f = open(output_fname, 'w') + input_f = open(input_fname, 'r') if input_fname else None + output_f = open(output_fname, 'w') if output_fname else None for opt_args in opts_args: cmd = [x for x in ' '.join([python_cmd, output_pyfile, opt_args]).split(' ') if x] if verbose: - print >> sys.stderr, "Input file is: %s" % input_fname - print >> sys.stderr, "Output file is: %s" % output_fname - print >> sys.stderr, cmd + print("Input file is: %s" % input_fname, file=sys.stderr) + print("Output file is: %s" % output_fname, file=sys.stderr) + print(cmd, file=sys.stderr) cmd_p = Popen(cmd, stdin=input_f, stdout=output_f) cmd_p.communicate() @@ -82,11 +85,12 @@ def run_chained(cmd_ts, python_cmd, output_dir, verbose): if output_f: output_f.close() + def main(python_cmd, paths_projects, options): output_dir = os.path.abspath(options.output or 'output') try: os.mkdir(output_dir) - except: + except OSError: if not options.continue_last: exit_with_error("Output directory exists and you have not specified -c") @@ -117,93 +121,154 @@ def main(python_cmd, paths_projects, options): # commands to chain together--the stdout of the first becomes the # stdin of the next. You can find the output of gen_file_stats.py # in output_dir/gen_file_stats.tsv, and so on. - cmd_ts = [] - cmd_ts.append([None, os.path.join(SCRIPT_PATH,'gen_file_stats.py'), - ['${interesting_file_option} ${not_interesting_file_option} ${case_sensitive_option} ${git_exe_option} ${svn_option} %s' % path_project \ - for path_project in paths_projects]]) - cmd_ts.append([os.path.join(SCRIPT_PATH,'gen_file_stats.py'), - os.path.join(SCRIPT_PATH,'estimate_unique_knowledge.py'), '${model_option}']) - cmd_ts.append([os.path.join(SCRIPT_PATH,'estimate_unique_knowledge.py'), - os.path.join(SCRIPT_PATH,'estimate_file_risk.py'), '-b ${bus_risk} ${risk_file_option}']) - cmd_ts.append([os.path.join(SCRIPT_PATH,'estimate_file_risk.py'), - os.path.join(SCRIPT_PATH,'summarize.py'), '${departed_dev_option} ${output_dir}']) - + cmd_ts = [ + [ + None, + os.path.join(SCRIPT_PATH, 'gen_file_stats.py'), + [ + f"${{interesting_file_option}} ${{not_interesting_file_option}} " + f"${{case_sensitive_option}} ${{git_exe_option}} ${{svn_option}} {path_project}" + for path_project in paths_projects + ], + ], + [ + os.path.join(SCRIPT_PATH, 'gen_file_stats.py'), + os.path.join(SCRIPT_PATH, 'estimate_unique_knowledge.py'), + '${model_option}', + ], + [ + os.path.join(SCRIPT_PATH, 'estimate_unique_knowledge.py'), + os.path.join(SCRIPT_PATH, 'estimate_file_risk.py'), + '-b ${bus_risk} ${risk_file_option}', + ], + [ + os.path.join(SCRIPT_PATH, 'estimate_file_risk.py'), + os.path.join(SCRIPT_PATH, 'summarize.py'), '${departed_dev_option} ${output_dir}', + ], + ] + for cmd_t in cmd_ts: if len(cmd_t) > 2: opts_args = cmd_t[2] if not isinstance(opts_args, list): opts_args = [opts_args] - opts_args = [Template(s).substitute(python_cmd=python_cmd, - risk_file_option=risk_file_option, - bus_risk=options.bus_risk, - departed_dev_option=departed_dev_option, - interesting_file_option=interesting_file_option, - not_interesting_file_option=not_interesting_file_option, - case_sensitive_option=case_sensitive_option, - git_exe_option=git_exe_option, - svn_option=svn_option, - model_option=model_option, - output_dir=output_dir) \ - for s in opts_args] + opts_args = [ + Template(s).substitute( + python_cmd=python_cmd, + risk_file_option=risk_file_option, + bus_risk=options.bus_risk, + departed_dev_option=departed_dev_option, + interesting_file_option=interesting_file_option, + not_interesting_file_option=not_interesting_file_option, + case_sensitive_option=case_sensitive_option, + git_exe_option=git_exe_option, + svn_option=svn_option, + model_option=model_option, + output_dir=output_dir + ) + for s in opts_args + ] cmd_t[2] = opts_args run_chained(cmd_ts, python_cmd, output_dir, options.verbose) - + + if __name__ == '__main__': - usage = """usage: %prog [options] [git_controlled_path1[=project_name1], git_controlled_path2[=project_name2],...] - - Analyze each git controlled path and create an html summary of orphaned / at-risk code knowledge. - - Paths must be absolute paths to local git-controlled directories (they may be subdirs in the git repo). - - Project names are optional and default to the last directory in the path. - - You may alternatively/additionally specify the list of paths/projects in a file with -p. - - Experimental svn support with --svn and an svn url for project path. - """ - usage = '\n'.join([line.strip() for line in usage.split('\n')]) - - parser = OptionParser(usage=usage) - parser.add_option('-b', '--bus-risk', dest='bus_risk', metavar='FLOAT', default=0.1, - help='The default estimated probability that a dev will be hit by a bus in your analysis timeframe (defaults to 0.1)') - parser.add_option('-r', '--risk-file', dest='risk_file', metavar='FILE', - help='File of dev=float lines (e.g. ejorgensen=0.4) with custom bus risks for devs') - parser.add_option('-d', '--departed-dev-file', dest='departed_dev_file', metavar='FILE', - help='File listing departed devs, one per line') - parser.add_option('-i', '--interesting', metavar="REGEXP", dest='interesting', action='append', - help='Regular expression to determine which files should be included in calculations. ' + \ - 'May be repeated, any match is sufficient to indicate interest. ' + \ - 'Defaults are \.java$ \.cs$ \.py$ \.c$ \.cpp$ \.h$ \.hpp$ \.pl$ \.rb$', default=[]) - parser.add_option('-n', '--not-interesting', metavar="REGEXP", dest="not_interesting", action='append', default=[], - help="Regular expression to override interesting files. May be repeated, any match is enough to squelch interest.") - parser.add_option('--case-sensitive', dest='case_sensitive', action='store_true', default=False, - help='Use case-sensitive regepxs when finding interesting / uninteresting files (defaults to case-insensitive)') - parser.add_option('-o', '--output', dest='output', metavar='DIRNAME', default='output', - help='Output directory for data files and html summary (defaults to "output"), error if already exists without -c') - parser.add_option('-p', '--projects-file', dest='projects_file', metavar='FILE', - help='File of path[=project_name] lines, where path is an absoluate path to the git-controlled ' + \ - 'directory or svn url to analyze and project_name is the name to use in the output summary (project_name defaults to ' + \ - 'the last directory name in the path)') - parser.add_option('-c', '--continue-last', dest='continue_last', default=False, action="store_true", - help="Continue last run, using existing output files and recreating missing. You can remove tsv files " + \ - "in the output dir and modify others to clean up bad runs.") - parser.add_option('-v', '--verbose', dest='verbose', default=False, action="store_true", help="Print debugging info") - parser.add_option('--python-exe', dest='python_exe', default='/usr/bin/env python', - help='Path to the python interpreter (defaults to "/usr/bin/env python")') - parser.add_option('--git-exe', dest='git_exe', help='Path to the git exe (defaults to "/usr/bin/env git")') - parser.add_option('--svn', dest='use_svn', default=False, action='store_true', - help='Use svn intead of git to generate file statistics. This requires you to install pysvn in your PYTHONPATH.') - parser.add_option('--model', dest='model', metavar='MODEL[:MARG1[:MARG2]...]', default='sequential:0.1', - help='Knowledge model to use, with arguments. Right now only sequential is supported.') - - options, paths_projects = parser.parse_args() - - if options.projects_file: - if not read_projects_file(options.projects_file, paths_projects): - exit_with_error("Could not read projects file %s" % options.projects_file) - - if not paths_projects: - parser.error('No paths/projects! You must either specify paths/projects on the command line and/or in a file with the -p option.') + descr = textwrap.dedent(""" + Analyze each git controlled path and create an html summary of orphaned / at-risk code knowledge. + + Paths must be absolute paths to local git-controlled directories (they may be subdirs in the git repo). + + Project names are optional and default to the last directory in the path. + + You may alternatively/additionally specify the list of paths/projects in a file with -p. + + Experimental svn support with --svn and an svn url for project path.""") + + parser = ArgumentParser(description=descr) + parser.add_argument( + '-b', '--bus-risk', dest='bus_risk', metavar='FLOAT', default=0.1, + help="The default estimated probability that a dev will be hit by a bus " + "in your analysis timeframe (defaults to 0.1)" + ) + parser.add_argument( + '-r', '--risk-file', dest='risk_file', metavar='FILE', + help='File of dev=float lines (e.g. ejorgensen=0.4) with custom bus risks for devs' + ) + parser.add_argument( + '-d', '--departed-dev-file', dest='departed_dev_file', metavar='FILE', + help="File listing departed devs, one per line" + ) + parser.add_argument( + '-i', '--interesting', metavar="REGEXP", dest='interesting', action='append', + help="Regular expression to determine which files should be included in calculations. " + "May be repeated, any match is sufficient to indicate interest. " + "Defaults are \\.java$ \\.cs$ \\.py$ \\.c$ \\.cpp$ \\.h$ \\.hpp$ \\.pl$ \\.rb$", + default=[] + ) + parser.add_argument( + '-n', '--not-interesting', metavar="REGEXP", dest="not_interesting", + action='append', default=[], + help="Regular expression to override interesting files. " + "May be repeated, any match is enough to squelch interest." + ) + parser.add_argument( + '--case-sensitive', dest='case_sensitive', action='store_true', default=False, + help="Use case-sensitive regepxs when finding interesting / uninteresting files " + "(defaults to case-insensitive)" + ) + parser.add_argument( + '-o', '--output', dest='output', metavar='DIRNAME', default='output', + help="Output directory for data files and html summary (defaults to \"output\"), " + "error if already exists without -c" + ) + parser.add_argument( + '-p', '--projects-file', dest='projects_file', metavar='FILE', + help='File of path[=project_name] lines, where path is an absoluate path ' + 'to the git-controlled directory or svn url to analyze and project_name is ' + 'the name to use in the output summary (project_name defaults ' + 'to the last directory name in the path)' + ) + parser.add_argument( + '-c', '--continue-last', dest='continue_last', + default=False, action="store_true", + help="Continue last run, using existing output files and recreating missing. " + "You can remove tsv files in the output dir and modify others to clean up bad runs." + ) + parser.add_argument( + '-v', '--verbose', dest='verbose', default=False, action="store_true", + help="Print debugging info" + ) + parser.add_argument( + '--python-exe', dest='python_exe', default='/usr/bin/python', + help='Path to the python interpreter (defaults to "/usr/bin/python")' + ) + parser.add_argument( + '--git-exe', dest='git_exe', + help='Path to the git exe (defaults to "/usr/bin/git")' + ) + parser.add_argument( + '--svn', dest='use_svn', default=False, action='store_true', + help='Use svn intead of git to generate file statistics. ' + 'This requires you to install pysvn in your PYTHONPATH.' + ) + parser.add_argument( + '--model', dest='model', metavar='MODEL[:MARG1[:MARG2]...]', + default='sequential:0.1', + help='Knowledge model to use, with arguments. Right now only sequential is supported.' + ) + parser.add_argument("paths", nargs="+") + + args = parser.parse_args() + + if args.projects_file: + if not read_projects_file(args.projects_file, args.paths): + exit_with_error("Could not read projects file %s" % args.projects_file) + + if not args.paths: + parser.error( + "No paths/projects! You must either specify paths/projects on the command line " + "and/or in a file with the -p option." + ) - main(options.python_exe, paths_projects, options) + main(args.python_exe, args.paths, args) diff --git a/git_file_stats.py b/git_file_stats.py index b40bfe0..d55fa0c 100644 --- a/git_file_stats.py +++ b/git_file_stats.py @@ -7,15 +7,14 @@ git_file_stats.gen_stats, but in practice they may differ by a line or two (appears to be whitespace handling, perhaps line endings?) """ - -import sys import os import re - -from subprocess import Popen, PIPE +import subprocess +import sys from common import is_interesting, FileData, safe_author_name - + + def gen_stats(root, project, interesting, not_interesting, options): """ root: the path a local, git controlled-directory that is the root @@ -55,14 +54,14 @@ def gen_stats(root, project, interesting, not_interesting, options): yield fd_line -def count_lines(f): - fil = open(f, 'r') - count = 0 - for line in fil: - count += 1 - fil.close() +def count_lines(f: str) -> int: + with open(f, 'r') as fil: + count = 0 + for _ in fil: + count += 1 return count + def parse_experience(log): """ Parse the dev experience from the git log. @@ -73,82 +72,68 @@ def parse_experience(log): # entry lines were zero separated with -z entry_lines = log.split('\0') - current_entry = [] - for entry_line in entry_lines: if not entry_line.strip(): - # blank entry line marks the end of an entry, we're ready to process - local_entry = current_entry - current_entry = [] - if len(local_entry) < 2: - print >> sys.stderr, "Weird entry, cannot parse: %s\n-----" % '\n'.join(local_entry) - continue - author, changes = local_entry[:2] - author = safe_author_name(author) - try: - changes_split = re.split(r'\s+', changes) - # this can be two fields if there were file renames - # detected, in which case the file names are on the - # following entry lines, or three fields (third being - # the filename) if there were no file renames - lines_added, lines_removed = changes_split[:2] - lines_added = int(lines_added) - lines_removed = int(lines_removed) - - # don't record revisions that don't have any removed or - # added lines...they mean nothing to our algorithm - if lines_added or lines_removed: - exp.append((author, lines_added, lines_removed)) - except ValueError: - print >> sys.stderr, "Weird entry, cannot parse: %s\n-----" % '\n'.join(local_entry) - continue - else: - # continue to aggregate the entry - lines = entry_line.split('\n') - current_entry.extend([line.strip() for line in lines]) + continue + current_entry = entry_line.split('\n') + if len(current_entry) < 2: + continue + author, changes = current_entry[:2] + # only spaces has been changed - ignoring + if not changes.strip(): + continue + author = safe_author_name(author) + m = re.match(r"^(\d+)\s+(\d+)\s+.*$", changes) + if not m: + continue + lines_added, lines_removed = m.groups() + if lines_added or lines_removed: + exp.append((author, lines_added, lines_removed)) # we need the oldest log entries first. exp.reverse() return exp - -def parse_dev_experience(f, git_exe): + + +def parse_dev_experience(f: str, git_exe: str): """ Run git log and parse the dev experience out of it. """ # -z = null byte separate logs # -w = ignore all whitespace when calculating changed lines # --follow = follow file history through renames - # --numstat = print a final ws separated line of the form 'num_added_lines num_deleted_lines file_name' + # --numstat = print a final ws separated line of the form + # 'num_added_lines num_deleted_lines file_name' # --format=format:%an = use only the author name for the log msg format - git_cmd = ("%s log -z -w --follow --numstat --format=format:%%an" % git_exe).split(' ') - git_cmd.append(f) - git_p = Popen(git_cmd, stdout=PIPE) - (out, err) = git_p.communicate() - return parse_experience(out) + git_cmd = [git_exe, "log", "-z", "-w", "--follow", "--numstat", "--format=format:%an", f] + git_p = subprocess.run(git_cmd, capture_output=True, text=True) + return parse_experience(git_p.stdout) -def git_ls(root, git_exe): + +def git_ls(root: str, git_exe: str): """ List the entire tree that git is aware of in this directory. """ # --full-tree = allow absolute path for final argument (pathname) # --name-only = don't show the git id for the object, just the file name # -r = recurse - git_cmd = ('%s ls-tree --full-tree --name-only -r HEAD' % git_exe).split(' ') - git_cmd.append(root) - git_p = Popen(git_cmd, stdout=PIPE) - files = git_p.communicate()[0].split('\n') + git_cmd = [git_exe, "ls-tree", "--full-tree", "--name-only", "-r", "HEAD", root] + git_p = subprocess.run(git_cmd, capture_output=True, text=True) + files = git_p.stdout.split('\n') return files + def git_root(git_exe): """ Given that we have chdir'd into a Git controlled dir, get the git root for purposes of adjusting paths. """ - git_cmd = ('%s rev-parse --show-toplevel' % git_exe).split(' ') - git_p = Popen(git_cmd, stdout=PIPE) - return git_p.communicate()[0].strip() + git_cmd = [git_exe, "rev-parse", "--show-toplevel"] + git_p = subprocess.run(git_cmd, capture_output=True, text=True) + return git_p.stdout.strip() + -def prepare(root, git_exe): +def prepare(root: str, git_exe: str): # first we have to get into the git repo to make the git_root work... os.chdir(root) # then we can change to the git root diff --git a/summarize.py b/summarize.py index ac2104b..5bb78d2 100644 --- a/summarize.py +++ b/summarize.py @@ -10,14 +10,16 @@ import math import hashlib -from optparse import OptionParser +from argparse import ArgumentParser +from collections import defaultdict from common import FileData, parse_departed_devs # we cut off any value below this as just noise. GLOBAL_CUTOFF = 10 -class Dat(object): + +class Dat: """ A single piece of data for the aggregate routines to aggregate, using the a_* fields to grab the appropriate keys. @@ -40,10 +42,11 @@ def __init__(self, valtype, file_data, dev, val): self.val = val def __repr__(self): - return "valtype: %s, file_data: %s, dev: %s, val: %s " % (self.valtype, - self.file_data, - self.dev, - str(self.val)) + return ( + f"valtype: {self.valtype}, file_data: {self.file_data}, " + f"dev: {self.dev}, val: {self.val}" + ) + # a_* methods. # @@ -57,24 +60,30 @@ def __repr__(self): def a_unique(dat): return 'unique' + def a_orphaned(dat): return 'orphaned' + def a_dev(dat): if isinstance(dat.dev, list): return ' and '.join(dat.dev) else: return dat.dev + def a_project(dat): return dat.file_data.project + def a_fname(dat): return dat.file_data.fname + def a_valtype(dat): return dat.valtype + def agg(path, diction, dat): """ dat has the value to aggregate and the data associated with the value (FileData, devs) @@ -100,13 +109,16 @@ def agg(path, diction, dat): diction[last_k] = 0 diction[last_k] += dat.val + def agg_all(aggs, dat): for path, diction in aggs.items(): agg(path, diction, dat) + def create_agg(aggs, path): aggs[path] = {} + def split_out_dev_vals(dev_vals, departed_devs): """ Split the values in dev_vals into those that are held by only @@ -119,12 +131,6 @@ def split_out_dev_vals(dev_vals, departed_devs): the second the values held by departed devs. """ - def is_departed(dev): - return dev in departed_devs - - def is_not_departed(dev): - return dev not in departed_devs - def add_dev_val_lookup(devs, lookup, val): devs.sort() lookup_str = '\0'.join(devs) @@ -136,8 +142,8 @@ def add_dev_val_lookup(devs, lookup, val): departed_lookup = {} for devs, val in dev_vals: - present = filter(is_not_departed, devs) - departed = filter(is_departed, devs) + present = [dev for dev in devs if dev not in departed_devs] + departed = [dev for dev in devs if dev in departed_devs] if departed: departed.sort() if present: @@ -157,6 +163,7 @@ def add_dev_val_lookup(devs, lookup, val): return [(devs_lookup.split('\0'), val) for devs_lookup, val in lookup.items()], \ [(dep_lookup.split('\0'), val) for dep_lookup, val in departed_lookup.items()] + def summarize(lines, departed_devs): """ Aggregate the FileData in lines, considering all devs in @@ -210,6 +217,7 @@ def summarize(lines, departed_devs): return aggs + def tupelize(agg, tuples_and_vals, key_list): for k, v in agg.items(): loc_key = list(key_list) @@ -219,6 +227,7 @@ def tupelize(agg, tuples_and_vals, key_list): else: tupelize(v, tuples_and_vals, loc_key) + def sort_agg(agg, desc): tuples_and_vals = [] tupelize(agg, tuples_and_vals, []) @@ -229,6 +238,7 @@ def sort_agg(agg, desc): tuples_and_vals = [t[1] for t in tuples_and_vals] return tuples_and_vals + def by_valtype_html(valtype, nouns, noun, linker, limit): html = [] limit_str = '' @@ -247,32 +257,43 @@ def by_valtype_html(valtype, nouns, noun, linker, limit): html.append("") return html + def project_fname(project): - return os.path.join('projects', "%s.html" % project) + return os.path.join('projects', f"{project}.html") + def project_linker(project): - return "%s" % (project_fname(project), project) + return f"{project}" + def fname_fname(fname): - return os.path.join('files', "%s.html" % fname.replace(':', '__').replace(os.path.sep, '__')) + fname = fname.replace(':', '__').replace(os.path.sep, '__') + return os.path.join("files", f"{fname}.html") + def fname_linker(fname): - return "%s" % (fname_fname(fname), fname) + return f"{fname}" + def dev_fname(dev): - return os.path.join('devs', "%s.html" % hashlib.md5(dev).hexdigest()) + return os.path.join('devs', f"{hashlib.md5(dev.encode()).hexdigest()}.html") + def dev_linker(dev): - return "%s" % (dev_fname(dev), dev) + return f"{dev}" + def parent_linker(fnamer): def f(to_link): - return "%s" % (os.path.join('..', fnamer(to_link)), to_link) + p_link = os.path.join('..', fnamer(to_link)) + return f"{to_link}" return f + def summarize_by_valtype(agg_by_single, noun, linker): return summarize_top_by_valtype(agg_by_single, noun, linker, None) + def summarize_top_by_valtype(agg_by_single, noun, linker, limit): html = [] for valtype, nouns in agg_by_single.items(): @@ -282,10 +303,15 @@ def summarize_top_by_valtype(agg_by_single, noun, linker, limit): html.extend(by_valtype_html(valtype, nouns, noun, linker, limit)) return html + def add_global_explanation(html): - html.append('

Note: values smaller than %d have been truncated in the interest of space.

' % GLOBAL_CUTOFF) + html.append( + f'

Note: values smaller than {GLOBAL_CUTOFF} ' + 'have been truncated in the interest of space.

' + ) html.append('

Note: the scale of the bars is relative only within, not across, tables.

') + def create_index(aggs, output_dir): html = [] html.append("\nGit By a Bus Summary Results\n") @@ -299,48 +325,67 @@ def create_index(aggs, output_dir): outfil.write('\n'.join(html)) outfil.close() + def create_detail_page(detail, noun, valtype_args, fname, custom_lines_f): - html = [] - html.append("\nGit By a Bus Summary Results for %s: %s\n" % (noun, detail)) - html.append("

Index

") - html.append("

Git by a Bus Summary Results for %s: %s

" % (noun, detail)) + html = [ + "", + f"Git By a Bus Summary Results for {noun}: {detail}", + "", + "

Index

", + f"

Git by a Bus Summary Results for {noun}: {detail}

" + ] add_global_explanation(html) if custom_lines_f: html.extend(custom_lines_f(detail, noun, valtype_args, fname)) for vtarg in valtype_args: html.extend(summarize_top_by_valtype(vtarg[0], vtarg[1], vtarg[2], vtarg[3])) html.append("\n") - outfil = open(fname, 'w') - outfil.write('\n'.join(html)) - outfil.close() + with open(fname, 'w') as outfil: + outfil.write('\n'.join(html)) + -def create_detail_pages(output_dir, subdir, details, noun, detail_fname, aggs_with_nouns, custom_lines_f = None): +def create_detail_pages( + output_dir, subdir, details, noun, detail_fname, aggs_with_nouns, custom_lines_f = None +): try: os.mkdir(os.path.join(output_dir, subdir)) - except: + except OSError: pass for detail in details: outfile_name = os.path.join(output_dir, detail_fname(detail)) - vt_args = [(agg[detail], nouns, linker, None) for agg, nouns, linker in aggs_with_nouns if detail in agg] + vt_args = [ + (agg[detail], nouns, linker, None) + for agg, nouns, linker in aggs_with_nouns if detail in agg + ] create_detail_page(detail, noun, vt_args, outfile_name, custom_lines_f) + def create_project_pages(aggs, output_dir): dev_agg = aggs[(a_project, a_valtype, a_dev)] fname_agg = aggs[(a_project, a_valtype, a_fname)] projects = fname_agg.keys() - create_detail_pages(output_dir, 'projects', projects, 'Project', project_fname, [(dev_agg, 'Devs', parent_linker(dev_fname)), (fname_agg, 'Files', parent_linker(fname_fname))]) - + create_detail_pages( + output_dir, + 'projects', + projects, + 'Project', + project_fname, + [ + (dev_agg, 'Devs', parent_linker(dev_fname)), + (fname_agg, 'Files', parent_linker(fname_fname)) + ], + ) def create_dev_pages(aggs, output_dir, departed_devs): - # callback to pass into create_detail_pages to make # # * the links to individual devs making up a group and # # * the table of devs with most shared knowledge for individual # devs + def dev_custom(devs, noun, valtype_args, fname): html = [] linker = parent_linker(dev_fname) @@ -379,30 +424,43 @@ def dev_custom(devs, noun, valtype_args, fname): project_agg = aggs[(a_dev, a_valtype, a_project)] fname_agg = aggs[(a_dev, a_valtype, a_fname)] devs = fname_agg.keys() - create_detail_pages(output_dir, 'devs', devs, 'Dev', dev_fname, [(project_agg, 'Projects', parent_linker(project_fname)), - (fname_agg, 'Files', parent_linker(fname_fname))], dev_custom) + create_detail_pages( + output_dir, + 'devs', + devs, + 'Dev', + dev_fname, + [ + (project_agg, 'Projects', parent_linker(project_fname)), + (fname_agg, 'Files', parent_linker(fname_fname)) + ], + dev_custom, + ) def create_file_pages(aggs, output_dir): dev_agg = aggs[(a_fname, a_valtype, a_dev)] fnames = dev_agg.keys() - create_detail_pages(output_dir, 'files', fnames, 'File', fname_fname, [(dev_agg, 'Devs', parent_linker(dev_fname))]) + create_detail_pages( + output_dir, + 'files', + fnames, + 'File', + fname_fname, + [ + (dev_agg, 'Devs', parent_linker(dev_fname)) + ], + ) -def add_dev_dev(dev_dev, dev1, dev2, diff): - if dev1 not in dev_dev: - dev_dev[dev1] = {} - dev_dev[dev1][dev2] = diff def read_dev_x_cmp(x_cmp_fname, make_sym): - dev_dev = {} - fil = open(x_cmp_fname, 'r') - for line in fil: - line = line.strip() - dev1, dev2, diff = line.split('\t') - diff = float(diff) - add_dev_dev(dev_dev, dev1, dev2, diff) - if make_sym: - add_dev_dev(dev_dev, dev2, dev1, diff) - fil.close() + dev_dev = defaultdict(dict) + with open(x_cmp_fname, 'r') as fil: + for line in fil: + dev1, dev2, diff = line.strip().split('\t') + diff = float(diff) + dev_dev[dev1][dev2] = diff + if make_sym: + dev_dev[dev2][dev1] = diff return dev_dev def create_summary(lines, output_dir, departed_devs): @@ -413,17 +471,20 @@ def create_summary(lines, output_dir, departed_devs): create_file_pages(aggs, output_dir) if __name__ == '__main__': - parser = OptionParser() - parser.add_option('-d', '--departed-dev-file', dest='departed_dev_file', metavar='FILE', - help='File listing departed devs, one per line') - options, args = parser.parse_args() + parser = ArgumentParser() + parser.add_argument( + '-d', '--departed-dev-file', dest='departed_dev_file', metavar='FILE', + help='File listing departed devs, one per line' + ) + parser.add_argument("output_path") + args = parser.parse_args() departed_devs = [] - if options.departed_dev_file: - parse_departed_devs(options.departed_dev_file, departed_devs) + if args.departed_dev_file: + parse_departed_devs(args.departed_dev_file, departed_devs) - create_summary(sys.stdin, args[0], departed_devs) + create_summary(sys.stdin, args.output_path, departed_devs) # print to the tsv so if folks look there they get redirected # correctly - print "Summary is available at %s/index.html" % args[0] + print(f"Summary is available at {args.output_path}/index.html") diff --git a/svn_file_stats.py b/svn_file_stats.py index 08d161c..51036c3 100644 --- a/svn_file_stats.py +++ b/svn_file_stats.py @@ -9,14 +9,14 @@ """ import sys -import os import re import pysvn from common import is_interesting, FileData, safe_author_name -def gen_stats(root, project, interesting, not_interesting, options): + +def gen_stats(root, project, interesting, not_interesting, _): """ root: the root svn url of the project we are generating stats for (does not need to be the root of the svn repo). Must be a url, @@ -41,20 +41,27 @@ def gen_stats(root, project, interesting, not_interesting, options): # not our project root repo_root = client.root_url_from_path(root) - interesting_fs = [f[0].repos_path for f in client.list(root, recurse=True) if - is_interesting(f[0].repos_path, interesting, not_interesting) and f[0].kind == pysvn.node_kind.file] + interesting_fs = [ + f[0].repos_path + for f in client.list(root, recurse=True) + if is_interesting(f[0].repos_path, interesting, not_interesting) and + f[0].kind == pysvn.node_kind.file + ] for f in interesting_fs: dev_experience = parse_dev_experience(f, client, repo_root) if dev_experience: fd = FileData(':'.join([project, f])) # don't take revisions that are 0 lines added and 0 removed, like properties - fd.dev_experience = [(dev, added, removed) for dev, added, removed in dev_experience if added or removed] + fd.dev_experience = [ + (dev, added, removed) for dev, added, removed in dev_experience if added or removed + ] fd.cnt_lines = count_lines(f, client, repo_root) fd_line = fd.as_line() if fd_line.strip(): yield fd_line + def parse_dev_experience(f, client, repo_root): """ f: a path relative to repo_root for a file from whose log we want @@ -146,21 +153,21 @@ def parse_dev_experience(f, client, repo_root): # on one occasion I saw a non-binary item that existed in # the filesystem with svn ls but errored out with a diff # against that revision. Note the error and proceed. - print >> sys.stderr, "Error diffing %s %s and %s %s: " % \ - (old_path, str(old_rev), new_path, str(new_rev)), sys.exc_info()[0] + print( + f"Error diffing {old_path} {old_rev} and {new_path} {new_rev}: ", + sys.exc_info()[0], + file=sys.stderr, + ) return dev_experience + def count_lines(f, client, repo_root): """ Count the lines in the file located at path f under repo root. """ - txt = client.cat("%s%s" % (repo_root, f)) - lines = txt.count('\n') - if not txt.endswith('\n'): + txt = client.cat(f"{repo_root}{f}") + lines = txt.count("\n") + if not txt.endswith("\n"): lines += 1 return lines - - - -