Use b4 to generate release notes from PR

b4/__init__.py

@@ -104,6 +104,7 @@ def _dkim_log_filter(record):
     'linkmask': LOREADDR + '/r/%s',
     'trailer-order': DEFAULT_TRAILER_ORDER,
     'listid-preference': '*.feeds.kernel.org,*.linux.dev,*.kernel.org,*',
+    'pr-tracker-email': '[email protected]',
     'save-maildirs': 'no',
     # off: do not bother checking attestation
     # check: print an attaboy when attestation is found
@@ -274,7 +275,7 @@ def get_series(self, revision=None, sloppytrailers=False, reroll=True):
             for member in lser.patches:
                 if member is not None and member.in_reply_to is not None:
                     potential = self.get_by_msgid(member.in_reply_to)
-                    if potential is not None and potential.has_diffstat and not potential.has_diff:
+                    if potential is not None and potential.maybe_cover():
                         # This is *probably* the cover letter
                         lser.patches[0] = potential
                         lser.has_cover = True
@@ -350,7 +351,7 @@ def get_series(self, revision=None, sloppytrailers=False, reroll=True):
 
         return lser
 
-    def add_message(self, msg):
+    def add_message(self, msg, needcover=False):
         msgid = LoreMessage.get_clean_msgid(msg)
         if msgid in self.msgid_map:
             logger.debug('Already have a message with this msgid, skipping %s', msgid)
@@ -366,20 +367,20 @@ def add_message(self, msg):
             self.followups.append(lmsg)
             return
 
-        if lmsg.counter == 0 and (not lmsg.counters_inferred or lmsg.has_diffstat):
+        if lmsg.likely_cover():
             # Cover letter
             # Add it to covers -- we'll deal with them later
             logger.debug('  adding as v%s cover letter', lmsg.revision)
             self.covers[lmsg.revision] = lmsg
             return
 
         if lmsg.has_diff:
-            if lmsg.revision not in self.series:
+            if lmsg.revision not in self.series or needcover:
                 if lmsg.revision_inferred and lmsg.in_reply_to:
                     # We have an inferred revision here.
                     # Do we have an upthread cover letter that specifies a revision?
                     irt = self.get_by_msgid(lmsg.in_reply_to)
-                    if irt is not None and irt.has_diffstat and not irt.has_diff:
+                    if irt is not None and irt.likely_cover():
                         # Yes, this is very likely our cover letter
                         logger.debug('  fixed revision to v%s', irt.revision)
                         lmsg.revision = irt.revision
@@ -860,6 +861,7 @@ def __init__(self, msg):
         self.pr_repo = None
         self.pr_ref = None
         self.pr_tip_commit = None
+        self.pr_merge_commit = None
         self.pr_remote_tip_commit = None
 
         # Patchwork hash
@@ -966,6 +968,12 @@ def __init__(self, msg):
                 if trailer[0].lower() not in badtrailers:
                     self.trailers.append(trailer)
 
+    def maybe_cover(self):
+        return self.has_diffstat and not self.has_diff
+
+    def likely_cover(self):
+        return self.counter == 0 and (not self.counters_inferred or self.mayby_cover())
+
     def get_trailers(self, sloppy=False):
         trailers = list()
         mismatches = set()
@@ -1285,6 +1293,17 @@ def get_clean_msgid(msg, header='Message-Id'):
                 msgid = matches.groups()[0]
         return msgid
 
+    # Get commit id from git am formatted patch
+    @staticmethod
+    def get_commit_id(msg):
+        commitid = None
+        unixhdr = msg.get_unixfrom()
+        if unixhdr:
+            matches = re.search(r'^From ([0-9a-f]+)', unixhdr)
+            if matches:
+                commitid = matches.groups()[0]
+        return commitid
+
     @staticmethod
     def get_preferred_duplicate(msg1, msg2):
         config = get_main_config()
@@ -2047,7 +2066,7 @@ def get_cache_dir(appname: str = 'b4') -> str:
         fullpath = os.path.join(cachedir, entry)
         st = os.stat(fullpath)
         if st.st_mtime < expage:
-            logger.debug('Cleaning up cache: %s', entry)
+            logger.debug('Cleaning up cache: %s mtime=%d < %d', entry, st.st_mtime, expage)
             if os.path.isdir(fullpath):
                 shutil.rmtree(fullpath)
             else:

b4/command.py

@@ -103,6 +103,11 @@ def cmd_diff(cmdargs):
     b4.diff.main(cmdargs)
 
 
+def cmd_rn(cmdargs):
+    import b4.rn
+    b4.rn.main(cmdargs)
+
+
 def cmd():
     # noinspection PyTypeChecker
     parser = argparse.ArgumentParser(
@@ -191,6 +196,8 @@ def cmd():
                              'the identity must match a [sendemail "identity"] config section'))
     sp_pr.add_argument('--dry-run', dest='dryrun', action='store_true', default=False,
                        help='Force a --dry-run on git-send-email invocation (use with -s)')
+    sp_pr.add_argument('--no-cover', dest='nocover', action='store_true', default=False,
+                       help='Do not save the cover letter (on by default when using -o -)')
     sp_pr.add_argument('msgid', nargs='?',
                        help='Message ID to process, or pipe a raw message')
     sp_pr.set_defaults(func=cmd_pr)
@@ -248,6 +255,15 @@ def cmd():
                        help='Show all developer keys found in a thread')
     sp_kr.set_defaults(func=cmd_kr)
 
+    # b4 rn
+    sp_rn = subparsers.add_parser('rn', help='Generate release notes from pull request')
+    cmd_retrieval_common_opts(sp_rn)
+    sp_rn.add_argument('-g', '--gitdir', default=None,
+                       help='Operate on this git tree instead of current dir')
+    sp_rn.add_argument('-o', '--output-file', dest='outfile', default=None,
+                       help='Write release notes into this file instead of outputting to stdout')
+    sp_rn.set_defaults(func=cmd_rn)
+
     cmdargs = parser.parse_args()
 
     logger.setLevel(logging.DEBUG)

b4/diff.py

@@ -44,7 +44,8 @@ def diff_same_thread_series(cmdargs):
         if not msgs:
             logger.critical('Unable to retrieve thread: %s', msgid)
             return
-        msgs = b4.mbox.get_extra_series(msgs, direction=-1, wantvers=wantvers, useproject=cmdargs.useproject)
+        msgs = b4.mbox.get_extra_series(msgs, direction=-1, wantvers=wantvers,
+                                        nocache=cmdargs.nocache, useproject=cmdargs.useproject)
         if os.path.exists(cachedir):
             shutil.rmtree(cachedir)
         pathlib.Path(cachedir).mkdir(parents=True)

b4/mbox.py

@@ -479,11 +479,15 @@ def save_as_quilt(am_msgs, q_dirname):
         for patch_filename in patch_filenames:
             sfh.write('%s\n' % patch_filename)
 
-
-def get_extra_series(msgs: list, direction: int = 1, wantvers: Optional[int] = None, nocache: bool = False,
-                     useproject: Optional[str] = None) -> list:
-    base_msg = None
-    latest_revision = None
+# Get older/newer revisions of a patch series from public-inbox
+#
+# @direction:
+#  1 - look for newer revisions (default)
+# -1 - look for older revisions
+#  0 - look for latest revision in public-inbox (regardless of @base_msg)
+def get_extra_series(msgs: list, direction: int = 1, wantvers: Optional[list] = None, nocache: bool = False,
+                     base_msg = None, useproject: Optional[str] = None) -> list:
+    latest_revision = 0
     seen_msgids = set()
     seen_covers = set()
     obsoleted = list()
@@ -521,6 +525,36 @@ def get_extra_series(msgs: list, direction: int = 1, wantvers: Optional[int] = N
         logger.debug('Could not find cover of 1st patch in mbox')
         return msgs
 
+    # For query by @base_msg, check if we have a cache of this lookup
+    base_msgid = b4.LoreMessage.get_clean_msgid(base_msg)
+    identifier = base_msgid
+    # Use commit id as key to cache of git am formatted @base_msg
+    if not identifier:
+        identifier = b4.LoreMessage.get_commit_id(base_msg)
+    if identifier is None:
+        logger.debug('Could not find find base msgid for series')
+        return msgs
+
+    cachedir = None
+    if identifier and len(msgs) == 0 and not wantvers:
+        if useproject:
+            identifier += '-' + useproject
+        if direction > 0:
+            identifier += '+'
+        elif direction < 0:
+            identifier += '-'
+        cachedir = b4.get_cache_file(identifier, suffix='extra.msgs')
+
+    if cachedir and os.path.exists(cachedir) and not nocache:
+        logger.debug('Using cached copy of %s at %s', identifier, cachedir)
+        msgs = list()
+        for msg in os.listdir(cachedir):
+            with open(os.path.join(cachedir, msg), 'rb') as fh:
+                msgs.append(email.message_from_binary_file(fh))
+        return msgs
+    else:
+        logger.debug('No cached copy for %s', identifier)
+
     config = b4.get_main_config()
     loc = urllib.parse.urlparse(config['midmask'])
     if not useproject:
@@ -552,7 +586,7 @@ def get_extra_series(msgs: list, direction: int = 1, wantvers: Optional[int] = N
     else:
         # Get subject info from base_msg again
         lsub = b4.LoreSubject(base_msg['Subject'])
-        if not len(lsub.prefixes):
+        if direction > 0 and not len(lsub.prefixes):
             logger.debug('Not checking for new revisions: no prefixes on the cover letter.')
             return msgs
         if direction < 0 and latest_revision <= 1:
@@ -561,18 +595,22 @@ def get_extra_series(msgs: list, direction: int = 1, wantvers: Optional[int] = N
         if direction < 0 and wantvers is None:
             wantvers = [latest_revision - 1]
 
-        base_msgid = b4.LoreMessage.get_clean_msgid(base_msg)
         fromeml = email.utils.getaddresses(base_msg.get_all('from', []))[0][1]
         msgdate = email.utils.parsedate_tz(str(base_msg['Date']))
         startdate = time.strftime('%Y%m%d', msgdate[:9])
         if direction > 0:
             q = 's:"%s" AND f:"%s" AND d:%s..' % (lsub.subject.replace('"', ''), fromeml, startdate)
             queryurl = '%s?%s' % (listarc, urllib.parse.urlencode({'q': q, 'x': 'A', 'o': '-1'}))
             logger.critical('Checking for newer revisions on %s', listarc)
-        else:
+        elif direction < 0:
             q = 's:"%s" AND f:"%s" AND d:..%s' % (lsub.subject.replace('"', ''), fromeml, startdate)
             queryurl = '%s?%s' % (listarc, urllib.parse.urlencode({'q': q, 'x': 'A', 'o': '1'}))
             logger.critical('Checking for older revisions on %s', listarc)
+        else:
+            # Find latest revision in public-inbox to match base_msg subject
+            q = 's:"%s"' % (lsub.subject.replace('"', ''))
+            queryurl = '%s?%s' % (listarc, urllib.parse.urlencode({'q': q, 'x': 'A'}))
+            logger.debug('Checking for revisions on %s', listarc)
 
         logger.debug('Query URL: %s', queryurl)
         session = b4.get_requests_session()
@@ -592,11 +630,11 @@ def get_extra_series(msgs: list, direction: int = 1, wantvers: Optional[int] = N
         for entry in entries:
             title = entry.find('atom:title', ns).text
             lsub = b4.LoreSubject(title)
-            if lsub.reply or lsub.counter > 1:
+            if lsub.reply or (direction != 0 and lsub.counter > 1):
                 logger.debug('Ignoring result (not interesting): %s', title)
                 continue
             link = entry.find('atom:link', ns).get('href')
-            if direction > 0 and lsub.revision <= latest_revision:
+            if direction >= 0 and lsub.revision <= latest_revision:
                 logger.debug('Ignoring result (not new revision): %s', title)
                 continue
             elif direction < 0 and lsub.revision >= latest_revision:
@@ -610,13 +648,13 @@ def get_extra_series(msgs: list, direction: int = 1, wantvers: Optional[int] = N
                 continue
             if lsub.revision == 1 and lsub.revision == latest_revision:
                 # Someone sent a separate message with an identical title but no new vX in the subject line
-                if direction > 0:
+                if direction >= 0:
                     # It's *probably* a new revision.
                     logger.debug('Likely a new revision: %s', title)
                 else:
                     # It's *probably* an older revision.
                     logger.debug('Likely an older revision: %s', title)
-            elif direction > 0 and lsub.revision > latest_revision:
+            elif direction >= 0 and lsub.revision > latest_revision:
                 logger.debug('Definitely a new revision [v%s]: %s', lsub.revision, title)
             elif direction < 0 and lsub.revision < latest_revision:
                 logger.debug('Definitely an older revision [v%s]: %s', lsub.revision, title)
@@ -633,6 +671,13 @@ def get_extra_series(msgs: list, direction: int = 1, wantvers: Optional[int] = N
                 nt_msgs += potentials
                 logger.info('   Added %s messages from that thread', len(potentials))
 
+    # Write results of @base_msg query to cache
+    if cachedir:
+        if os.path.exists(cachedir):
+            shutil.rmtree(cachedir)
+        pathlib.Path(cachedir).mkdir(parents=True)
+        at = 0
+
     # Append all of these to the existing mailbox
     for nt_msg in nt_msgs:
         nt_msgid = b4.LoreMessage.get_clean_msgid(nt_msg)
@@ -643,6 +688,10 @@ def get_extra_series(msgs: list, direction: int = 1, wantvers: Optional[int] = N
         logger.debug('Adding: %s', nt_subject)
         msgs.append(nt_msg)
         seen_msgids.add(nt_msgid)
+        if cachedir:
+            with open(os.path.join(cachedir, '%04d' % at), 'wb') as fh:
+                fh.write(nt_msg.as_bytes(policy=b4.emlpolicy))
+            at += 1
 
     return msgs
 
@@ -725,7 +774,8 @@ def main(cmdargs):
         return
 
     if len(msgs) and cmdargs.checknewer:
-        msgs = get_extra_series(msgs, direction=1, useproject=cmdargs.useproject)
+        msgs = get_extra_series(msgs, direction=1, nocache=cmdargs.nocache,
+                                useproject=cmdargs.useproject)
 
     if cmdargs.subcmd in ('am', 'shazam'):
         make_am(msgs, cmdargs, msgid)