diff --git a/.github/scripts/add_psi_ms_id.py b/.github/scripts/add_psi_ms_id.py new file mode 100644 index 000000000..79d472344 --- /dev/null +++ b/.github/scripts/add_psi_ms_id.py @@ -0,0 +1,147 @@ +#!/usr/bin/env python3 +"""Back-propagate PSI-MS column-model MS:IDs into the repo-rt catalog. + +The return leg of the cross-repo loop. psi-ms-CV is the sole authority for ID +assignment; this script only MIRRORS the assigned ids into the "psi_ms_id" column of +resources/column_database/column_database.tsv, by an exact (company, Column name) join +against the mapping psi-ms-CV publishes at .github/repo-rt/psi-ms-column-ids.tsv. The +keys are normalised the same way the generator normalises its keys, so the join is a +dumb exact-string lookup -- no leaf-label or collision logic is duplicated. + +It is deliberately conservative, because repo-rt is *upstream* of id assignment: + * a row whose (company, Column name) is in the mapping is set to that id -- filling a + blank cell or updating a changed one (e.g. after a rename minted a new id); + * a row NOT in the mapping is LEFT AS-IS -- a brand-new repo-rt model that psi-ms-CV + has not minted yet stays blank until a later cycle, and an existing id is never + erased just because the (possibly older) mapping does not list it; + * an empty mapping aborts (the download almost certainly failed) rather than touching + the catalog. + +Only the psi_ms_id field is edited (the column is appended if absent); quoting and +spacing on every other field survive byte-for-byte, and re-running with an unchanged +mapping is a no-op. + +In repo-rt CI the mapping is fetched by raw URL: + curl -fsSL -o psi-ms-column-ids.tsv \\ + https://raw.githubusercontent.com/HUPO-PSI/psi-ms-CV/master/.github/repo-rt/psi-ms-column-ids.tsv + +Usage: + python add_psi_ms_id.py --mapping psi-ms-column-ids.tsv \\ + --catalog resources/column_database/column_database.tsv [--report report.md] +""" +import argparse +import sys + +ID_COL = "psi_ms_id" + + +def norm(text): + """Normalise a join key the way the generator does -- clean(s.strip()): strip, then + drop C0 control chars -- so the catalog keys match the mapping's cleaned + (company, Column name) even if a cell carries an interior control char.""" + return "".join(ch for ch in text.strip() if ch >= " ") + + +def read_mapping(path): + """{(company, Column name): psi_ms_id} from the published mapping TSV.""" + mapping = {} + for line in open(path, encoding="utf-8").read().split("\n"): + if not line or line.startswith("company\t"): + continue + company, name, ms_id = line.split("\t") + mapping[(norm(company), norm(name))] = ms_id.strip() + return mapping + + +def back_propagate(catalog_path, mapping): + """Fill/refresh the psi_ms_id column of catalog_path in place from `mapping`. + + Returns a report dict {filled, changed, blank_new, preserved}. Raises ValueError on + an empty mapping or a header missing the company / Column name columns.""" + if not mapping: + raise ValueError("mapping is empty; the download almost certainly failed -- aborting") + + # Read raw bytes so line endings survive exactly; a strict UTF-8 decode is a sanity check. + lines = open(catalog_path, "rb").read().decode("utf-8").split("\n") + header = lines[0].split("\t") + if "company" not in header or "Column name" not in header: + raise ValueError(f"{catalog_path} header lacks 'company'/'Column name': {header[:5]}...") + company_i, name_i = header.index("company"), header.index("Column name") + has_id = ID_COL in header + id_i = header.index(ID_COL) if has_id else None + need = max(company_i, name_i, id_i) if has_id else max(company_i, name_i) + + rep = {"filled": 0, "changed": 0, "blank_new": 0, "preserved": 0} + for k, line in enumerate(lines): + if line == "": + continue + fields = line.split("\t") + if k == 0: + if not has_id: + fields.append(ID_COL) + lines[k] = "\t".join(fields) + continue + if len(fields) <= need: + if not has_id: # appending the column: keep every row's width consistent + fields.append("") + lines[k] = "\t".join(fields) + continue + existing = fields[id_i].strip() if has_id else "" + key = (norm(fields[company_i]), norm(fields[name_i])) + if not all(key): + new = existing # blank company/name: not a model, leave as-is + elif key in mapping: + new = mapping[key] + if existing == "": + rep["filled"] += 1 + elif existing != new: + rep["changed"] += 1 + elif existing: + new = existing # not in this mapping: keep the id we already have + rep["preserved"] += 1 + else: + new = "" # new repo-rt model, no id yet -> stays blank, fills a later cycle + rep["blank_new"] += 1 + if has_id: + fields[id_i] = new + else: + fields.append(new) + lines[k] = "\t".join(fields) + + open(catalog_path, "w", encoding="utf-8", newline="").write("\n".join(lines)) + return rep + + +def report_markdown(rep, catalog_path): + return "\n".join([ + f"- catalog: `{catalog_path}`", + f"- psi_ms_id filled (was blank): **{rep['filled']}**", + f"- psi_ms_id changed (e.g. rename): **{rep['changed']}**", + f"- still blank (new models awaiting a PSI-MS id): **{rep['blank_new']}**", + f"- existing id preserved (not in this mapping): **{rep['preserved']}**", + ]) + "\n" + + +def main(): + ap = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + ap.add_argument("--mapping", default="psi-ms-column-ids.tsv", + help="the psi-ms-CV mapping TSV (company / Column name / psi_ms_id)") + ap.add_argument("--catalog", default="resources/column_database/column_database.tsv", + help="the repo-rt column_database.tsv to fill in place") + ap.add_argument("--report", help="write a Markdown summary to this path (for the PR body)") + args = ap.parse_args() + + rep = back_propagate(args.catalog, read_mapping(args.mapping)) + print(f"wrote {args.catalog}") + print(f"filled={rep['filled']} changed={rep['changed']} " + f"blank_new={rep['blank_new']} preserved={rep['preserved']}") + if rep["filled"] + rep["changed"] == 0 and rep["preserved"] + rep["blank_new"] > 0: + print("WARNING: nothing matched the mapping -- check key normalisation / the " + "downloaded mapping is current", file=sys.stderr) + if args.report: + open(args.report, "w", encoding="utf-8").write(report_markdown(rep, args.catalog)) + print(f"wrote {args.report}") + + +if __name__ == "__main__": + main() diff --git a/.github/workflows/sync-psi-ms-ids.yml b/.github/workflows/sync-psi-ms-ids.yml new file mode 100644 index 000000000..0d5b1d103 --- /dev/null +++ b/.github/workflows/sync-psi-ms-ids.yml @@ -0,0 +1,83 @@ +# DESTINED FOR THE repo-rt REPOSITORY — place at .github/workflows/sync-psi-ms-ids.yml +# (and put add_psi_ms_id.py at .github/scripts/add_psi_ms_id.py, or adjust the path below). +# +# Return leg of the cross-repo loop: pulls the MS:ID mapping that HUPO-PSI/psi-ms-CV +# publishes and back-fills the psi_ms_id column of the column database, then opens a PR. +# psi-ms-CV is the authority for ID assignment; this only mirrors the ids. It runs a +# couple of days after psi-ms-CV's own weekly sync so the latest mapping is usually +# merged there; brand-new repo-rt models psi-ms-CV hasn't minted yet are left blank and +# picked up on a later run (no error). +name: Sync PSI-MS column IDs + +on: + schedule: + - cron: "0 6 * * 3" # weekly, Wednesday 06:00 UTC (after psi-ms-CV's Monday run) + workflow_dispatch: {} # allow manual runs + +permissions: + contents: write + pull-requests: write + +concurrency: + group: sync-psi-ms-ids + cancel-in-progress: false + +jobs: + sync: + # Pinned (not ubuntu-latest) for reproducible runs; bump when GitHub deprecates the image. + runs-on: ubuntu-24.04 + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.11" + # add_psi_ms_id.py is pure stdlib — no dependencies to install. + + - name: Download the psi-ms-CV id mapping + run: | + curl -fsSL --retry 3 --connect-timeout 30 --max-time 300 -o psi-ms-column-ids.tsv \ + https://raw.githubusercontent.com/HUPO-PSI/psi-ms-CV/master/.github/repo-rt/psi-ms-column-ids.tsv + + - name: Back-propagate MS:IDs into the catalog + run: | + python .github/scripts/add_psi_ms_id.py \ + --mapping psi-ms-column-ids.tsv \ + --catalog resources/column_database/column_database.tsv \ + --report sync-report.md + + - name: Build PR body + id: prbody + run: | + { + echo "body<> "$GITHUB_OUTPUT" + + - name: Remove downloaded scratch files + run: rm -f psi-ms-column-ids.tsv sync-report.md + + - name: Create pull request + uses: peter-evans/create-pull-request@v6 + with: + # A repo-rt secret with contents+pull-requests write; or use ${{ secrets.GITHUB_TOKEN }} + # plus Settings → Actions → "Allow GitHub Actions to create and approve pull requests". + token: ${{ secrets.PSI_MS_ID_SYNC_TOKEN }} + add-paths: resources/column_database/column_database.tsv + branch: sync/psi-ms-column-ids + delete-branch: true + commit-message: "Sync psi_ms_id column from PSI-MS-CV" + title: "Sync psi_ms_id column from PSI-MS-CV" + body: ${{ steps.prbody.outputs.body }}