From 32a4a5ff38e323c70a058dd9d466d9b35b2e76c4 Mon Sep 17 00:00:00 2001 From: 0verwrite <31691645+overwrite00@users.noreply.github.com> Date: Thu, 23 Jul 2026 19:44:08 +0200 Subject: [PATCH 1/3] release: prepare 2.1.0-beta.1 --- .github/pull_request_template.md | 6 +- .github/workflows/beta-release.yml | 23 +- .github/workflows/release.yml | 10 +- .github/workflows/test_build.yml | 15 +- .gitignore | 18 +- ARCHITECTURE.md | 761 ++++++++--------------------- CHANGELOG.md | 29 +- CONTRIBUTING.md | 12 +- DEVELOPMENT.md | 132 ++--- NullifyPDF.py | 458 +++++++++++++++-- OCR_SETUP.md | 44 ++ README.md | 56 ++- SECURITY.md | 41 +- TROUBLESHOOTING.md | 22 +- USER_GUIDE.md | 51 +- build_local.py | 172 ++++++- privacy_core.py | 146 ++++++ requirements.txt | 1 + scripts/download_ocr_data.py | 43 ++ tests/test_build_config.py | 96 ++++ tests/test_privacy_core.py | 53 ++ tests/test_validation.py | 27 +- 22 files changed, 1423 insertions(+), 793 deletions(-) create mode 100644 OCR_SETUP.md create mode 100644 privacy_core.py create mode 100644 scripts/download_ocr_data.py create mode 100644 tests/test_build_config.py create mode 100644 tests/test_privacy_core.py diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 7faf4f3..2be4fd9 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -27,7 +27,8 @@ Describe what testing you've done: - [ ] Tested on macOS - [ ] Tested on Linux - [ ] All tests pass locally: `pytest tests/ -v` -- [ ] Build succeeds locally: `python build_local.py` +- [ ] Build succeeds locally: `python build_local.py --lite` +- [ ] Full OCR build checked when packaging/OCR changes: `python build_local.py --full` **Manual Testing Steps:** 1. Load PDF: [describe what PDF you used] @@ -70,7 +71,8 @@ Any additional context that reviewers should know? Before marking as ready, please ensure: - [ ] Tests pass: `pytest tests/ -v` -- [ ] Build passes: `python build_local.py` +- [ ] Build passes: `python build_local.py --lite` +- [ ] Full OCR build passes when packaging/OCR changes: `python build_local.py --full` - [ ] Code is clean and follows style guide - [ ] All checkboxes above are complete - [ ] You've tested your changes locally diff --git a/.github/workflows/beta-release.yml b/.github/workflows/beta-release.yml index f70ab03..cdee2f3 100644 --- a/.github/workflows/beta-release.yml +++ b/.github/workflows/beta-release.yml @@ -22,6 +22,7 @@ jobs: fail-fast: false matrix: os: [windows-latest, ubuntu-22.04, macos-latest] + variant: [lite, full] steps: - name: Checkout Code @@ -57,18 +58,20 @@ jobs: echo "NULLIFYPDF_BETA_SUFFIX=${TAG#*-}" >> "$GITHUB_ENV" - name: Build + env: + NULLIFYPDF_BUILD_VARIANT: ${{ matrix.variant }} run: python build_local.py - name: Upload Artifact uses: actions/upload-artifact@v7 with: - name: beta-build-${{ runner.os }} + name: beta-build-${{ runner.os }}-${{ matrix.variant }} retention-days: 5 path: | - dist/NullifyPDF_v*_Windows.exe - dist/NullifyPDF_v*_macOS.zip - dist/NullifyPDF_v*_Ubuntu.deb - dist/NullifyPDF_v*_Fedora.rpm + dist/NullifyPDF_v*_Windows_*.exe + dist/NullifyPDF_v*_macOS_*.zip + dist/NullifyPDF_v*_Ubuntu_*.deb + dist/NullifyPDF_v*_Fedora_*.rpm create_beta_release: name: Create GitHub Beta Release @@ -88,10 +91,10 @@ jobs: with: prerelease: true files: | - all-builds/NullifyPDF_v*_Windows.exe - all-builds/NullifyPDF_v*_macOS.zip - all-builds/NullifyPDF_v*_Ubuntu.deb - all-builds/NullifyPDF_v*_Fedora.rpm - body: "Beta release NullifyPDF (build da develop). Non usare in produzione. Se supera i test, questa build verra' promossa a stable senza ricompilazione." + all-builds/NullifyPDF_v*_Windows_*.exe + all-builds/NullifyPDF_v*_macOS_*.zip + all-builds/NullifyPDF_v*_Ubuntu_*.deb + all-builds/NullifyPDF_v*_Fedora_*.rpm + body: "Beta release NullifyPDF (build da develop). Include varianti Lite e Full OCR. Non usare in produzione. Se supera i test, questa build verra' promossa a stable senza ricompilazione." env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index c3956b3..899714b 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -51,10 +51,10 @@ jobs: run: | mkdir -p stable-assets gh release download "${{ steps.beta.outputs.tag }}" --repo "${{ github.repository }}" --dir stable-assets \ - --pattern "NullifyPDF_v*_Windows.exe" \ - --pattern "NullifyPDF_v*_macOS.zip" \ - --pattern "NullifyPDF_v*_Ubuntu.deb" \ - --pattern "NullifyPDF_v*_Fedora.rpm" + --pattern "NullifyPDF_v*_Windows_*.exe" \ + --pattern "NullifyPDF_v*_macOS_*.zip" \ + --pattern "NullifyPDF_v*_Ubuntu_*.deb" \ + --pattern "NullifyPDF_v*_Fedora_*.rpm" - name: Strip beta suffix from filenames run: | @@ -75,6 +75,6 @@ jobs: body: | Release Stabile NullifyPDF. Promossa dalla beta `${{ steps.beta.outputs.tag }}` (build gia' verificata, nessuna ricompilazione). - Include App Bundle macOS (ZIP), EXE per Windows e pacchetti nativi Linux. + Include varianti Lite e Full OCR per Windows, macOS e Linux. env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/test_build.yml b/.github/workflows/test_build.yml index 4f5f340..cf7995b 100644 --- a/.github/workflows/test_build.yml +++ b/.github/workflows/test_build.yml @@ -22,6 +22,7 @@ jobs: fail-fast: false matrix: os: [windows-latest, ubuntu-22.04, macos-latest] + variant: [lite, full] steps: - name: Checkout Code @@ -50,15 +51,17 @@ jobs: run: mypy NullifyPDF.py --ignore-missing-imports --no-error-summary 2>&1 | head -50 || true - name: Build + env: + NULLIFYPDF_BUILD_VARIANT: ${{ matrix.variant }} run: python build_local.py - name: Upload Artifact uses: actions/upload-artifact@v7 with: - name: test-build-${{ runner.os }} + name: test-build-${{ runner.os }}-${{ matrix.variant }} path: | - dist/NullifyPDF_v*_Windows.exe - dist/NullifyPDF_v*_macOS.zip - dist/NullifyPDF_v*_Ubuntu.deb - dist/NullifyPDF_v*_Fedora.rpm - retention-days: 3 \ No newline at end of file + dist/NullifyPDF_v*_Windows_*.exe + dist/NullifyPDF_v*_macOS_*.zip + dist/NullifyPDF_v*_Ubuntu_*.deb + dist/NullifyPDF_v*_Fedora_*.rpm + retention-days: 3 diff --git a/.gitignore b/.gitignore index 4afbbb1..7b0cf14 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,11 @@ __pycache__/ *$py.class *.so .Python +.pytest_cache/ +.mypy_cache/ +.coverage +.coverage.* +htmlcov/ build/ develop-eggs/ dist/ @@ -63,4 +68,15 @@ desktop.ini # Se vuoi includere un PDF di esempio specifico, usa: !esempio.pdf # Log del checker forense -*.log \ No newline at end of file +*.log + +# File generati da export privacy / pseudonimizzazione +*.nullifypdf.tmp +*.nullifypdf-map + +# OCR data is downloaded for Full builds, not stored in git. +ocr/tessdata/*.traineddata + +# Packaging temp directories +rpm_build_tmp/ +deb_build_tmp/ diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index b556592..81b74be 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -1,660 +1,317 @@ # πŸ—οΈ Architecture β€” NullifyPDF -This document explains the system design, components, and data flow of NullifyPDF. Understanding this helps with contributing code and debugging issues. +This document explains how NullifyPDF is structured today: application layers, privacy export flow, OCR support, release variants, and the main security boundaries that matter during maintenance. > [!TIP] -> Start with [System Overview](#system-overview), then explore specific components based on your interest. +> Read this after the [README](./README.md) if you want the technical view of how detection, redaction, pseudonymization, and packaging work together. --- -## πŸ“‹ System Overview +## πŸ“‹ Current Scope -``` -β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” -β”‚ NullifyPDF Application β”‚ -β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€ -β”‚ UI Layer β”‚ Core Logic β”‚ Data Layer β”‚ -β”‚ (PySide6) β”‚ (Python) β”‚ (Persistence) β”‚ -β”œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€ -β”‚ β€’ Main Window β”‚ β€’ PDF Engine β”‚ β€’ Blocklist β”‚ -β”‚ β€’ Canvas/Render β”‚ β€’ AI Pipeline β”‚ β€’ Allowlist β”‚ -β”‚ β€’ Dialogs β”‚ β€’ Text Extract β”‚ β€’ Logs β”‚ -β”‚ β€’ Sidebar β”‚ β€’ Redaction β”‚ β€’ Config β”‚ -β”‚ β€’ Events β”‚ β€’ Export β”‚ β”‚ -β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ - ↑ ↑ ↑ - QThread Worker Disk - (UI) (AI/PDF) (~/.nullifypdf/) -``` +NullifyPDF is a local-first desktop application for preparing PDFs before they are shared with third parties or uploaded to AI systems. -### πŸ”„ Data Flow +It supports two privacy export modes: -``` -User loads PDF - ↓ -[UI Thread] Render in QGraphicsView - ↓ -User clicks "Auto Redact" - ↓ -[Worker Thread] Extract text (QMutex lock) - ↓ -[Worker Thread] Run AI scan (Presidio + spaCy) - ↓ -[Worker Thread] Emit results to UI - ↓ -[UI Thread] Draw redaction boxes on canvas - ↓ -User clicks "Export PDF" - ↓ -[Worker Thread] Disk-backed temp export - ↓ -[Worker Thread] Destructive scrubbing (metadata, links, AcroForms) - ↓ -Export completed βœ… -``` +- **Irreversible anonymization** β€” selected content is redacted and removed from the exported PDF +- **Reversible pseudonymization** β€” selected content is replaced with stable placeholders, while original values are stored in a separate encrypted restore map ---- - -## πŸ”§ Core Components +> [!NOTE] +> Reversible mode is pseudonymization, not true anonymization, because the encrypted restore map can re-identify the original values. -### 1️⃣ Main Entry Point β€” `NullifyPDF.py` +--- -**Purpose:** Application initialization, GUI setup, event handling. +## 🧩 High-Level Components -**Key Classes:** +| πŸ“„ File | ✨ Responsibility | +| ------ | ----------------- | +| `NullifyPDF.py` | Main PySide6 UI, PDF rendering, AI scan orchestration, redaction, export, OCR integration | +| `privacy_core.py` | Privacy mode enum, placeholder registry, encrypted restore-map payloads | +| `PDF_Checker.py` | Heuristic post-export inspection helper | +| `build_local.py` | PyInstaller build script with Lite/Full release variants | +| `scripts/download_ocr_data.py` | Downloads EN/IT Tesseract `tessdata_fast` files for Full builds | +| `tests/` | Unit and smoke tests for validation, privacy primitives, OCR config, and build variants | -| Class | Purpose | Threading | -| ---------------- | ------------------------------- | ------------- | -| `NullifyPDFApp` | Main application window | UI Thread | -| `PDFListManager` | Blocklist/Allowlist persistence | Main | -| `AIWorker` | NLP scanning in background | Worker Thread | -| `PDFCanvas` | QGraphicsView for PDF rendering | UI Thread | +--- -**Key Methods:** +## 🧠 Runtime Architecture -```python -class NullifyPDFApp(QMainWindow): - def load_pdf(path: str) -> None - # Load PDF, render first page, enable UI controls - - def on_auto_redact(self) -> None - # Emit signal to start AI scan in worker thread - - def on_export_pdf(self) -> None - # Export with forensic scrubbing - - def handle_ai_results(results: dict) -> None - # Receive detections from worker, draw on canvas +```text +User + | + v +PySide6 UI (NullifyPDF) + | + |-- PDFView renders the current page with PyMuPDF pixmaps + |-- Manual redaction draws pending redaction annotations + |-- Export dialog selects anonymization or pseudonymization + | + v +AIWorker (QThread) + | + |-- Extracts digital text from pages + |-- Runs OCR on scanned/image-only pages when enabled + |-- Runs Presidio + spaCy entity detection + |-- Emits detections back to UI thread + | + v +UI thread applies pending redaction annotations + | + v +Export pipeline writes a cleaned PDF copy ``` -**Threading Model:** - -- **UI Thread:** Rendering, user interactions, dialog management -- **Worker Thread:** `AIWorker` runs NLP without blocking UI -- **Synchronization:** `QMutex` serializes PDF document access - -**Key Files:** -- `NullifyPDF.py` (β‰ˆ2500 lines) - --- -### 2️⃣ PDF Engine β€” PyMuPDF (fitz) - -**Purpose:** Load, render, manipulate, and export PDF documents. +## πŸ›οΈ Main Classes -**Operations:** +| 🧱 Class | πŸ“ Location | 🎯 Purpose | +| ------- | ----------- | ---------- | +| `PDFListManager` | `NullifyPDF.py` | Loads and saves blocklist/allowlist files in `~/.nullifypdf` | +| `AIWorker` | `NullifyPDF.py` | Background NLP/OCR worker that emits page-level detections | +| `PDFView` | `NullifyPDF.py` | Custom `QGraphicsView` for rendering pages, drawing rectangles, and zooming | +| `NullifyPDF` | `NullifyPDF.py` | Main window and application controller | +| `PlaceholderRegistry` | `privacy_core.py` | Creates stable placeholders such as `PERSON_001` and `EMAIL_ADDRESS_001` | -```python -import fitz +--- -# Load PDF -doc = fitz.open("document.pdf") +## πŸ”„ Data Flow -# Render page to image -page = doc[0] -pix = page.get_pixmap(matrix=fitz.Matrix(2, 2)) +### πŸ“‚ Load -# Extract text -text = page.get_text() +1. User opens or drops a PDF. +2. `NullifyPDF.load_path()` validates the path and extension. +3. PyMuPDF opens the document. +4. Password-protected PDFs are rejected. +5. The first page is rendered in `PDFView`. +6. `_warn_if_scanned_pdf()` warns when pages look image-only and may need OCR. -# Add annotation (redaction box) -rect = fitz.Rect(100, 100, 200, 150) # (x0, y0, x1, y1) -annot = page.add_redact_annot(rect) -page.apply_redactions() # Destructive! +### πŸ€– AI Scan -# Destroy metadata -doc.metadata = {} -doc.set_xml_metadata(None) +1. User selects language: `EN`, `IT`, or `BOTH`. +2. User optionally enables: + - `OCR PDF scansionati` + - `Oscura Immagini` +3. `cmd_auto_ai()` checks OCR configuration if OCR is enabled. +4. `AIWorker.run_scan()` processes pages in a background thread. +5. For each page: + - digital text is extracted with `page.get_text()` + - if the page looks scanned, OCR is run with `page.get_textpage_ocr()` + - Presidio analyzes the extracted/OCR text + - detected entities are emitted with text, entity type, source, and optional OCR rectangles +6. `apply_ai_to_page()` runs on the UI thread and creates pending redaction annotations. -# Export -doc.save("output.pdf", deflate=True) -``` +### πŸ–ŠοΈ Manual Review -**In NullifyPDF:** +Redactions remain annotations until export. The original in-memory document is not flattened while the user is reviewing it. -- **Load:** Read file, detect if encrypted/protected -- **Render:** Convert pages to QImage for display -- **Extract:** Get text for AI analysis (with QMutex lock) -- **Annotate:** Store redaction data (non-destructive, in-memory) -- **Export:** Apply redactions, scrub metadata, flatten forms +Manual drawing over text: -**Key Constraints:** +- adds a redaction annotation +- records the clipped text in annotation metadata when available +- adds the text to the blocklist -- βœ… PyMuPDF wheels are pre-compiled β†’ **requires Python 3.13** -- βœ… `doc.get_text()` is CPU-bound β†’ run in worker thread -- βœ… Metadata destruction is binary-level β†’ cannot undo after export -- βœ… Digital signatures invalidated after redaction +Clicking an existing redaction: -**See:** [PyMuPDF Documentation](https://pymupdf.io/) +- removes the pending annotation +- adds the clipped text to the allowlist when available --- -### 3️⃣ AI Pipeline β€” NLP Entity Detection - -**Purpose:** Identify Personally Identifiable Information (PII) in extracted text. - -**Components:** - -``` -Extracted Text - ↓ -β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” -β”‚ Microsoft Presidio Analyzer β”‚ -β”‚ (PII detection via regexes) β”‚ -β”‚ β€’ IBAN, Credit Cards, Emails β”‚ -β”‚ β€’ Phone numbers, URLs β”‚ -β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ - ↓ -β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” -β”‚ spaCy NLP Models (EN/IT) β”‚ -β”‚ (Named Entity Recognition) β”‚ -β”‚ β€’ PERSON, LOCATION, ORG β”‚ -β”‚ β€’ Custom entities (crypto address) β”‚ -β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ - ↓ -Merged Results β†’ Deduplicate β†’ Allowlist Filter - ↓ -Final Detections (coordinates + text) -``` +## πŸ›‘οΈ Privacy Export Pipeline + +`cmd_export()` is the main export entry point. + +1. User chooses privacy mode: + - irreversible anonymization + - reversible pseudonymization +2. User chooses output PDF path. +3. For pseudonymization only: + - user chooses `.nullifypdf-map` path + - user enters a password of at least 12 characters + - `cryptography` availability is checked before writing output +4. The in-memory PDF is saved to a sibling temp file: + - `.nullifypdf.tmp` +5. The temp PDF is reopened as `ex_doc`. +6. Each page is processed: + - overlapping links are deleted for redacted areas + - redactions are applied with PyMuPDF + - widgets are removed +7. Document metadata and selected catalog keys are cleared. +8. The final PDF is saved with cleanup options. +9. The temp file is removed. + +The original open document remains editable after export because the export works on a disk-backed copy. -**Models Used:** - -| Model | Language | Size | Purpose | -| ------------------- | -------- | ------ | ------------------ | -| `en_core_web_md` | English | 40 MB | General NER | -| `it_core_news_md` | Italian | 45 MB | General NER | -| `presidio-analyzer` | Multi | Custom | PII regex patterns | - -**User Language Selection:** - -- **EN only:** Use English spaCy + Presidio (fast) -- **IT only:** Use Italian spaCy + Presidio (fast) -- **BOTH:** Run both pipelines, merge results (slower, thorough) +--- -**Allowlist Integration:** +## πŸ”’ Anonymization Mode -```python -# AI detects: "John Smith", "john@example.com" -# Allowlist contains: "john" +Irreversible anonymization applies redaction annotations directly. The exported PDF does not contain a restore map. -# Pre-compiled allowlist regexes -if is_in_allowlist("john", allowlist_patterns): - skip_detection # Don't redact -else: - add_to_redactions -``` +This mode should be used when the output PDF must not contain recoverable personal data from the selected redaction areas. -**O(1) Fast-path Optimization:** - -```python -# Exact-match check first (fast) -if detected_text in allowlist_set: - skip # Common case (90%) -else: - # Regex matching fallback (slow) - for pattern in allowlist_regexes: - if pattern.match(detected_text): - skip -``` - -**See:** -- [Presidio Documentation](https://microsoft.github.io/presidio/) -- [spaCy Models](https://spacy.io/models) +> [!WARNING] +> Residual risk still exists for complex PDFs. NullifyPDF removes selected redacted content and common metadata, form, and link structures, but the output should still be reviewed before high-risk sharing. --- -### 4️⃣ Data Persistence β€” `PDFListManager` - -**Purpose:** Save and load Blocklist/Allowlist from disk. +## πŸ” Pseudonymization Mode -**Storage Location:** +Reversible pseudonymization replaces each selected value with a placeholder. -| OS | Path | -| --------------- | ---------------------------------- | -| **Windows** | `C:\Users\\.nullifypdf\` | -| **macOS/Linux** | `~/.nullifypdf/` | +Example placeholders: -**File Structure:** - -``` -~/.nullifypdf/ -β”œβ”€β”€ blocklist.txt (words to always redact) -β”œβ”€β”€ allowlist.txt (words to never redact) -β”œβ”€β”€ logs/ -β”‚ └── nullifypdf.log (rotating, max 5 MB per file) +```text +PERSON_001 +EMAIL_ADDRESS_001 +IBAN_CODE_001 +PHONE_NUMBER_001 +DATA_001 ``` -**Format:** - -- One word per line -- Lowercase, trimmed whitespace -- UTF-8 encoding -- Mutual exclusivity: word cannot be in both lists - -**Example:** +The mapping is stored in a separate encrypted file: ```text -# blocklist.txt -john smith -acme corp -admin@company.com - -# allowlist.txt -and -the -common +document.nullifypdf-map ``` -**Implementation:** - -```python -class PDFListManager: - def load_blocklist(self) -> Set[str]: - """Load from ~/.nullifypdf/blocklist.txt""" - - def save_allowlist(self, allowlist: Set[str]) -> None: - """Persist to ~/.nullifypdf/allowlist.txt""" - - # Mutual exclusivity check - if word in blocklist: - blocklist.remove(word) # Remove from block if in allow -``` +The restore map contains: ---- +- source filename +- source SHA-256 when available +- output SHA-256 +- placeholder entries +- original values +- entity types +- page numbers -### 5️⃣ Threading & Synchronization +The map is encrypted with `cryptography` using Fernet and a key derived from the user password with PBKDF2-HMAC-SHA256. -**Problem:** PDF document operations are CPU-bound. Running NLP analysis on UI thread freezes the GUI. +> [!IMPORTANT] +> The restore map is never embedded in the pseudonymized PDF. -**Solution:** Background worker thread with `QMutex` synchronization. +--- -**Architecture:** +## πŸ”Ž OCR Architecture -``` -β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” -β”‚ Main Qt Event Loop (UI Thread) β”‚ -β”‚ β”‚ -β”‚ β€’ Handle user clicks β”‚ -β”‚ β€’ Render PDF pages β”‚ -β”‚ β€’ Update progress bar β”‚ -β”‚ β”‚ -β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”‚ -β”‚ β”‚ QMutex Lock β”‚ β”‚ -β”‚ β”‚ (Access PDF document safely) β”‚ β”‚ -β”‚ β”‚ fitz.open().get_text() { β”‚ β”‚ -β”‚ β”‚ locked access only β”‚ β”‚ -β”‚ β”‚ } β”‚ β”‚ -β”‚ β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ -β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ - ↑ Signals - β”‚ -β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” -β”‚ AIWorker Thread (Background) β”‚ -β”‚ β”‚ -β”‚ 1. Lock QMutex β”‚ -β”‚ 2. Extract text: doc.get_text() β”‚ -β”‚ 3. Unlock QMutex β”‚ -β”‚ 4. Run NLP (no lock needed) β”‚ -β”‚ 5. Emit signal: results_ready.emit() β”‚ -β”‚ β”‚ -β”‚ Runs in: QThreadPool β”‚ -β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ -``` +OCR is implemented through PyMuPDF's integrated Tesseract support. -**Signal/Slot Pattern:** - -```python -class AIWorker(QObject): - results_ready = Signal(dict) # Emitted when scan done - progress_update = Signal(int) # Emitted for progress bar - - def run_scan(self) -> None: - # Acquire lock - with QMutexLocker(self.mutex): - text = self.pdf_doc.get_text() - - # Run AI (no lock, safe in worker thread) - detections = self.analyze_text(text) - - # Signal results back to UI - self.results_ready.emit(detections) - -# In main UI thread -worker.results_ready.connect(self.on_redactions_detected) -``` +Runtime behavior: -**Key Guarantees:** +1. `find_tessdata_dir()` resolves Tesseract language data in this order: + - `TESSDATA_PREFIX` + - bundled `ocr/tessdata` via `resource_path()` + - PyMuPDF's `fitz.get_tessdata()` + - common platform locations +2. `ocr_language_for_choice()` maps UI language to Tesseract codes: + - `EN` -> `eng` + - `IT` -> `ita` + - `BOTH` -> `eng+ita` +3. `AIWorker._page_needs_ocr()` runs OCR only when text extraction is missing or very poor. +4. OCR text is used for Presidio detection. +5. OCR search rectangles are used to place redactions on scanned pages. -βœ… UI never blocks -βœ… PDF document accessed safely (one thread at a time) -βœ… No race conditions -βœ… Progress updates smooth +The Full release bundles EN/IT OCR data. The Lite release requires local Tesseract `tessdata` when OCR is enabled. --- -### 6️⃣ Export Pipeline (Forensic Scrubbing) +## 🧡 Threading Model -**Problem:** In-memory PDF modification could use 2x RAM (original + modified copy). +The UI thread owns rendering and document mutation. -**Solution:** Disk-backed temporary file with lazy parsing. +The worker thread performs text extraction, OCR, and NLP analysis. Access to the shared PyMuPDF document is serialized with `QMutex`. -**Flow:** +Important rules: -``` -1. Load original PDF into memory - ↓ -2. Create temp file (disk-backed) - ↓ -3. For each page: - a. Copy page from original - b. Apply redaction annotations - c. Destroy metadata, links, AcroForms - d. Write to temp file - ↓ -4. Once complete, move temp β†’ final output - ↓ -5. Original + temp deleted -``` - -**Memory Efficiency:** - -```python -# OLD (Memory doubling) ❌ -doc = fitz.open("input.pdf") -doc.saveIncr() # Doubles RAM usage - -# NEW (Memory efficient) βœ… -doc = fitz.open("input.pdf") -temp = fitz.open() # Empty doc -for page_num in range(doc.page_count): - page = doc[page_num] - temp.insert_pdf(doc, page_num, page_num) - # Temp is smaller, only current page in buffer -temp.save("output.pdf") # Atomic write -``` - -**Destructive Operations:** +- `AIWorker` should not mutate page annotations +- `apply_ai_to_page()` runs on the UI thread +- export happens after detection and review, not during the worker scan -| Operation | Effect | Reversible? | -| ------------------------ | -------------------------------- | --------------------- | -| **Redaction overlay** | Covers text with black box | No (binary destroyed) | -| **Metadata removal** | Strips /Info, /CreationDate | No | -| **Link destruction** | Removes /Annot array entries | No | -| **AcroForms flattening** | Disables interactive fields | No | -| **Digital signature** | Becomes invalid (binary changed) | No | +This separation prevents UI freezes and reduces concurrent document-mutation risks. --- -## πŸ“Š Logging & Diagnostics - -### Log Locations +## πŸ“¦ Build Variants -| OS | Path | -| --------------- | ----------------------------------------------------- | -| **Windows** | `C:\Users\\.nullifypdf\logs\nullifypdf.log` | -| **macOS/Linux** | `~/.nullifypdf/logs/nullifypdf.log` | +### πŸͺΆ Lite -### Log Format +Lite builds do not bundle OCR language data. They are smaller and still support OCR if the user has Tesseract `tessdata` configured locally. -``` -2026-06-06 14:23:45 - INFO - Loaded PDF: /path/to/document.pdf -2026-06-06 14:23:46 - DEBUG - Page count: 10 -2026-06-06 14:23:50 - INFO - AI scan complete: 5 detections found -2026-06-06 14:24:10 - ERROR - Export failed: Permission denied -2026-06-06 14:24:10 - DEBUG - Traceback: ... -``` - -### Debug Mode +### 🧰 Full -Enable verbose logging via environment variable: +Full builds bundle: -```bash -# Windows (PowerShell) -$env:NULLIFYPDF_DEBUG = "true" -python NullifyPDF.py - -# macOS/Linux (Bash) -export NULLIFYPDF_DEBUG=true -python3.13 NullifyPDF.py +```text +ocr/tessdata/eng.traineddata +ocr/tessdata/ita.traineddata ``` -**Effect:** Log level changes from `INFO` to `DEBUG`, includes full stack traces. +If the files are missing, `build_local.py --full` automatically downloads them from `tesseract-ocr/tessdata_fast` through `scripts/download_ocr_data.py`. -### Log Rotation - -- Max file size: 5 MB -- Backup count: 3 (old logs auto-deleted) -- Encoding: UTF-8 +The `.traineddata` files are ignored by git and are not stored in the repository. --- -## πŸ› οΈ Utility Scripts - -### `setup_env.py` β€” Environment Configuration - -**Purpose:** Prepare development environment on any OS. - -**Steps:** - -1. Detect OS (Windows/macOS/Linux) -2. Create virtual environment with Python 3.13 -3. Upgrade pip, setuptools, wheel -4. Install requirements.txt dependencies -5. Download spaCy models (EN + IT) -6. Run smoke tests - -**Dependencies Added:** - -- `pyside6` β€” GUI framework -- `pymupdf` β€” PDF manipulation -- `presidio-analyzer` β€” PII detection -- `spacy` β€” NLP engine -- `pytest` β€” Testing framework - ---- - -### `build_local.py` β€” Executable Compilation - -**Purpose:** Generate standalone executable for distribution. - -**Process:** - -1. Clean temporary directories (`build/`, `dist/`) -2. Detect OS (Windows/macOS/Linux) -3. Read version from `NullifyPDF.py` (`__version__`) -4. Run PyInstaller with OS-specific settings -5. Rename executable: `NullifyPDF_vX.Y.Z_Windows.exe` +## πŸš€ CI/CD -**Output Artifacts:** +GitHub Actions build both variants on each supported OS: -- **Windows:** `.exe` executable -- **macOS:** `.app` bundle (signed, if certificate available) -- **Linux:** Binary + optional `.deb`, `.rpm` packages +- Windows Lite / Full +- macOS Lite / Full +- Ubuntu Lite / Full +- Fedora Lite / Full -**PyInstaller Configuration:** +Beta releases build and upload all artifacts. Stable releases promote the latest verified beta artifacts without recompilation. -```python -# Bundles: -# - NullifyPDF.py -# - All .py dependencies -# - spaCy models -# - Resource files (images/) +Expected artifact names include: -# Excludes (to reduce size): -# - Development tools -# - Test files -# - Docs +```text +NullifyPDF_vX.Y.Z_Windows_Lite.exe +NullifyPDF_vX.Y.Z_Windows_Full.exe +NullifyPDF_vX.Y.Z_macOS_Lite.zip +NullifyPDF_vX.Y.Z_macOS_Full.zip +NullifyPDF_vX.Y.Z_Ubuntu_Lite.deb +NullifyPDF_vX.Y.Z_Ubuntu_Full.deb +NullifyPDF_vX.Y.Z_Fedora_Lite.rpm +NullifyPDF_vX.Y.Z_Fedora_Full.rpm ``` --- -### `PDF_Checker.py` β€” Post-Processing Analysis - -**Purpose:** Verify exported PDF for successful redaction. +## πŸ§ͺ Tests -**Features:** +Current tests cover: -- Scan for remaining text under redaction boxes -- Check metadata was removed -- Verify links destroyed -- Confirm AcroForms flattened +- list persistence +- resource path resolution +- OCR configuration helpers +- privacy placeholder registry +- encrypted restore-map round trip +- build variant selection +- Full build OCR data inclusion behavior -**Usage:** +Run: ```bash -python PDF_Checker.py output.pdf -``` - ---- - -## πŸ“ File Structure - -``` -NullifyPDF/ -β”œβ”€β”€ NullifyPDF.py ← Main application (GUI, logic) -β”œβ”€β”€ PDF_Checker.py ← Utility for verification -β”œβ”€β”€ setup_env.py ← Environment setup -β”œβ”€β”€ build_local.py ← Build executable -β”œβ”€β”€ requirements.txt ← Python dependencies -β”œβ”€β”€ README.md ← English overview -β”œβ”€β”€ README_IT.md ← Italian version -β”œβ”€β”€ USER_GUIDE.md ← Usage instructions -β”œβ”€β”€ CONTRIBUTING.md ← Contributor guidelines -β”œβ”€β”€ ARCHITECTURE.md ← This file -β”œβ”€β”€ CHANGELOG.md ← Release history -β”œβ”€β”€ LICENSE ← MIT License -β”œβ”€β”€ images/ -β”‚ └── NullifyPDF.png ← App logo -β”œβ”€β”€ tests/ -β”‚ β”œβ”€β”€ test_pdf_manager.py ← PDFListManager tests -β”‚ β”œβ”€β”€ test_validation.py ← Input validation tests -β”‚ └── ... -└── .github/ - └── workflows/ - └── release.yml ← CI/CD automation +pytest tests/ -v ``` --- -## πŸ” Security Considerations - -### What's Protected - -βœ… **Data Destruction** β€” Binary-level scrubbing, not recoverable -βœ… **Offline Processing** β€” No network calls (except GitHub release checks) -βœ… **Input Validation** β€” Path traversal, type checking, bounds -βœ… **Resource Limits** β€” Graceful handling of large PDFs - -### What's NOT in Scope - -❌ **OCR** β€” Scanned images not analyzed -❌ **Handwriting** β€” Not digitized text -❌ **Password Cracking** β€” Encrypted PDFs rejected at load -❌ **Signature Forgery** β€” Signatures invalidated, not forged - ---- - -## πŸš€ Performance Characteristics - -### Typical Operations - -| Operation | Time | Scaling | -| ------------------------ | ------------- | ------------------------- | -| Load PDF (100 pages) | 500 ms | O(n pages) | -| Extract text (100 pages) | 2-5 seconds | O(n pages) | -| AI scan (100 pages) | 10-30 seconds | O(n pages Γ— n words) | -| Export with redaction | 15-60 seconds | O(n pages Γ— n redactions) | - -### Memory Usage - -| Scenario | RAM | Optimization | -| ------------------------ | ---------- | ------------------ | -| Small PDF (10 pages) | 50-100 MB | Efficient | -| Large PDF (500 pages) | 200-500 MB | Disk-backed export | -| Very large (1000+ pages) | May swap | Consider splitting | - -### Optimization Opportunities - -1. **Allowlist caching** β€” Pre-compile regexes (done βœ…) -2. **Text extraction parallelism** β€” Multiple pages at once (future) -3. **GPU acceleration** β€” For spaCy models (future) -4. **Lazy loading** β€” Only load pages user views (future) - ---- - -## πŸ”„ Development Workflow - -### Adding a New Feature - -1. **Create branch:** `git checkout -b feature/your-feature develop` -2. **Write code:** Follow standards in [CONTRIBUTING.md](./CONTRIBUTING.md) -3. **Add tests:** Cover happy path + edge cases -4. **Run smoke tests:** `pytest tests/ -v` -5. **Build locally:** `python build_local.py` -6. **Update CHANGELOG.md:** Document changes -7. **Open PR:** Describe what + why - -### Debugging Tips - -1. **Enable debug mode:** - ```bash - export NULLIFYPDF_DEBUG=true - python3.13 NullifyPDF.py - ``` - -2. **Check logs:** `~/.nullifypdf/logs/nullifypdf.log` - -3. **Add print statements:** - ```python - logger.debug(f"Variable value: {var}") # Use logging, not print() - ``` - -4. **Test in isolation:** - ```bash - pytest tests/test_specific.py::test_function -v - ``` - ---- - -## πŸ“š External Resources - -| Resource | Purpose | -| --------------------------------------------------- | ---------------------- | -| [PyMuPDF Docs](https://pymupdf.io/) | PDF manipulation API | -| [PySide6 Docs](https://doc.qt.io/qtforpython/) | GUI framework | -| [spaCy Models](https://spacy.io/models) | NLP entity recognition | -| [Presidio](https://microsoft.github.io/presidio/) | PII detection patterns | -| [Qt Threading](https://doc.qt.io/qt-6/qthread.html) | Thread synchronization | - ---- +## ⚠️ Known Limits -## ❓ Questions? +- Detection is probabilistic. Presidio, spaCy, and OCR can miss data or create false positives. +- OCR depends on Tesseract language data and scan quality. +- Handwriting is not reliably supported. +- Complex PDFs may contain structures not covered by automated cleanup. +- Pseudonymized PDFs are still personal-data processing when the restore map exists. +- Digital signatures are invalidated by privacy export. -- πŸ’¬ See [CONTRIBUTING.md](./CONTRIBUTING.md) for contact info -- πŸ› Report issues on [GitHub Issues](https://github.com/overwrite00/NullifyPDF/issues) -- πŸ“– Read [USER_GUIDE.md](./USER_GUIDE.md) for usage questions +> [!WARNING] +> For high-risk workflows, exported PDFs should be reviewed independently before sharing. --- -*Last updated: 2026-07-14* -*← [Back to README](./README.md) | [Contributing β†’](./CONTRIBUTING.md)* +*Last updated: 2026-07-23* +*[Back to README](./README.md) | [Development β†’](./DEVELOPMENT.md)* diff --git a/CHANGELOG.md b/CHANGELOG.md index b5b26d0..317a846 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,31 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [2.1.0-beta.1] - 2026-07-23 + +Prima beta pubblica della linea 2.1.0, focalizzata su privacy export avanzato, OCR completo per PDF scansionati e nuove varianti di release Lite/Full. + +### ✨ Added + +- **Privacy Export Modes**: The export flow now lets the user choose between irreversible anonymization and reversible pseudonymization before writing the output PDF. +- **Encrypted Restore Maps**: Pseudonymized exports can generate a separate encrypted `.nullifypdf-map` file with placeholders, original values, entity types, page numbers, and document hashes. +- **OCR for Scanned PDFs**: Added OCR-assisted entity detection for scanned or image-only PDFs using PyMuPDF OCR and Tesseract language data resolution. +- **Lite/Full Release Variants**: Build and release pipelines now produce Lite artifacts and Full artifacts with bundled EN/IT OCR data. +- **Automatic OCR Data Download for Full Builds**: Local Full builds download missing `eng.traineddata` and `ita.traineddata` automatically with a clear message. +- **Privacy Core Module**: Added `privacy_core.py` to centralize placeholder generation and encrypted restore-payload handling. + +### πŸ”’ Security + +- **Separated Re-identification Data**: Restore maps are never embedded into exported PDFs and are encrypted with Fernet using a PBKDF2-HMAC-SHA256 derived key. +- **Safer Export Metadata Handling**: Privacy export now clears document metadata and removes common residual structures such as widgets and overlapping links in redacted areas. +- **Safer UI Logging**: Log messages shown in the UI are HTML-escaped before rendering. + +### πŸ”§ Internals + +- **Expanded Test Coverage**: Added dedicated tests for privacy primitives, build variant behavior, OCR helper resolution, and prerelease version handling. +- **Documentation Refresh**: Updated README, ARCHITECTURE, USER_GUIDE, SECURITY, TROUBLESHOOTING, CONTRIBUTING, DEVELOPMENT, and added `OCR_SETUP.md` for the new privacy and OCR workflows. +- **Release Automation Updates**: GitHub workflows now build Lite/Full variants across supported operating systems and support the beta-to-stable promotion flow. + ## [2.0.7] - 2026-07-21 ### ✨ Added @@ -45,7 +70,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - **Complete Type Hints**: Added type hints to 100% of functions and methods to improve static analysis and IDE support. - **Google-Style Docstrings**: Introduced full documentation with Args, Returns, Raises for easier maintenance. -- **File-based Logging**: Added file logging (`~/.nullifypdf/logs/nullifypdf.log`) with automatic rotation and debug mode support via `NULLIFYPDF_DEBUG=true`. +- **File-based Logging**: Added file logging (`~/.nullifypdf/logs/nullifypdf.log`) with debug mode support via `NULLIFYPDF_DEBUG=true`. - **Input Validation**: Added explicit validation for paths, page ranges, and language choices to avoid crashes on malformed input. ### πŸ› Fixed (Bug Fixes) @@ -61,7 +86,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### ⚑ Optimized (Performance) -- **Memory Doubling in Export**: Implemented a disk-backed temp file with lazy-parsing via PyMuPDF to reduce peak RAM from 2x to 1x document size during forensic scrubbing. +- **Memory Doubling in Export**: Implemented a disk-backed temp file with lazy-parsing via PyMuPDF to reduce peak RAM from 2x to 1x document size during privacy export. - **Regex Recompilation**: Precompiled regex patterns outside loops in `AIWorker.run_scan()` for a 10-100x speedup on large allowlists. - **Allowlist Fast-path**: Added an O(1) exact-match check against a set before the regex `any()` scan, short-circuiting 90% of common cases. - **AI Scan Responsiveness**: Moved `get_text()` off the UI thread into the `AIWorker` thread pool with QMutex serialization to avoid UI freezing on large PDFs. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 6796339..96f2888 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -57,7 +57,7 @@ source .venv/bin/activate # or .venv\Scripts\Activate.ps1 on Windows pytest tests/ -v # Build to verify compilation -python build_local.py +python build_local.py --lite ``` ### 5️⃣ Commit with Clear Messages @@ -88,8 +88,8 @@ Then open Pull Request on GitHub with description from template. ### ❌ Not Currently Accepting ❌ **Major Architecture Changes** β€” Discuss first via issue -❌ **New Dependencies** β€” Heavy libraries (keep slim & local) -❌ **OCR Implementation** β€” Out of scope (use Blindfold Mode instead) +❌ **New Heavy Dependencies** β€” Discuss first; keep the app local and maintainable +βœ… **OCR Improvements** β€” In scope when they stay local/offline and preserve the Lite/Full build split ❌ **Cloud Features** β€” Must remain 100% local > [!TIP] @@ -197,7 +197,7 @@ Body with more details (optional) git commit -m "feat(ai): add IBAN detection to Presidio pipeline" # Bug fix -git commit -m "fix(export): resolve memory doubling in forensic scrubbing" +git commit -m "fix(export): reduce memory usage in privacy export" # Documentation git commit -m "docs: update installation guide for Python 3.13" @@ -219,7 +219,7 @@ thread when scanning large PDFs concurrently." ### Before Opening PR 1. βœ… Run tests: `pytest tests/ -v` -2. βœ… Build locally: `python build_local.py` +2. βœ… Build locally: `python build_local.py --lite` and, when OCR packaging changes, `python build_local.py --full` 3. βœ… Update CHANGELOG.md with your changes 4. βœ… Self-review your code (would you understand this in 6 months?) 5. βœ… Check links in documentation @@ -378,5 +378,5 @@ Every contribution β€” code, docs, bug reports, ideas β€” helps make NullifyPDF --- -*Last updated: 2026-07-14* +*Last updated: 2026-07-23* *← [Back to README](./README.md) | [Architecture β†’](./ARCHITECTURE.md)* diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 3c6cb08..07a53ec 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -52,7 +52,6 @@ This automatically: - βœ… Creates `.venv/` virtual environment - βœ… Installs dependencies - βœ… Downloads spaCy models (EN + IT) -- βœ… Runs smoke tests ### 4️⃣ Activate Virtual Environment @@ -105,20 +104,25 @@ If the GUI opens β†’ **You're ready to develop!** πŸŽ‰ ``` NullifyPDF/ -β”œβ”€β”€ NullifyPDF.py ← Main application (β‰ˆ2500 lines) -β”œβ”€β”€ PDF_Checker.py ← Verification utility -β”œβ”€β”€ setup_env.py ← Environment setup script -β”œβ”€β”€ build_local.py ← Build executable script -β”œβ”€β”€ requirements.txt ← Python dependencies -β”œβ”€β”€ tests/ -β”‚ β”œβ”€β”€ test_pdf_manager.py ← PDFListManager tests -β”‚ β”œβ”€β”€ test_validation.py ← Input validation tests -β”‚ └── ... -β”œβ”€β”€ images/ -β”‚ └── NullifyPDF.png ← App logo -└── .github/ - └── workflows/ - └── release.yml ← CI/CD automation +|-- NullifyPDF.py # Main PySide6 application +|-- privacy_core.py # Privacy modes and encrypted restore maps +|-- PDF_Checker.py # Heuristic verification utility +|-- setup_env.py # Environment setup script +|-- build_local.py # Lite/Full PyInstaller build script +|-- requirements.txt # Python dependencies +|-- scripts/ +| `-- download_ocr_data.py # EN/IT OCR data downloader for Full builds +|-- tests/ +| |-- test_validation.py +| |-- test_privacy_core.py +| `-- test_build_config.py +|-- images/ +| `-- NullifyPDF.png +`-- .github/ + `-- workflows/ + |-- test_build.yml + |-- beta-release.yml + `-- release.yml ``` --- @@ -129,11 +133,12 @@ NullifyPDF/ | Package | Version | Purpose | | --------------------- | ------- | ---------------------------- | -| **pyside6** | ^6.x | GUI framework (Qt6 bindings) | -| **pymupdf** | ^1.24.x | PDF manipulation | -| **presidio-analyzer** | ^2.2.x | PII detection (regex-based) | -| **spacy** | ^3.7.x | NLP for entity recognition | -| **pytest** | ^8.x | Testing framework | +| **PySide6** | 6.11.1 | GUI framework (Qt6 bindings) | +| **PyMuPDF** | 1.28.0 | PDF manipulation and OCR bridge | +| **presidio-analyzer** | 2.2.363 | PII detection | +| **spaCy** | 3.8.13 | NLP for entity recognition | +| **cryptography** | 45.0.7 | Encrypted restore maps | +| **pytest** | 9.1.1 | Testing framework | ### Language Models (Auto-Downloaded) @@ -153,27 +158,28 @@ These are downloaded automatically by `setup_env.py`. **Class Hierarchy:** ```python -NullifyPDFApp(QMainWindow) -β”œβ”€β”€ PDFListManager # Manages blocklist/allowlist -β”œβ”€β”€ AIWorker(QObject) # NLP scanning in thread -β”œβ”€β”€ PDFCanvas(QGraphicsView) # PDF rendering -└── UI Components - β”œβ”€β”€ Sidebar - β”œβ”€β”€ Toolbar - β”œβ”€β”€ Progress bar - β”œβ”€β”€ Dialogs +NullifyPDF(QMainWindow) +|-- PDFListManager # Manages blocklist/allowlist +|-- AIWorker(QObject) # NLP/OCR scanning in thread +|-- PDFView(QGraphicsView) # PDF rendering and rectangle drawing +|-- privacy_core.py # Privacy modes and encrypted restore maps +`-- UI Components + |-- Sidebar + |-- Toolbar + |-- Progress bar + `-- Dialogs ``` **Key Methods:** ```python -class NullifyPDFApp: +class NullifyPDF: def __init__(self) # Initialize GUI - def load_pdf(self, path: str) # Load PDF file - def on_auto_redact(self) # Start AI scan - def handle_ai_results(self, data) # Receive AI results - def on_export_pdf(self) # Export with redactions - def add_redaction_box(self, rect) # Draw on canvas + def load_path(self, path: str) # Load PDF file + def cmd_auto_ai(self) # Start AI/OCR scan + def apply_ai_to_page(self, i, data) # Receive AI results + def cmd_export(self) # Export privacy PDF + def user_draw_rect(self, rect) # Draw manual redaction ``` ### Threading Model @@ -204,7 +210,7 @@ pytest tests/ -v ### Run Specific Test ```bash -pytest tests/test_pdf_manager.py::test_load_blocklist -v +pytest tests/test_validation.py::TestPDFListManager::test_save_and_load_blocklist -v ``` ### Test Coverage @@ -218,7 +224,9 @@ Opens `htmlcov/index.html` in browser. ### What's Tested - βœ… **PDFListManager** β€” File I/O, persistence -- βœ… **Input Validation** β€” Path handling, type checking +- βœ… **OCR Config** β€” Tesseract language selection and tessdata discovery +- βœ… **Privacy Core** β€” Placeholder mapping and encrypted restore maps +- βœ… **Build Config** β€” Lite/Full build variant behavior - βœ… **Resource Paths** β€” PyInstaller compatibility --- @@ -228,10 +236,11 @@ Opens `htmlcov/index.html` in browser. ### Quick Build ```bash -python build_local.py +python build_local.py --lite +python build_local.py --full ``` -**Output:** `dist/NullifyPDF_vX.Y.Z_Windows.exe` (on Windows) +**Output:** `dist/NullifyPDF_vX.Y.Z_Windows_Lite.exe` or `dist/NullifyPDF_vX.Y.Z_Windows_Full.exe` (on Windows) ### What It Does @@ -239,15 +248,15 @@ python build_local.py 2. Detects your OS (Windows/macOS/Linux) 3. Reads version from `NullifyPDF.py` (`__version__`) 4. Compiles with PyInstaller -5. Renames with version: `NullifyPDF_v{VERSION}_{OS}.exe` +5. Renames with version and variant: `NullifyPDF_v{VERSION}_{OS}_{Lite|Full}.exe` ### Distribution Artifacts | OS | Output | | ----------- | --------------------------------- | -| **Windows** | `.exe` executable | -| **macOS** | `.app` bundle | -| **Linux** | Binary + `.deb` + `.rpm` packages | +| **Windows** | Lite/Full `.exe` executables | +| **macOS** | Lite/Full `.app` bundle ZIPs | +| **Linux** | Lite/Full binary + `.deb` + `.rpm` packages | ### Troubleshooting Build Issues @@ -270,7 +279,7 @@ sys.setrecursionlimit(5000) Build succeeds but executable won't run 1. Check antivirus isn't blocking -2. Run in debug mode: `NullifyPDF_vX.Y.Z_Windows.exe` from PowerShell +2. Run in debug mode: `NullifyPDF_vX.Y.Z_Windows_Lite.exe` or `NullifyPDF_vX.Y.Z_Windows_Full.exe` from PowerShell 3. Check `.stdout` file if created 4. Report on GitHub @@ -296,7 +305,7 @@ git checkout -b feature/my-feature # 4. Test pytest tests/ -v -python build_local.py +python build_local.py --lite # 5. Commit with clear message git commit -m "feat(ai): add IBAN detection" @@ -318,7 +327,7 @@ Optional longer explanation **Examples:** ```bash feat(ai): add cryptocurrency address detection -fix(export): reduce memory usage in forensic scrubbing +fix(export): reduce memory usage in privacy export docs: update installation guide perf(allowlist): implement O(1) fast-path lookup ``` @@ -423,23 +432,24 @@ isort NullifyPDF.py ## πŸ“š Key Files to Know -| File | Purpose | Lines | -| ------------------ | ----------------------- | ----- | -| `NullifyPDF.py` | Main app, GUI, logic | 2500+ | -| `setup_env.py` | Environment setup | 150 | -| `build_local.py` | PyInstaller build | 200 | -| `PDF_Checker.py` | Post-processing utility | 300+ | -| `requirements.txt` | Dependencies | 10 | -| `tests/` | Unit tests | 500+ | +| File | Purpose | +| ------------------ | ----------------------- | +| `NullifyPDF.py` | Main app, GUI, OCR, export logic | +| `privacy_core.py` | Placeholder and restore-map logic | +| `setup_env.py` | Environment setup | +| `build_local.py` | PyInstaller Lite/Full build | +| `PDF_Checker.py` | Post-processing utility | +| `requirements.txt` | Dependencies | +| `tests/` | Unit and smoke tests | ### Quick Edit Locations | Feature | File | Method | | ------------------- | --------------- | --------------------- | -| Load PDF | `NullifyPDF.py` | `load_pdf()` | -| Auto Redact | `NullifyPDF.py` | `on_auto_redact()` | +| Load PDF | `NullifyPDF.py` | `load_path()` | +| Auto Redact | `NullifyPDF.py` | `cmd_auto_ai()` | | AI Processing | `NullifyPDF.py` | `AIWorker.run_scan()` | -| Export | `NullifyPDF.py` | `on_export_pdf()` | +| Export | `NullifyPDF.py` | `cmd_export()` | | Blocklist/Allowlist | `NullifyPDF.py` | `PDFListManager` | --- @@ -467,8 +477,8 @@ Never hardcode API keys or passwords. ### Resource Limits -- Cap PDF page count for reasonable memory usage -- Timeout long-running operations +- Avoid unbounded memory growth on large PDFs +- Keep long-running work off the UI thread - Clean up temp files --- @@ -494,7 +504,7 @@ Never hardcode API keys or passwords. 2. Edit code following code standards 3. Add tests: `pytest tests/test_my_feature.py` 4. Run full test suite: `pytest tests/ -v` -5. Build locally: `python build_local.py` +5. Build locally: `python build_local.py --lite` 6. Commit and push ### Q: How do I test on different OS? @@ -550,5 +560,5 @@ Ready to contribute? See [CONTRIBUTING.md](./CONTRIBUTING.md) for: --- -*Last updated: 2026-07-14* +*Last updated: 2026-07-23* *← [Troubleshooting](./TROUBLESHOOTING.md) | [Back to README β†’](./README.md)* diff --git a/NullifyPDF.py b/NullifyPDF.py index 2d1348b..8844136 100644 --- a/NullifyPDF.py +++ b/NullifyPDF.py @@ -9,6 +9,10 @@ import logging import traceback import atexit +import hashlib +import html +import importlib.util +import json from typing import Optional, List, Set, Dict, Any, Tuple import fitz from PySide6.QtWidgets import ( @@ -31,6 +35,7 @@ QButtonGroup, QGraphicsRectItem, QMessageBox, + QInputDialog, ) from PySide6.QtGui import ( QIcon, @@ -44,11 +49,24 @@ ) from PySide6.QtCore import Qt, QThread, QObject, Signal, Slot, QRectF, QPointF, QMutex, QMutexLocker -__version__ = "2.0.7" +from privacy_core import ( + PlaceholderRegistry, + PrivacyMode, + build_restore_payload, + encrypt_restore_payload, +) + +__version__ = "2.1.0" +__version_prerelease__ = "beta.1" +APP_VERSION = ( + f"{__version__}-{__version_prerelease__}" + if __version_prerelease__ + else __version__ +) def setup_logging() -> logging.Logger: - """Configure file-based logging with rotation. + """Configure file-based logging. Respects NULLIFYPDF_DEBUG environment variable for verbose output. @@ -99,6 +117,56 @@ def resource_path(relative_path: str) -> str: return os.path.join(base_path, relative_path) +def sha256_file(path: str) -> str: + """Return the SHA-256 digest for a local file.""" + digest = hashlib.sha256() + with open(path, "rb") as fh: + for chunk in iter(lambda: fh.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def find_tessdata_dir() -> Optional[str]: + """Find a local Tesseract tessdata directory for PyMuPDF OCR.""" + candidates = [] + env_value = os.environ.get("TESSDATA_PREFIX") + if env_value: + candidates.append(env_value) + try: + candidates.append(fitz.get_tessdata()) + except Exception: + pass + candidates.append(resource_path(os.path.join("ocr", "tessdata"))) + candidates.extend( + [ + r"C:\Program Files\Tesseract-OCR\tessdata", + r"C:\Program Files (x86)\Tesseract-OCR\tessdata", + "/usr/share/tesseract-ocr/4.00/tessdata", + "/usr/share/tesseract-ocr/5/tessdata", + "/usr/share/tessdata", + ] + ) + for candidate in candidates: + if not candidate: + continue + path = pathlib.Path(candidate) + if path.is_dir() and any(path.glob("*.traineddata")): + return str(path) + return None + + +def ocr_language_for_choice(choice: str, tessdata_dir: Optional[str]) -> str: + """Map the UI language choice to installed Tesseract language codes.""" + requested = ["eng", "ita"] if choice == "BOTH" else ["ita" if choice == "IT" else "eng"] + if not tessdata_dir: + return "+".join(requested) + available = [ + lang for lang in requested + if (pathlib.Path(tessdata_dir) / f"{lang}.traineddata").exists() + ] + return "+".join(available or requested) + + class PDFListManager: """Manages blocklist/allowlist persistence to disk. @@ -219,13 +287,13 @@ class AIWorker(QObject): Signals: log_sig: Emits log message (str). progress_sig: Emits (current_page, total_pages) progress. - page_done_sig: Emits (page_index, found_words) when page scan completes. + page_done_sig: Emits (page_index, detections) when page scan completes. finished_sig: Emitted when all pages processed. """ log_sig = Signal(str) progress_sig = Signal(int, int) - page_done_sig = Signal(int, set) + page_done_sig = Signal(int, object) finished_sig = Signal() def __init__(self) -> None: @@ -251,10 +319,11 @@ def cleanup(self) -> None: except Exception as e: logging.getLogger("nullifypdf").debug(f"Error during AI cleanup: {e}") - @Slot(object, object, str, list, set) + @Slot(object, object, str, list, set, bool, object) def run_scan(self, doc: Any, doc_mutex: Any, choice: str, compiled_allowlist: List[Tuple[str, Any]], - allowlist_set: Set[str]) -> None: + allowlist_set: Set[str], use_ocr: bool, + tessdata_dir: Optional[str]) -> None: """Run AI scan on PDF pages and emit detected sensitive entities. Text extraction (`page.get_text()`) is performed inside this worker so @@ -295,6 +364,7 @@ def run_scan(self, doc: Any, doc_mutex: Any, choice: str, allowlist_set = set() target_langs = ["en", "it"] if choice == "BOTH" else [choice.lower()] + ocr_language = ocr_language_for_choice(choice, tessdata_dir) if not self.analyzer or sorted(self.loaded_langs) != sorted(target_langs): self.log_sig.emit(f"Inizializzazione AI ({choice})...") from presidio_analyzer import AnalyzerEngine @@ -315,7 +385,7 @@ def run_scan(self, doc: Any, doc_mutex: Any, choice: str, supported_languages=target_langs, ) self.loaded_langs = target_langs - self.log_sig.emit("Scansione forense in corso...") + self.log_sig.emit("Scansione privacy in corso...") targets = [ "PERSON", "LOCATION", @@ -336,13 +406,29 @@ def run_scan(self, doc: Any, doc_mutex: Any, choice: str, break # Extract text in worker thread (off the UI thread) under mutex # because UI thread renders / mutates annotations on the same doc. + used_ocr = False + textpage = None with QMutexLocker(doc_mutex): try: - text = doc[i].get_text() + page = doc[i] + text = page.get_text() + needs_ocr = use_ocr and self._page_needs_ocr(page, text) + if needs_ocr: + self.log_sig.emit(f"OCR pagina {i + 1}...") + textpage = page.get_textpage_ocr( + language=ocr_language, + dpi=300, + full=True, + tessdata=tessdata_dir, + ) + text = page.get_text(textpage=textpage) + used_ocr = True except Exception as e: - self.log_sig.emit(f"Avviso: get_text fallito pagina {i+1}: {e}") + self.log_sig.emit( + f"Avviso: estrazione testo/OCR fallita pagina {i+1}: {e}" + ) text = "" - found = set() + found: Dict[str, str] = {} for lang in self.loaded_langs: res = self.analyzer.analyze( text=text, entities=targets, language=lang @@ -350,9 +436,9 @@ def run_scan(self, doc: Any, doc_mutex: Any, choice: str, for r in res: w = text[r.start : r.end].strip() if len(w) > 2: - found.add(w) - final_words = set() - for m in found: + found[w] = r.entity_type + detections = [] + for m, entity_type in found.items(): clean = " ".join(m.strip(string.punctuation).lower().split()) if not clean: continue @@ -369,8 +455,26 @@ def run_scan(self, doc: Any, doc_mutex: Any, choice: str, f_reg.search(clean) or clean_pattern.search(a_str) for a_str, f_reg in compiled_allowlist ): - final_words.add(m) - self.page_done_sig.emit(i, final_words) + detection = { + "text": m, + "entity_type": entity_type, + "rects": [], + "source": "ocr" if used_ocr else "text", + } + if used_ocr and textpage is not None: + with QMutexLocker(doc_mutex): + try: + rects = doc[i].search_for(m, textpage=textpage) + detection["rects"] = [ + (r.x0, r.y0, r.x1, r.y1) for r in rects + ] + except Exception as e: + self.log_sig.emit( + f"Avviso: coordinate OCR non disponibili " + f"pagina {i+1}: {e}" + ) + detections.append(detection) + self.page_done_sig.emit(i, detections) self.progress_sig.emit(i + 1, total_pages) if not self._stop_requested: self.log_sig.emit("Anonimizzazione completata.") @@ -383,6 +487,18 @@ def run_scan(self, doc: Any, doc_mutex: Any, choice: str, finally: self.finished_sig.emit() + def _page_needs_ocr(self, page: Any, extracted_text: str) -> bool: + """Return True when a page looks scanned or text extraction is too poor.""" + text = (extracted_text or "").strip() + if len(text) >= 20 and text.count("\ufffd") / max(len(text), 1) < 0.05: + return False + try: + if page.get_image_info(hashes=False): + return True + except Exception: + return len(text) < 20 + return len(text) < 20 + class PDFView(QGraphicsView): """Custom graphics view for interactive PDF page display. @@ -449,10 +565,10 @@ def wheelEvent(self, event: Any) -> None: class NullifyPDF(QMainWindow): - """Main application window for PDF redaction using AI. + """Main application window for PDF privacy redaction using AI. Handles PDF loading, manual/AI-assisted redaction, blocklist/allowlist - management, and secure export with forensic scrubbing. + management, and privacy-focused export. Attributes: doc: Currently loaded PDF document (None if not loaded). @@ -463,14 +579,14 @@ class NullifyPDF(QMainWindow): config_dir: Path to user config directory (~/.nullifypdf). """ - start_scan_sig = Signal(object, object, str, list, set) + start_scan_sig = Signal(object, object, str, list, set, bool, object) def __init__(self) -> None: """Initialize main application window and setup UI.""" super().__init__() self.logger = setup_logging() self.logger.info("Application started") - self.setWindowTitle("NullifyPDF - AI Forensic Edition") + self.setWindowTitle("NullifyPDF - Privacy Edition") self.resize(1350, 950) self.setStyleSheet(STYLESHEET) self.setAcceptDrops(True) @@ -593,7 +709,7 @@ def build_ui(self) -> None: lbl_title = QLabel("NullifyPDF") lbl_title.setStyleSheet("font-size: 24px; font-weight: bold; color: #0ea5e9;") s_lay.addWidget(lbl_title) - s_lay.addWidget(QLabel("AI Forensic Edition\n")) + s_lay.addWidget(QLabel("AI Privacy Edition\n")) btn_open = QPushButton("Apri PDF") btn_open.setObjectName("Primary") btn_open.clicked.connect(self.cmd_open) @@ -613,6 +729,9 @@ def build_ui(self) -> None: s_lay.addSpacing(10) self.chk_img = QCheckBox("Oscura Immagini") s_lay.addWidget(self.chk_img) + self.chk_ocr = QCheckBox("OCR PDF scansionati") + self.chk_ocr.setChecked(True) + s_lay.addWidget(self.chk_ocr) s_lay.addSpacing(20) btn_dict = QPushButton("Dizionari") btn_dict.clicked.connect(self.cmd_dict) @@ -625,7 +744,7 @@ def build_ui(self) -> None: btn_clear.clicked.connect(self.cmd_clear) s_lay.addWidget(btn_clear) s_lay.addSpacing(20) - btn_export = QPushButton("Esporta PDF Sicuro") + btn_export = QPushButton("Esporta Privacy") btn_export.setStyleSheet("border-color: #0ea5e9; color: #0ea5e9;") btn_export.clicked.connect(self.cmd_export) s_lay.addWidget(btn_export) @@ -714,7 +833,8 @@ def write_log(self, m: str) -> None: else "#10b981" if "successo" in m or "completata" in m else "#94a3b8" ) ) - self.log.append(f"[{t}] {m}") + safe_message = html.escape(m) + self.log.append(f"[{t}] {safe_message}") def update_progress(self, c: int, t: int) -> None: """Update progress bar. @@ -767,12 +887,35 @@ def load_path(self, path: str) -> None: self.scale = 1.5 self.adjust_zoom(0) self.write_log(f"Caricato: {os.path.basename(path)}") + self._warn_if_scanned_pdf() except FileNotFoundError: self.write_log(f"ERRORE: File non trovato") except Exception as e: self.logger.error(f"Error loading PDF: {traceback.format_exc()}") self.write_log(f"ERRORE: {type(e).__name__}: {str(e)}") + def _warn_if_scanned_pdf(self) -> None: + """Warn when pages look image-only and therefore need OCR support.""" + if not self.doc: + return + scanned_pages = [] + with QMutexLocker(self.doc_mutex): + if not self.doc: + return + for page in self.doc: + text = page.get_text("text").strip() + has_images = bool(page.get_image_info(hashes=False)) + if not text and has_images: + scanned_pages.append(page.number + 1) + if scanned_pages: + preview = ", ".join(str(n) for n in scanned_pages[:10]) + suffix = "..." if len(scanned_pages) > 10 else "" + self.write_log( + "Avviso: probabile PDF scansionato senza testo OCR " + f"nelle pagine {preview}{suffix}. " + "Serve OCR prima della rilevazione automatica dei dati personali." + ) + def render(self) -> None: """Render current PDF page to display.""" if not self.doc: @@ -860,7 +1003,12 @@ def user_draw_rect(self, qrect: QRectF) -> None: fontsize=8, ) else: - p.add_redact_annot(r, fill=(0, 0, 0)) + self._add_privacy_redaction( + p, + r, + original=txt, + entity_type=self._infer_entity_type(txt), + ) cl = " ".join(txt.split()).lower() if len(cl) > 2: self.allowlist.discard(cl) @@ -979,7 +1127,7 @@ def cmd_about(self) -> None: lbl_title.setStyleSheet("font-size: 24px; font-weight: bold;") lbl_title.setAlignment(Qt.AlignCenter) lay.addWidget(lbl_title) - lbl_ver = QLabel(f"v{__version__} AI Forensic") + lbl_ver = QLabel(f"v{APP_VERSION} AI Privacy Beta") lbl_ver.setStyleSheet("color: #0ea5e9; font-weight: bold;") lbl_ver.setAlignment(Qt.AlignCenter) lay.addWidget(lbl_ver) @@ -1013,14 +1161,161 @@ def cmd_auto_ai(self) -> None: if self.rb_en.isChecked() else "IT" if self.rb_it.isChecked() else "BOTH" ) + use_ocr = self.chk_ocr.isChecked() + tessdata_dir = find_tessdata_dir() if use_ocr else None + if use_ocr and not tessdata_dir: + self.btn_ai.setEnabled(True) + QMessageBox.warning( + self, + "OCR non configurato", + "Per i PDF scansionati serve Tesseract tessdata. " + "Installa Tesseract-OCR e imposta TESSDATA_PREFIX oppure " + "disattiva OCR per analizzare solo il testo digitale.", + ) + return # Pass a snapshot copy of the allowlist set so the worker is insulated # from concurrent mutation by the UI (user_draw_rect / cmd_dict). self.start_scan_sig.emit( - self.doc, self.doc_mutex, lang, c_allow, set(self.allowlist) + self.doc, + self.doc_mutex, + lang, + c_allow, + set(self.allowlist), + use_ocr, + tessdata_dir, ) - @Slot(int, set) - def apply_ai_to_page(self, i: int, words: Set[str]) -> None: + def _select_privacy_export(self) -> Optional[PrivacyMode]: + """Ask the user which privacy export mode to use.""" + box = QMessageBox(self) + box.setWindowTitle("Modalita privacy") + box.setIcon(QMessageBox.Icon.Question) + box.setText("Scegli come trattare i dati personali nel PDF esportato.") + box.setInformativeText( + "Anonimizzazione: rimozione irreversibile.\n" + "Pseudonimizzazione: segnaposto reversibili con mappa cifrata separata." + ) + anonymize_btn = box.addButton( + "Anonimizzazione irreversibile", QMessageBox.ButtonRole.AcceptRole + ) + pseudonymize_btn = box.addButton( + "Pseudonimizzazione reversibile", QMessageBox.ButtonRole.ActionRole + ) + box.addButton(QMessageBox.StandardButton.Cancel) + box.exec() + clicked = box.clickedButton() + if clicked == anonymize_btn: + return PrivacyMode.ANONYMIZE + if clicked == pseudonymize_btn: + return PrivacyMode.PSEUDONYMIZE + return None + + def _infer_entity_type(self, value: str) -> str: + """Infer a coarse placeholder type for manually marked text.""" + compact = " ".join(value.split()) + if re.search(r"\b[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}\b", compact, re.I): + return "EMAIL_ADDRESS" + if re.search(r"\b[A-Z]{2}\d{2}[A-Z0-9]{11,30}\b", compact.replace(" ", ""), re.I): + return "IBAN_CODE" + if re.search(r"\+?\d[\d\s()./-]{6,}\d", compact): + return "PHONE_NUMBER" + if re.search(r"\b\d{13,19}\b", compact.replace(" ", "")): + return "CREDIT_CARD" + return "DATA" + + def _collect_redaction_rects(self, page: Any) -> List[fitz.Rect]: + """Return redaction rectangles for a page.""" + return [ + a.rect for a in (page.annots() or []) + if a.type[0] == fitz.PDF_ANNOT_REDACT + ] + + def _add_privacy_redaction( + self, + page: Any, + rect: fitz.Rect, + *, + original: str = "", + entity_type: str = "DATA", + fill: Tuple[float, float, float] = (0, 0, 0), + ) -> None: + """Add a redaction annotation carrying optional in-memory privacy data.""" + annot = page.add_redact_annot(rect, fill=fill) + clean_original = " ".join((original or "").split()) + if clean_original: + payload = { + "nullifypdf": 1, + "original": clean_original, + "entity_type": PlaceholderRegistry.normalize_entity_type(entity_type), + } + try: + annot.set_info(content=json.dumps(payload, ensure_ascii=False)) + except Exception as e: + self.logger.debug(f"Could not store redaction metadata: {e}") + + def _read_redaction_payload(self, annot: Any) -> Dict[str, str]: + """Read privacy data stored on a pending redaction annotation.""" + try: + raw = annot.info.get("content", "") + payload = json.loads(raw) if raw else {} + except Exception: + return {} + if payload.get("nullifypdf") != 1: + return {} + return { + "original": str(payload.get("original", "")), + "entity_type": str(payload.get("entity_type", "DATA")), + } + + def _prepare_pseudonymized_page( + self, page: Any, registry: PlaceholderRegistry + ) -> None: + """Replace pending redaction annotations with placeholder redactions.""" + pending = [ + ( + annot.rect, + self._read_redaction_payload(annot), + page.get_text("text", clip=annot.rect).strip(), + ) + for annot in (page.annots() or []) + if annot.type[0] == fitz.PDF_ANNOT_REDACT + ] + for annot in list(page.annots() or []): + if annot.type[0] == fitz.PDF_ANNOT_REDACT: + page.delete_annot(annot) + for rect, payload, clipped_text in pending: + clean_original = " ".join( + (payload.get("original") or clipped_text).split() + ) + if clean_original: + entity_type = payload.get("entity_type") or self._infer_entity_type( + clean_original + ) + placeholder = registry.placeholder_for( + clean_original, + entity_type, + page=page.number, + ) + page.add_redact_annot( + rect, + text=placeholder, + fill=(1, 1, 1), + text_color=(0, 0, 0), + align=1, + fontsize=8, + ) + else: + page.add_redact_annot( + rect, + text="[IMAGE_REMOVED]", + fill=(1, 1, 1), + text_color=(0, 0, 0), + align=1, + fontsize=8, + ) + + @Slot(int, object) + def apply_ai_to_page(self, i: int, detections: Any) -> None: """Apply AI-detected redactions to page. Runs on the UI thread (Qt slot). Holds `doc_mutex` while mutating the @@ -1028,15 +1323,25 @@ def apply_ai_to_page(self, i: int, words: Set[str]) -> None: Args: i: Page index. - words: Set of words detected as sensitive. + detections: Sensitive data detections with optional OCR rectangles. """ if not self.doc: return + if isinstance(detections, set): + normalized_detections = [ + {"text": word, "entity_type": self._infer_entity_type(word), "rects": []} + for word in detections + ] + else: + normalized_detections = list(detections or []) with QMutexLocker(self.doc_mutex): if i >= len(self.doc): return page = self.doc[i] - e_rects = [a.rect for a in (page.annots() or []) if a.type[0] == fitz.PDF_ANNOT_REDACT] + e_rects = [ + a.rect for a in (page.annots() or []) + if a.type[0] == fitz.PDF_ANNOT_REDACT + ] if self.chk_img.isChecked(): for img in page.get_image_info(hashes=False): ir = fitz.Rect(img["bbox"]) @@ -1054,18 +1359,34 @@ def apply_ai_to_page(self, i: int, words: Set[str]) -> None: e.contains(fitz.Point((r.x0 + r.x1) / 2, (r.y0 + r.y1) / 2)) for e in e_rects ): - page.add_redact_annot(r, fill=(0, 0, 0)) + self._add_privacy_redaction( + page, + r, + original=bw, + entity_type=self._infer_entity_type(bw), + ) e_rects.append(r) p_rects = [r for aw in self.allowlist for r in page.search_for(aw)] - for w in words: - for r in page.search_for(w): + for detection in normalized_detections: + word = str(detection.get("text", "")) + entity_type = str(detection.get("entity_type", "DATA")) + raw_rects = detection.get("rects") or [] + rects = [fitz.Rect(*coords) for coords in raw_rects] + if not rects and word: + rects = list(page.search_for(word)) + for r in rects: if any(r.intersects(pr) for pr in p_rects): continue if not any( e.contains(fitz.Point((r.x0 + r.x1) / 2, (r.y0 + r.y1) / 2)) for e in e_rects ): - page.add_redact_annot(r, fill=(0, 0, 0)) + self._add_privacy_redaction( + page, + r, + original=word, + entity_type=entity_type, + ) e_rects.append(r) @Slot() @@ -1075,7 +1396,7 @@ def ai_finished(self) -> None: self.render() def cmd_export(self) -> None: - """Export current PDF with forensic scrubbing. + """Export current PDF in irreversible or reversible privacy mode. Memory strategy (fix C1): instead of materializing the whole document in RAM via `self.doc.write()` (which would double peak memory on large @@ -1094,21 +1415,63 @@ def cmd_export(self) -> None: """ if not self.doc: return + mode = self._select_privacy_export() + if mode is None: + return p, _ = QFileDialog.getSaveFileName( self, "Esporta", - f"{os.path.splitext(self.doc.name)[0]}_secured.pdf", + f"{os.path.splitext(self.doc.name)[0]}_{mode.value}.pdf", "PDF (*.pdf)", ) if not p: return + map_path = "" + map_password = "" + if mode == PrivacyMode.PSEUDONYMIZE: + map_path, _ = QFileDialog.getSaveFileName( + self, + "Salva mappa cifrata", + f"{os.path.splitext(p)[0]}.nullifypdf-map", + "NullifyPDF map (*.nullifypdf-map)", + ) + if not map_path: + return + map_password, ok = QInputDialog.getText( + self, + "Password mappa", + "Password per cifrare la mappa di ripristino (minimo 12 caratteri):", + QLineEdit.Password, + ) + if not ok: + return + if len(map_password) < 12: + QMessageBox.warning( + self, + "Password troppo corta", + "La password della mappa deve avere almeno 12 caratteri.", + ) + return + if importlib.util.find_spec("cryptography") is None: + QMessageBox.critical( + self, + "Dipendenza mancante", + "La pseudonimizzazione richiede la libreria 'cryptography'.", + ) + return - self.write_log("Scrubbing Forense...") + self.write_log( + "Export anonimizzato..." if mode == PrivacyMode.ANONYMIZE + else "Export pseudonimizzato..." + ) # Sibling temp file (same directory as target so rename/cleanup are # always on the same filesystem; avoids cross-device issues). tmp_path = p + ".nullifypdf.tmp" ex_doc = None + registry = PlaceholderRegistry() try: + source_name = self.doc.name or "document.pdf" + source_sha256 = sha256_file(source_name) if os.path.exists(source_name) else "" # Step 1: serialize `self.doc` (with annotations) to disk under # the mutex so the AI worker thread cannot mutate mid-write. with QMutexLocker(self.doc_mutex): @@ -1125,11 +1488,10 @@ def cmd_export(self) -> None: # Step 3: scrub on the disk-backed copy. for page in ex_doc: + if mode == PrivacyMode.PSEUDONYMIZE: + self._prepare_pseudonymized_page(page, registry) # page.annots() may return None for pages with no annotations - r_rects = [ - a.rect for a in (page.annots() or []) - if a.type[0] == fitz.PDF_ANNOT_REDACT - ] + r_rects = self._collect_redaction_rects(page) try: links_to_delete = [ lnk for lnk in page.get_links() @@ -1158,7 +1520,21 @@ def cmd_export(self) -> None: # path than the open source (`tmp_path`) to avoid PyMuPDF's # "save to original" restriction on incremental saves. ex_doc.save(p, garbage=4, deflate=True, clean=True) - self.write_log("ESPORTATO.") + if mode == PrivacyMode.PSEUDONYMIZE: + payload = build_restore_payload( + source_name=source_name, + source_sha256=source_sha256, + output_sha256=sha256_file(p), + entries=registry.entries(), + ) + encrypted = encrypt_restore_payload(payload, map_password) + with open(map_path, "wb") as fh: + fh.write(encrypted) + self.write_log( + f"ESPORTATO. Mappa cifrata creata con {len(registry.entries())} segnaposto." + ) + else: + self.write_log("ESPORTATO.") except Exception as e: self.logger.error(f"Export failed: {traceback.format_exc()}") self.write_log(f"ERRORE export: {type(e).__name__}: {str(e)}") diff --git a/OCR_SETUP.md b/OCR_SETUP.md new file mode 100644 index 0000000..b8d1190 --- /dev/null +++ b/OCR_SETUP.md @@ -0,0 +1,44 @@ +# OCR Setup for Scanned PDFs + +NullifyPDF uses PyMuPDF's integrated OCR support for scanned PDFs. PyMuPDF +contains the OCR integration code, but it still needs Tesseract language data +(`tessdata`) installed on the machine. + +Windows: + +```powershell +setx TESSDATA_PREFIX "C:\Program Files\Tesseract-OCR\tessdata" +``` + +Linux: + +```bash +export TESSDATA_PREFIX=/usr/share/tesseract-ocr/4.00/tessdata +``` + +Install or download the language data you need: + +- `eng.traineddata` for English OCR. +- `ita.traineddata` for Italian OCR. + +For release builds, NullifyPDF uses the `Full` variant to bundle these two +files automatically. Developers can prepare a local Full build with: + +```bash +python build_local.py --full +``` + +If the OCR files are missing, the Full build downloads them automatically from +`tesseract-ocr/tessdata_fast` before running PyInstaller. + +Use `python build_local.py --lite` for a smaller build without bundled OCR +data. + +If OCR is enabled in the app but `tessdata` cannot be found, NullifyPDF stops +before scanning and asks you to configure Tesseract instead of silently +processing scanned pages without OCR. + +--- + +*Last updated: 2026-07-23* +*[Back to README](./README.md) | [Architecture β†’](./ARCHITECTURE.md)* diff --git a/README.md b/README.md index fdb9144..d5f7eff 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# πŸ”’ NullifyPDF β€” AI Forensic Edition +# NullifyPDF - AI Privacy Edition ![GitHub Release](https://img.shields.io/github/v/release/overwrite00/NullifyPDF?style=flat-square&color=1fb2e0&label=stable) ![GitHub Release (beta)](https://img.shields.io/github/v/release/overwrite00/NullifyPDF?include_prereleases&style=flat-square&color=orange&label=beta) @@ -10,7 +10,7 @@ NullifyPDF Logo

-> πŸ” **NullifyPDF** is a professional tool for forensic PDF anonymization. Designed for absolute privacy, it operates entirely locally using artificial intelligence to identify and permanently destroy sensitive data without ever uploading files to the cloud. +> **NullifyPDF** is a local-first PDF privacy tool. It helps detect personal data, redact it irreversibly, or replace it with reversible placeholders before a PDF is shared with third parties or uploaded to AI systems. > [!TIP] > First time using NullifyPDF? Start with our [**User Guide**](./USER_GUIDE.md) β€” takes 5 minutes. @@ -19,14 +19,14 @@ ## πŸ“‹ Quick Overview -NullifyPDF goes beyond simple text covering. It uses **Natural Language Processing (NLP)** engines to understand context and identify entities like *names*, *addresses*, *email addresses*, *IBANs*, and *credit card numbers*. Unlike common PDF editors, this tool performs **forensic scrubbing**, destroying metadata, hyperlinks, and hidden vector layers to ensure censorship is irreversible. +NullifyPDF goes beyond simple text covering. It uses **Natural Language Processing (NLP)** engines to identify entities like *names*, *addresses*, *email addresses*, *IBANs*, and *credit card numbers*. At export, the user chooses irreversible anonymization or reversible pseudonymization with an encrypted restore map. | 🎯 Feature | ✨ Benefit | | -------------------- | ------------------------------------- | | 🧠 **AI-Powered** | Bilingual (EN/IT) automatic detection | -| πŸ” **100% Local** | No cloud uploads, complete privacy | -| ⚑ **Real-time** | Instant scanning with live preview | -| πŸ›‘οΈ **Forensic-Grade** | Binary-level data destruction | +| πŸ” **Local-first** | No cloud uploads during PDF processing | +| ⚑ **Responsive UI** | Background scanning with live preview | +| πŸ›‘οΈ **Privacy Export** | Irreversible redaction or reversible placeholders | --- @@ -34,12 +34,13 @@ NullifyPDF goes beyond simple text covering. It uses **Natural Language Processi - 🧠 **AI-Powered Redaction** β€” Automatic bilingual (EN/IT) detection of PII: names, locations, emails, phones, IBANs, credit cards, crypto addresses - πŸ—„οΈ **Fluid UI & Thread-Safe** β€” PySide6 modern dark-mode interface with zero UI freezing. Text extraction in worker thread with QMutex serialization -- πŸ“– **Intelligent Persistent Dictionaries** β€” Blocklist and Allowlist synchronized to disk (`~/.nullifypdf`) with mutual exclusivity and anti-stacking logic. O(1) fast-path matching -- πŸ›‘οΈ **Forensic Scrubbing** β€” Not just black boxes. Binary-level destruction of metadata, hidden links, and flattened interactive forms (AcroForms) at export +- πŸ“– **Persistent Dictionaries** β€” Blocklist and Allowlist synchronized to disk (`~/.nullifypdf`) with O(1) fast-path matching +- πŸ›‘οΈ **Privacy Export Modes** β€” Choose irreversible anonymization or reversible pseudonymization with a separately encrypted restore map +- πŸ”Ž **OCR Support** β€” Full builds bundle EN/IT Tesseract language data for scanned PDFs - πŸ–ΌοΈ **Blindfold Mode** β€” One-click image/logo censoring with professional placeholder: `[ IMAGE REMOVED ]` -- πŸ“¦ **Native Cross-Platform** β€” Automated build scripts generate `.exe` (Windows), `.app` bundles (macOS), and `.deb`/`.rpm` packages (Linux) +- πŸ“¦ **Native Cross-Platform** β€” Automated build scripts generate Windows `.exe`, macOS ZIP bundles, and Linux `.deb`/`.rpm` packages - 🎯 **Drag & Drop Support** β€” Native file drag-and-drop on main window -- πŸ“Š **Logging & Diagnostics** β€” Rotating file-based logging (`~/.nullifypdf/logs/`) with debug mode for advanced troubleshooting +- πŸ“Š **Logging & Diagnostics** β€” Local file-based logging (`~/.nullifypdf/logs/`) with debug mode for advanced troubleshooting --- @@ -49,10 +50,10 @@ To keep NullifyPDF lightweight, 100% offline, and secure, be aware of these tech | ❌ Limitation | πŸ’‘ Workaround | | ---------------------------------- | ------------------------------------------------------------------------------------------------------- | -| **No Built-in OCR** | AI reads only digital text, not scanned images. Use Blindfold Mode to remove photo blocks entirely. | +| **OCR availability** | Full builds include EN/IT OCR data. Lite builds require local Tesseract tessdata for scanned PDFs. | | **Handwritten Text** | NLP models cannot analyze non-digitized handwriting. | | **Password-Protected PDFs** | Encrypted documents are blocked at load. Decrypt before importing. | -| **Digital Signatures Invalidated** | Forensic scrubbing destroys binary objects; cryptographic signatures (PAdES, notarized) become invalid. | +| **Digital Signatures Invalidated** | Privacy export changes the PDF; cryptographic signatures (PAdES, notarized) become invalid. | > [!WARNING] > Digital signatures will be invalidated after redaction. Save unredacted originals separately for formal records. @@ -81,11 +82,14 @@ To keep NullifyPDF lightweight, 100% offline, and secure, be aware of these tech Download the latest pre-compiled executable from [Releases](https://github.com/overwrite00/NullifyPDF/releases): -- **Windows:** `NullifyPDF_vX.Y.Z_Windows.exe` -- **macOS:** `NullifyPDF_vX.Y.Z_macOS.app` -- **Linux:** `nullifypdf_X.Y.Z_amd64.deb` or `.rpm` +- **Windows Lite:** `NullifyPDF_vX.Y.Z_Windows_Lite.exe` +- **Windows Full OCR:** `NullifyPDF_vX.Y.Z_Windows_Full.exe` +- **macOS Lite:** `NullifyPDF_vX.Y.Z_macOS_Lite.zip` +- **macOS Full OCR:** `NullifyPDF_vX.Y.Z_macOS_Full.zip` +- **Ubuntu Lite/Full:** `NullifyPDF_vX.Y.Z_Ubuntu_Lite.deb` or `NullifyPDF_vX.Y.Z_Ubuntu_Full.deb` +- **Fedora Lite/Full:** `NullifyPDF_vX.Y.Z_Fedora_Lite.rpm` or `NullifyPDF_vX.Y.Z_Fedora_Full.rpm` -No installation needed on Windows/macOS β€” just run. Linux users: `sudo dpkg -i nullifypdf_*.deb` +No installation needed on Windows/macOS - just run or unzip. Linux users can install the `.deb` or `.rpm` package. @@ -152,7 +156,6 @@ python setup_env.py - Creates `.venv` with Python 3.13 - Installs `requirements.txt` dependencies - Downloads spaCy models (English, Italian, both) -- Runs smoke tests to verify installation **Automatic OS detection:** - Windows: Uses `py -3.13` launcher @@ -166,14 +169,15 @@ python setup_env.py Compiles standalone executable with PyInstaller. ```bash -python build_local.py +python build_local.py --lite +python build_local.py --full ``` **Features:** - Cleans temporary directories - Auto-detects your OS - Reads version dynamically from code -- Generates named executable: `NullifyPDF_vX.Y.Z_Windows.exe` +- Generates named executables such as `NullifyPDF_vX.Y.Z_Windows_Lite.exe` or `NullifyPDF_vX.Y.Z_Windows_Full.exe` **Linux bonus:** On Ubuntu/Fedora, automatically builds `.deb` and `.rpm` packages in `dist/` @@ -182,7 +186,7 @@ python build_local.py
βœ“ Running Tests -Verify critical fixes with smoke tests: +Verify core behavior with smoke tests: ```bash # Activate venv first @@ -193,8 +197,10 @@ pytest tests/ -v **Test coverage:** - PDFListManager (blocklist/allowlist persistence) -- Input validation (path, range, language selection) - Resource path resolution +- OCR configuration helpers +- Privacy placeholder and encrypted restore-map primitives +- Build variant configuration
@@ -208,6 +214,7 @@ pytest tests/ -v | [CONTRIBUTING.md](./CONTRIBUTING.md) | How to contribute code & report issues | | [ARCHITECTURE.md](./ARCHITECTURE.md) | System design & technical overview | | [DEVELOPMENT.md](./DEVELOPMENT.md) | Local dev setup, testing & builds | +| [OCR_SETUP.md](./OCR_SETUP.md) | OCR data setup and Lite/Full build notes | | [TROUBLESHOOTING.md](./TROUBLESHOOTING.md) | Common issues & solutions | | [CHANGELOG.md](./CHANGELOG.md) | Release history & updates | @@ -216,12 +223,12 @@ pytest tests/ -v ## πŸ”’ Security & Privacy βœ… **100% Local Processing** β€” All analysis happens on your machine -βœ… **No Network Calls** β€” Except GitHub release checks +βœ… **No Cloud Uploads** β€” PDF processing stays local βœ… **Open Source** β€” Full code transparency βœ… **No Telemetry** β€” Zero user tracking > [!IMPORTANT] -> NullifyPDF performs binary-level data destruction. Always keep backups of original documents. +> Irreversible anonymization cannot be undone from the exported PDF. Always keep backups of original documents. Pseudonymization requires the encrypted restore map and its password. See [SECURITY.md](./SECURITY.md) for responsible disclosure and privacy details. @@ -236,6 +243,7 @@ See [SECURITY.md](./SECURITY.md) for responsible disclosure and privacy details. | **PyMuPDF (fitz)** | High-performance PDF manipulation | | **Microsoft Presidio** | PII (Personally Identifiable Information) detection | | **spaCy** | NLP for entity recognition (bilingual EN/IT) | +| **cryptography** | Encrypted pseudonymization restore maps | --- @@ -255,5 +263,5 @@ Want to help improve NullifyPDF? See [CONTRIBUTING.md](./CONTRIBUTING.md) for gu --- -*Last updated: 2026-07-14* +*Last updated: 2026-07-23* *[User Guide β†’](./USER_GUIDE.md)* diff --git a/SECURITY.md b/SECURITY.md index 9eb85e2..e093e94 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -1,9 +1,9 @@ # πŸ”’ Security β€” NullifyPDF -Information about NullifyPDF's security model, privacy guarantees, and how to report security vulnerabilities. +Information about NullifyPDF's security model, privacy boundaries, and how to report security vulnerabilities. > [!IMPORTANT] -> NullifyPDF is designed for maximum privacy. This document explains how we achieve it and what to do if you discover a vulnerability. +> NullifyPDF is designed for local-first privacy workflows. This document explains the security boundaries and what to do if you discover a vulnerability. --- @@ -19,13 +19,13 @@ NullifyPDF follows a **privacy-first architecture**: | **No Cloud** | No file uploads, no network transmission | | **No Telemetry** | Zero user tracking or analytics | | **Open Source** | Full code transparency, auditable by anyone | -| **Cryptographic Scrubbing** | Binary-level data destruction (not just covering) | +| **Privacy Export** | Irreversible redaction or encrypted reversible pseudonymization | ### What NullifyPDF Does NOT Do -❌ **No Internet Connections** (except GitHub release checks) +❌ **No Cloud Uploads During PDF Processing** ❌ **No Data Collection** (no logs sent anywhere) -❌ **No Third-party APIs** (everything local) +❌ **No Third-party Processing APIs** (analysis runs locally) ❌ **No User Accounts** (no registration required) ❌ **No Tracking** (no cookies, no analytics) @@ -37,10 +37,10 @@ NullifyPDF follows a **privacy-first architecture**: When you export a PDF with redactions: -1. βœ… **Metadata Stripped** β€” Creation date, author, embedded text removed -2. βœ… **Links Destroyed** β€” Hyperlinks and form fields eliminated -3. βœ… **Binary Scrubbing** β€” Text beneath redactions overwritten at binary level -4. βœ… **Forensically Sound** β€” Redacted data is unrecoverable +1. βœ… **Metadata Stripped** β€” Creation date, author, and document metadata removed +2. βœ… **Links Removed When Redacted** β€” Links overlapping redaction areas are deleted +3. βœ… **Redactions Applied** β€” Selected text/image areas are removed from the exported PDF +4. βœ… **Pseudonymization Maps Encrypted** β€” Restore maps are stored separately and encrypted with a user password ### Temporary Files @@ -65,9 +65,9 @@ During export, NullifyPDF uses **disk-backed temporary files**: ### File Type Verification - βœ… PDF files only (blocked: `.exe`, `.zip`, etc.) -- βœ… File size limits to prevent DOS attacks +- βœ… Password-protected PDFs are blocked before processing - βœ… Encryption detection (blocks password-protected PDFs) -- βœ… Path traversal protection (prevents `../../../etc/passwd` exploits) +- βœ… Extension and existence checks before opening PDFs ### User Input Validation @@ -129,7 +129,7 @@ We follow responsible disclosure principles: 2. **Malware on Your Computer** β€” If your machine is compromised 3. **Unencrypted Storage** β€” Save your PDF to an encrypted drive if sensitive 4. **Physical Access** β€” If someone accesses your hard drive directly -5. **Forensic Recovery** β€” If sophisticated attackers do disk forensics +5. **Residual PDF Artifacts** β€” Complex PDFs may contain structures not covered by automated checks ### What You Can Do @@ -171,9 +171,9 @@ bandit NullifyPDF.py Security-relevant tests cover: -- Input validation (path traversal, injection) +- Input validation smoke tests - Resource cleanup (file handles, memory) -- Permission handling (file mode, ownership) +- Privacy-core and build-configuration tests Run tests: ```bash @@ -201,12 +201,8 @@ Your redaction preferences (blocklist/allowlist) are stored locally: ### File Permissions -On Linux/macOS, directory permissions default to: -``` -drwx------ user group .nullifypdf/ -``` - -Only your user can read/write. On Windows, standard user ACLs apply. +The application relies on the operating system's standard user permissions for +`~/.nullifypdf/`. Store sensitive files on encrypted storage when required. --- @@ -222,6 +218,7 @@ NullifyPDF uses trusted, actively-maintained libraries: | **pymupdf** | PDF manipulation | βœ… Actively maintained | | **presidio-analyzer** | PII detection | βœ… Maintained by Microsoft | | **spacy** | NLP engine | βœ… Actively maintained | +| **cryptography** | Restore-map encryption | βœ… Actively maintained | ### Vulnerability Scanning @@ -240,7 +237,7 @@ NullifyPDF is provided **as-is** without warranty. While we take security seriou - **No Guarantee of Unrecoverability** β€” For highly sensitive data, consult legal/security experts - **No Liability** β€” Use at your own risk - **Not a Legal Tool** β€” Consult lawyers for document redaction in legal cases -- **Forensic Limitations** β€” Determined attackers with forensic tools may recover data +- **Residual Risk** β€” For highly sensitive or legally critical documents, perform independent review and validation For mission-critical or legal redactions, consider: - Professional redaction services @@ -282,5 +279,5 @@ For **security vulnerabilities only**: --- -*Last updated: 2026-06-07* +*Last updated: 2026-07-23* *← [Contributing](./CONTRIBUTING.md) | [Back to README β†’](./README.md)* diff --git a/TROUBLESHOOTING.md b/TROUBLESHOOTING.md index e0f76ec..0dcf8aa 100644 --- a/TROUBLESHOOTING.md +++ b/TROUBLESHOOTING.md @@ -58,7 +58,7 @@ If log isn't updating: ## πŸ”΄ Export Fails or Crashes ### Symptom -Clicking "Export Secure PDF" causes error or app crash. +Clicking "Esporta Privacy" causes error or app crash. ### Root Cause - Insufficient disk space @@ -150,7 +150,7 @@ Names, emails, or other PII should be detected but aren't highlighted. ### Root Cause - Wrong language selected -- Text is in an image (not searchable) +- Text is in an image and OCR is disabled or not configured - Unusual formatting - Text is handwritten @@ -177,21 +177,24 @@ Try copying text from the PDF: 3. Paste in text editor If you can copy text β†’ it's digital text β†’ AI should detect it -If you can't copy β†’ it's an image β†’ use **Blindfold Mode** to hide image blocks +If you can't copy β†’ it's an image β†’ enable **OCR PDF scansionati** -**Check 3 β€” Enable Blindfold Mode** +**Check 3 β€” Enable OCR** For scanned PDFs: -1. Toggle **"Blindfold Mode"** in sidebar -2. Run **"Auto Redact"** again -3. All images replaced with placeholder +1. Enable **"OCR PDF scansionati"** in the sidebar +2. Make sure you use a Full build or have local Tesseract `tessdata` +3. Run **"Auto Redact (AI)"** again + +Use image redaction only when you intentionally want to remove whole images or +scanned photo blocks. **Check 4 β€” Manual Redaction** If AI still misses something: 1. Click and drag mouse over the text 2. A redaction box appears -3. Export β€” it will be destroyed +3. Export with irreversible anonymization or pseudonymization, depending on the selected privacy mode ### Allowlist Issue? @@ -452,10 +455,11 @@ The more details, the faster we can fix it! πŸš€ - πŸ“– **User Guide:** [USER_GUIDE.md](./USER_GUIDE.md) - πŸ—οΈ **Architecture:** [ARCHITECTURE.md](./ARCHITECTURE.md) +- πŸ”Ž **OCR Setup:** [OCR_SETUP.md](./OCR_SETUP.md) - 🀝 **Contributing:** [CONTRIBUTING.md](./CONTRIBUTING.md) - πŸ”’ **Security:** [SECURITY.md](./SECURITY.md) --- -*Last updated: 2026-07-14* +*Last updated: 2026-07-23* *← [User Guide](./USER_GUIDE.md) | [Development β†’](./DEVELOPMENT.md)* diff --git a/USER_GUIDE.md b/USER_GUIDE.md index 35c6bea..2966bd9 100644 --- a/USER_GUIDE.md +++ b/USER_GUIDE.md @@ -22,12 +22,12 @@ Your step-by-step guide to redacting sensitive data from PDFs safely and securel ## Step 1️⃣ β€” Load Your PDF -1. Click the blue **"Open PDF"** button in the top left +1. Click the blue **"Apri PDF"** button in the top left 2. Select a PDF file from your computer 3. The document appears in the center. Scroll with mouse or use arrow buttons (top right) > [!NOTE] -> Supported formats: PDF (unencrypted, uncompressed text) +> Supported format: unencrypted PDF. Digital text is scanned directly; scanned pages require OCR. --- @@ -98,29 +98,28 @@ Make small text larger: Remove logos, signatures, or scanned photos: -1. **Enable** the toggle: **"Blindfold Mode"** (sidebar) +1. **Enable** the toggle: **"Oscura Immagini"** (sidebar) 2. Click **"Auto Redact (AI)"** again 3. All images replaced with gray placeholder: `[ IMAGE REMOVED ]` > [!NOTE] -> Use this for scanned documents where text is embedded in images (no OCR available). +> For scanned documents, enable **OCR PDF scansionati** before running the AI scan. Full builds include EN/IT OCR data; Lite builds require local Tesseract tessdata. --- -## Step 6️⃣ β€” Export the Secure PDF +## Step 6️⃣ β€” Export the Privacy PDF When satisfied with redactions: -1. Click **"Export Secure PDF"** button -2. Choose filename and location -3. The new PDF is now **forensically sanitized**: - - Black boxes are **binary-level destroyed** (not recoverable) - - Metadata removed - - Hidden links destroyed - - Interactive forms flattened +1. Click **"Esporta Privacy"** +2. Choose the export mode: + - **Anonimizzazione irreversibile** removes selected personal data permanently + - **Pseudonimizzazione reversibile** replaces selected personal data with placeholders and creates a separate encrypted restore map +3. Choose filename and location +4. Review the exported PDF before sharing it > [!CAUTION] -> **Export is destructive and permanent.** Keep a backup of the original PDF. +> **Irreversible anonymization cannot be undone from the exported PDF.** Keep a backup of the original PDF. For pseudonymization, store the encrypted restore map separately from the PDF. --- @@ -165,11 +164,11 @@ admin@example.com -### Mutual Exclusivity +### Blocklist and Allowlist -⚠️ **A word cannot be in BOTH lists simultaneously.** - -If you add a word to Allowlist that's already in Blocklist, it's removed from Blocklist automatically. +The manual redaction workflow keeps clicked/drawn terms mutually exclusive. +When editing dictionary files or the dictionary dialog directly, review both +lists and avoid adding the same term to both. --- @@ -228,30 +227,30 @@ python3.13 NullifyPDF.py **Effect:** Debug logs include full stack traces. Useful when reporting bugs. -### Log Rotation +### Log File -- **Max file size:** 5 MB -- **Backup copies:** 3 old logs auto-deleted - **Encoding:** UTF-8 +- **Location:** `~/.nullifypdf/logs/nullifypdf.log` +- **Rotation:** not currently implemented; remove old logs manually if needed --- ## ❓ Common Questions ### Q: Why doesn't AI detect text in my scanned PDF? -**A:** NullifyPDF analyzes only digital text, not images. Use **Blindfold Mode** to hide entire image blocks. +**A:** Enable **OCR PDF scansionati** before running the AI scan. Full builds include EN/IT OCR data; Lite builds need local Tesseract `tessdata`. ### Q: Can I password-protect the exported PDF? **A:** Not built-in. Use a PDF editor after export for password protection. -### Q: How do I know redactions are truly destroyed? -**A:** NullifyPDF performs binary-level destruction. See [SECURITY.md](./SECURITY.md) for technical details. +### Q: How do I know redactions are applied? +**A:** Export applies PDF redactions and removes selected metadata. Review the exported file before sharing, especially for high-risk documents. ### Q: Can I undo changes after export? -**A:** **No.** Export is permanent and destructive. Always keep the original. +**A:** Irreversible anonymization cannot be undone from the exported PDF. Pseudonymization can be restored only with the encrypted restore map and its password. ### Q: Does NullifyPDF send data to the cloud? -**A:** **No.** 100% local processing. No network calls except GitHub release checks. +**A:** **No.** PDF processing is local. Full builds may download OCR data during build time, not while processing your PDFs. --- @@ -264,5 +263,5 @@ python3.13 NullifyPDF.py --- -*Last updated: 2026-07-14* +*Last updated: 2026-07-23* *← [Back to README](./README.md) | [Troubleshooting β†’](./TROUBLESHOOTING.md)* diff --git a/build_local.py b/build_local.py index c91e3b3..a3293d5 100644 --- a/build_local.py +++ b/build_local.py @@ -4,26 +4,56 @@ import subprocess import platform import re -from typing import Optional +import argparse +import pathlib +from typing import List, Optional, Tuple +from scripts.download_ocr_data import download_ocr_data + + +VALID_BUILD_VARIANTS = {"lite", "full"} +OCR_TESSDATA_FILES = ("eng.traineddata", "ita.traineddata") -def get_version() -> str: - """Extract version from NullifyPDF.py with fallback. + +def get_version_info() -> Tuple[str, str]: + """Extract base version and optional prerelease label from NullifyPDF.py. Returns: - str: Version string from __version__ or 'unknown' fallback. + Tuple[str, str]: Base version and optional prerelease label. """ try: if not os.path.exists("NullifyPDF.py"): - return "unknown" + return "unknown", "" with open("NullifyPDF.py", "r", encoding="utf-8") as f: content = f.read() - match = re.search(r'__version__\s*=\s*[\'"]([^\'"]+)[\'"]', content) - if match: - return match.group(1) + version_match = re.search(r'__version__\s*=\s*[\'"]([^\'"]+)[\'"]', content) + prerelease_match = re.search( + r'__version_prerelease__\s*=\s*[\'"]([^\'"]*)[\'"]', + content, + ) + if version_match: + prerelease = prerelease_match.group(1).strip() if prerelease_match else "" + return version_match.group(1), prerelease except (IOError, OSError) as e: print(f"[WARNING] Could not read version: {e}") - return "unknown" + return "unknown", "" + + +def get_version() -> str: + """Return the base application version.""" + return get_version_info()[0] + + +def get_file_version( + version: str, code_prerelease: str, env_beta_suffix: Optional[str] +) -> str: + """Return the version string used in artifact filenames.""" + effective_suffix = (env_beta_suffix or code_prerelease or "").strip() + if not effective_suffix: + return version + if version.endswith(f"-{effective_suffix}"): + return version + return f"{version}-{effective_suffix}" def ensure_icon(sys_os: str) -> Optional[str]: @@ -45,14 +75,95 @@ def ensure_icon(sys_os: str) -> Optional[str]: return os.path.join(base_dir, "NullifyPDF_icon.png").replace("\\", "/") -def build_rpm(version: str, file_version: str, executable_name: str) -> None: +def normalize_build_variant(value: Optional[str]) -> str: + """Return a supported build variant name.""" + variant = (value or os.environ.get("NULLIFYPDF_BUILD_VARIANT") or "lite").lower() + if variant not in VALID_BUILD_VARIANTS: + allowed = ", ".join(sorted(VALID_BUILD_VARIANTS)) + raise ValueError(f"Build variant non valida: {variant}. Valori: {allowed}") + return variant + + +def parse_args() -> argparse.Namespace: + """Parse build command-line options.""" + parser = argparse.ArgumentParser(description="Build NullifyPDF with PyInstaller") + group = parser.add_mutually_exclusive_group() + group.add_argument("--lite", action="store_true", help="Build without bundled OCR") + group.add_argument("--full", action="store_true", help="Build with bundled OCR data") + return parser.parse_args() + + +def variant_from_args(args: argparse.Namespace) -> str: + """Resolve build variant from CLI flags or environment.""" + if args.lite: + return "lite" + if args.full: + return "full" + return normalize_build_variant(None) + + +def ensure_ocr_data(download_func=download_ocr_data) -> None: + """Ensure EN/IT OCR data is available for Full builds.""" + tessdata_dir = os.path.join("ocr", "tessdata") + missing = [ + name for name in OCR_TESSDATA_FILES + if not os.path.exists(os.path.join(tessdata_dir, name)) + ] + if not missing: + return + + missing_list = ", ".join(missing) + print( + "[INFO] Build Full richiesto: mancano dati OCR " + f"({missing_list}). Download automatico da tesseract-ocr/tessdata_fast..." + ) + download_func(pathlib.Path(tessdata_dir)) + still_missing = [ + name for name in OCR_TESSDATA_FILES + if not os.path.exists(os.path.join(tessdata_dir, name)) + ] + if still_missing: + raise FileNotFoundError( + "Download OCR incompleto. Mancano ancora: " + f"{', '.join(still_missing)}. Usa --lite oppure controlla la rete." + ) + + +def pyinstaller_datas( + build_variant: str, download_missing_ocr: bool = True +) -> List[Tuple[str, str]]: + """Return data files/directories to include in the PyInstaller bundle.""" + datas: List[Tuple[str, str]] = [] + if os.path.exists("images"): + datas.append(("images", "images")) + if build_variant == "full": + tessdata_dir = os.path.join("ocr", "tessdata") + if download_missing_ocr: + ensure_ocr_data() + elif any( + not os.path.exists(os.path.join(tessdata_dir, name)) + for name in OCR_TESSDATA_FILES + ): + raise FileNotFoundError( + "Build Full richiesto, ma mancano file OCR. " + "Usa download_missing_ocr=True oppure --lite." + ) + for name in OCR_TESSDATA_FILES: + source = os.path.join(tessdata_dir, name).replace("\\", "/") + datas.append((source, "ocr/tessdata")) + return datas + + +def build_rpm( + version: str, file_version: str, executable_name: str, variant_label: str +) -> None: """Build RPM package for Fedora/RHEL. Args: version: Application version used for the RPM package metadata (must not contain hyphens, which the RPM Version tag disallows). file_version: Version string used for the output artifact filename - (may include a beta suffix, e.g. "2.0.7-beta.3"). + (may include a beta suffix, e.g. "2.1.0-beta.1"). executable_name: Name of compiled executable. """ print("\n[INFO] Creazione pacchetto RPM per Fedora/RHEL...") @@ -121,7 +232,7 @@ def build_rpm(version: str, file_version: str, executable_name: str) -> None: if file.endswith(".rpm"): shutil.move( os.path.join(root, file), - f"dist/NullifyPDF_v{file_version}_Fedora.rpm", + f"dist/NullifyPDF_v{file_version}_Fedora_{variant_label}.rpm", ) print("[OK] RPM creato con successo.") except Exception as e: @@ -130,13 +241,15 @@ def build_rpm(version: str, file_version: str, executable_name: str) -> None: shutil.rmtree(rpm_dir, ignore_errors=True) -def build_deb(version: str, file_version: str, executable_name: str) -> None: +def build_deb( + version: str, file_version: str, executable_name: str, variant_label: str +) -> None: """Build DEB package for Ubuntu/Debian. Args: version: Application version used for the DEB package metadata. file_version: Version string used for the output artifact filename - (may include a beta suffix, e.g. "2.0.7-beta.3"). + (may include a beta suffix, e.g. "2.1.0-beta.1"). executable_name: Name of compiled executable. """ print("\n[INFO] Creazione pacchetto DEB per Ubuntu/Debian...") @@ -180,7 +293,12 @@ def build_deb(version: str, file_version: str, executable_name: str) -> None: try: subprocess.run( - ["dpkg-deb", "--build", pkg_dir, f"dist/NullifyPDF_v{file_version}_Ubuntu.deb"], + [ + "dpkg-deb", + "--build", + pkg_dir, + f"dist/NullifyPDF_v{file_version}_Ubuntu_{variant_label}.deb", + ], check=True, stdout=subprocess.DEVNULL, ) @@ -191,7 +309,7 @@ def build_deb(version: str, file_version: str, executable_name: str) -> None: shutil.rmtree(pkg_dir, ignore_errors=True) -def build_app() -> None: +def build_app(build_variant: Optional[str] = None) -> None: """Build NullifyPDF application for current OS using PyInstaller. Automatically generates platform-specific executables: @@ -200,9 +318,11 @@ def build_app() -> None: - Linux: portable binary + .deb + .rpm packages """ print("--- Avvio Compilazione NullifyPDF (PySide6) ---") - version = get_version() + build_variant = normalize_build_variant(build_variant) + variant_label = build_variant.capitalize() + version, code_prerelease = get_version_info() beta_suffix = os.environ.get("NULLIFYPDF_BETA_SUFFIX", "").strip() - file_version = f"{version}-{beta_suffix}" if beta_suffix else version + file_version = get_file_version(version, code_prerelease, beta_suffix) sys_os = platform.system() for item in ["build", "dist", "NullifyPDF.spec"]: @@ -214,16 +334,18 @@ def build_app() -> None: if sys_os == "Windows" else ("macOS", "") if sys_os == "Darwin" else ("Linux_Portable", "") ) - final_name = f"NullifyPDF_v{file_version}_{os_name}{ext}" + final_name = f"NullifyPDF_v{file_version}_{os_name}_{variant_label}{ext}" icon_path = ensure_icon(sys_os) # Use repr() to safely embed the path as a Python literal in the spec file. # Manual single-quote wrapping is unsafe for paths containing quotes/backslashes. icon_str = repr(icon_path) if icon_path else "None" + datas_literal = repr(pyinstaller_datas(build_variant)) + print(f"[INFO] Variante build: {variant_label}") if sys_os == "Darwin": spec_content = f"""# -*- mode: python ; coding: utf-8 -*- from PyInstaller.utils.hooks import collect_all -datas = [('images', 'images')] if __import__('os').path.exists('images') else [] +datas = {datas_literal} binaries = [] hiddenimports = ['spacy', 'presidio_analyzer'] for pkg in ['presidio_analyzer', 'spacy', 'en_core_web_md', 'it_core_news_md']: @@ -239,7 +361,7 @@ def build_app() -> None: else: spec_content = f"""# -*- mode: python ; coding: utf-8 -*- from PyInstaller.utils.hooks import collect_all -datas = [('images', 'images')] if __import__('os').path.exists('images') else [] +datas = {datas_literal} binaries = [] hiddenimports = ['spacy', 'presidio_analyzer'] for pkg in ['presidio_analyzer', 'spacy', 'en_core_web_md', 'it_core_news_md']: @@ -264,7 +386,7 @@ def build_app() -> None: print(f"[OK] Compilazione completata: dist/{final_name}") elif sys_os == "Darwin": print("[INFO] Compressione App Bundle per macOS in formato ZIP...") - zip_filename = f"NullifyPDF_v{file_version}_macOS.zip" + zip_filename = f"NullifyPDF_v{file_version}_macOS_{variant_label}.zip" subprocess.run( ["zip", "-r", "-y", zip_filename, "NullifyPDF.app"], cwd="dist", @@ -277,9 +399,9 @@ def build_app() -> None: os.rename("dist/NullifyPDF", f"dist/{final_name}") print(f"[OK] Eseguibile portatile pronto: dist/{final_name}") if shutil.which("rpmbuild"): - build_rpm(version, file_version, final_name) + build_rpm(version, file_version, final_name, variant_label) if shutil.which("dpkg-deb"): - build_deb(version, file_version, final_name) + build_deb(version, file_version, final_name, variant_label) except subprocess.CalledProcessError as e: print(f"\n[ERROR] ERRORE CRITICO: Compilazione fallita (exit {e.returncode}).") @@ -287,4 +409,4 @@ def build_app() -> None: if __name__ == "__main__": - build_app() + build_app(variant_from_args(parse_args())) diff --git a/privacy_core.py b/privacy_core.py new file mode 100644 index 0000000..afa5e8b --- /dev/null +++ b/privacy_core.py @@ -0,0 +1,146 @@ +"""Privacy primitives for NullifyPDF export workflows. + +This module is deliberately independent from PySide / PyMuPDF so that the +security-sensitive policy and restore-map logic can be tested without a GUI. +""" + +from __future__ import annotations + +import base64 +import json +import os +import re +from dataclasses import asdict, dataclass +from enum import Enum +from typing import Dict, Iterable, List, Optional + + +class PrivacyMode(str, Enum): + """Supported privacy export modes.""" + + ANONYMIZE = "anonymize" + PSEUDONYMIZE = "pseudonymize" + + +@dataclass(frozen=True) +class PlaceholderEntry: + """One reversible placeholder mapping entry.""" + + placeholder: str + original: str + entity_type: str + page: int + + +class PlaceholderRegistry: + """Create stable placeholders for detected personal data.""" + + def __init__(self) -> None: + self._counters: Dict[str, int] = {} + self._by_value: Dict[tuple[str, str], str] = {} + self._entries: List[PlaceholderEntry] = [] + + @staticmethod + def normalize_entity_type(entity_type: Optional[str]) -> str: + value = (entity_type or "DATA").upper() + value = re.sub(r"[^A-Z0-9_]+", "_", value).strip("_") + return value or "DATA" + + def placeholder_for( + self, original: str, entity_type: Optional[str] = None, page: int = 0 + ) -> str: + clean_original = " ".join((original or "").split()) + clean_type = self.normalize_entity_type(entity_type) + key = (clean_type, clean_original.casefold()) + if key in self._by_value: + return self._by_value[key] + + next_index = self._counters.get(clean_type, 0) + 1 + self._counters[clean_type] = next_index + placeholder = f"{clean_type}_{next_index:03d}" + self._by_value[key] = placeholder + self._entries.append( + PlaceholderEntry( + placeholder=placeholder, + original=clean_original, + entity_type=clean_type, + page=max(0, int(page)), + ) + ) + return placeholder + + def entries(self) -> List[PlaceholderEntry]: + return list(self._entries) + + +def build_restore_payload( + *, + source_name: str, + source_sha256: str, + output_sha256: Optional[str], + entries: Iterable[PlaceholderEntry], +) -> Dict[str, object]: + """Build the JSON-serializable restore-map payload.""" + + return { + "format": "NullifyPDF restore map", + "version": 1, + "source_name": os.path.basename(source_name), + "source_sha256": source_sha256, + "output_sha256": output_sha256, + "entries": [asdict(entry) for entry in entries], + } + + +def encrypt_restore_payload(payload: Dict[str, object], password: str) -> bytes: + """Encrypt and authenticate a restore-map payload with a password.""" + + if not password or len(password) < 12: + raise ValueError("La password della mappa deve avere almeno 12 caratteri.") + + from cryptography.fernet import Fernet + from cryptography.hazmat.primitives import hashes + from cryptography.hazmat.primitives.kdf.pbkdf2 import PBKDF2HMAC + + salt = os.urandom(16) + kdf = PBKDF2HMAC( + algorithm=hashes.SHA256(), + length=32, + salt=salt, + iterations=600000, + ) + key = base64.urlsafe_b64encode(kdf.derive(password.encode("utf-8"))) + token = Fernet(key).encrypt( + json.dumps(payload, ensure_ascii=False, sort_keys=True).encode("utf-8") + ) + envelope = { + "format": "NullifyPDF encrypted restore map", + "version": 1, + "kdf": "PBKDF2-HMAC-SHA256", + "iterations": 600000, + "salt": base64.b64encode(salt).decode("ascii"), + "token": token.decode("ascii"), + } + return json.dumps(envelope, ensure_ascii=False, sort_keys=True, indent=2).encode( + "utf-8" + ) + + +def decrypt_restore_payload(data: bytes, password: str) -> Dict[str, object]: + """Decrypt an encrypted restore map.""" + + from cryptography.fernet import Fernet + from cryptography.hazmat.primitives import hashes + from cryptography.hazmat.primitives.kdf.pbkdf2 import PBKDF2HMAC + + envelope = json.loads(data.decode("utf-8")) + salt = base64.b64decode(envelope["salt"]) + kdf = PBKDF2HMAC( + algorithm=hashes.SHA256(), + length=32, + salt=salt, + iterations=int(envelope["iterations"]), + ) + key = base64.urlsafe_b64encode(kdf.derive(password.encode("utf-8"))) + plaintext = Fernet(key).decrypt(envelope["token"].encode("ascii")) + return json.loads(plaintext.decode("utf-8")) diff --git a/requirements.txt b/requirements.txt index 275e1f0..dea32c1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,6 +11,7 @@ PyMuPDF==1.28.0 # NLP & Security Analysis presidio-analyzer==2.2.363 spacy==3.8.13 # Mantenuto in 3.8.13 per compatibilitΓ  con presidio-analyzer 2.2.363 +cryptography==45.0.7 # spaCy language models (pinned to exact versions for reproducibility) # IMPORTANT: model minor version MUST match the spaCy core minor version # (spaCy 3.8.x <-> models 3.8.x). The models declare spacy>=3.8.0,<3.9.0. diff --git a/scripts/download_ocr_data.py b/scripts/download_ocr_data.py new file mode 100644 index 0000000..14a3ade --- /dev/null +++ b/scripts/download_ocr_data.py @@ -0,0 +1,43 @@ +"""Download bundled OCR language data for NullifyPDF Full builds.""" + +from __future__ import annotations + +import argparse +import pathlib +import urllib.request + +TESSDATA_FAST_REF = "4.1.0" +LANGUAGES = ("eng", "ita") +BASE_URL = ( + "https://raw.githubusercontent.com/tesseract-ocr/" + f"tessdata_fast/{TESSDATA_FAST_REF}" +) + + +def download_ocr_data(output_dir: pathlib.Path) -> None: + """Download required Tesseract traineddata files.""" + output_dir.mkdir(parents=True, exist_ok=True) + for lang in LANGUAGES: + target = output_dir / f"{lang}.traineddata" + url = f"{BASE_URL}/{lang}.traineddata" + print(f"[INFO] Download {url}") + urllib.request.urlretrieve(url, target) + if target.stat().st_size < 1024: + raise RuntimeError(f"Downloaded OCR file is unexpectedly small: {target}") + print(f"[OK] {target}") + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser(description="Download NullifyPDF OCR data") + parser.add_argument( + "--output-dir", + default="ocr/tessdata", + type=pathlib.Path, + help="Destination tessdata directory", + ) + return parser.parse_args() + + +if __name__ == "__main__": + args = parse_args() + download_ocr_data(args.output_dir) diff --git a/tests/test_build_config.py b/tests/test_build_config.py new file mode 100644 index 0000000..37e45cc --- /dev/null +++ b/tests/test_build_config.py @@ -0,0 +1,96 @@ +"""Tests for build variant configuration.""" + +import pathlib + +import pytest + +from build_local import ( + ensure_ocr_data, + get_file_version, + get_version_info, + normalize_build_variant, + pyinstaller_datas, + variant_from_args, +) + + +class Args: + def __init__(self, *, lite=False, full=False): + self.lite = lite + self.full = full + + +def test_normalize_build_variant_defaults_to_lite(monkeypatch): + monkeypatch.delenv("NULLIFYPDF_BUILD_VARIANT", raising=False) + + assert normalize_build_variant(None) == "lite" + + +def test_variant_from_args_prefers_cli_over_environment(monkeypatch): + monkeypatch.setenv("NULLIFYPDF_BUILD_VARIANT", "full") + + assert variant_from_args(Args(lite=True)) == "lite" + assert variant_from_args(Args(full=True)) == "full" + + +def test_invalid_build_variant_is_rejected(): + with pytest.raises(ValueError): + normalize_build_variant("huge") + + +def test_get_version_info_reads_base_and_prerelease(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + (tmp_path / "NullifyPDF.py").write_text( + '__version__ = "9.9.9"\n__version_prerelease__ = "beta.7"\n', + encoding="utf-8", + ) + + assert get_version_info() == ("9.9.9", "beta.7") + + +def test_get_file_version_prefers_env_suffix_without_duplication(): + assert get_file_version("2.1.0", "beta.1", "") == "2.1.0-beta.1" + assert get_file_version("2.1.0", "beta.1", "beta.1") == "2.1.0-beta.1" + assert get_file_version("2.1.0", "", "beta.2") == "2.1.0-beta.2" + + +def test_lite_build_does_not_require_ocr_data(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + + assert pyinstaller_datas("lite") == [] + + +def test_full_build_can_reject_missing_ocr_data_when_download_disabled( + tmp_path, monkeypatch +): + monkeypatch.chdir(tmp_path) + + with pytest.raises(FileNotFoundError): + pyinstaller_datas("full", download_missing_ocr=False) + + +def test_full_build_downloads_missing_ocr_data(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + + def fake_download(output_dir): + output_dir.mkdir(parents=True) + (output_dir / "eng.traineddata").write_bytes(b"eng") + (output_dir / "ita.traineddata").write_bytes(b"ita") + + ensure_ocr_data(download_func=fake_download) + + assert (pathlib.Path("ocr") / "tessdata" / "eng.traineddata").exists() + assert (pathlib.Path("ocr") / "tessdata" / "ita.traineddata").exists() + + +def test_full_build_includes_ocr_data(tmp_path, monkeypatch): + monkeypatch.chdir(tmp_path) + tessdata = pathlib.Path("ocr") / "tessdata" + tessdata.mkdir(parents=True) + (tessdata / "eng.traineddata").write_bytes(b"eng") + (tessdata / "ita.traineddata").write_bytes(b"ita") + + datas = pyinstaller_datas("full") + + assert ("ocr/tessdata/eng.traineddata", "ocr/tessdata") in datas + assert ("ocr/tessdata/ita.traineddata", "ocr/tessdata") in datas diff --git a/tests/test_privacy_core.py b/tests/test_privacy_core.py new file mode 100644 index 0000000..97e2097 --- /dev/null +++ b/tests/test_privacy_core.py @@ -0,0 +1,53 @@ +"""Tests for privacy export primitives.""" + +import pytest + +from privacy_core import ( + PlaceholderRegistry, + build_restore_payload, + decrypt_restore_payload, + encrypt_restore_payload, +) + + +def test_placeholder_registry_reuses_same_value(): + registry = PlaceholderRegistry() + + first = registry.placeholder_for("Mario Rossi", "person", page=1) + second = registry.placeholder_for("Mario Rossi", "PERSON", page=2) + + assert first == "PERSON_001" + assert second == first + assert len(registry.entries()) == 1 + + +def test_placeholder_registry_separates_entity_types(): + registry = PlaceholderRegistry() + + person = registry.placeholder_for("Roma", "person") + location = registry.placeholder_for("Roma", "location") + + assert person == "PERSON_001" + assert location == "LOCATION_001" + + +def test_encrypted_restore_map_round_trip(): + registry = PlaceholderRegistry() + registry.placeholder_for("mario.rossi@example.com", "EMAIL_ADDRESS", page=0) + payload = build_restore_payload( + source_name="input.pdf", + source_sha256="a" * 64, + output_sha256=None, + entries=registry.entries(), + ) + + encrypted = encrypt_restore_payload(payload, "Password lunga 123!") + decrypted = decrypt_restore_payload(encrypted, "Password lunga 123!") + + assert b"mario.rossi@example.com" not in encrypted + assert decrypted == payload + + +def test_restore_map_requires_strong_enough_password(): + with pytest.raises(ValueError): + encrypt_restore_payload({}, "short") diff --git a/tests/test_validation.py b/tests/test_validation.py index 0c00237..80967a6 100644 --- a/tests/test_validation.py +++ b/tests/test_validation.py @@ -13,7 +13,12 @@ sys.path.insert(0, str(pathlib.Path(__file__).parent.parent)) -from NullifyPDF import PDFListManager, resource_path +from NullifyPDF import ( + PDFListManager, + find_tessdata_dir, + ocr_language_for_choice, + resource_path, +) class TestPDFListManager: @@ -86,6 +91,26 @@ def test_resource_path_not_empty(self): assert len(result) > 0 +class TestOcrConfiguration: + """Test OCR configuration helpers.""" + + def test_ocr_language_without_tessdata_uses_requested_languages(self): + assert ocr_language_for_choice("EN", None) == "eng" + assert ocr_language_for_choice("IT", None) == "ita" + assert ocr_language_for_choice("BOTH", None) == "eng+ita" + + def test_ocr_language_filters_to_installed_languages(self, tmp_path): + (tmp_path / "eng.traineddata").write_text("placeholder", encoding="utf-8") + + assert ocr_language_for_choice("BOTH", str(tmp_path)) == "eng" + + def test_find_tessdata_dir_reads_environment(self, tmp_path, monkeypatch): + (tmp_path / "ita.traineddata").write_text("placeholder", encoding="utf-8") + monkeypatch.setenv("TESSDATA_PREFIX", str(tmp_path)) + + assert find_tessdata_dir() == str(tmp_path) + + @pytest.mark.parametrize("invalid_input", [None, "", 123, []]) def test_list_manager_handles_invalid_paths(invalid_input): """PDFListManager should not crash on invalid paths.""" From f091408cf9799d2113686d0ccef09399f01e84a3 Mon Sep 17 00:00:00 2001 From: 0verwrite <31691645+overwrite00@users.noreply.github.com> Date: Thu, 23 Jul 2026 19:54:11 +0200 Subject: [PATCH 2/3] fix: repair beta release CI and prepare 2.1.0-beta.2 --- CHANGELOG.md | 4 ++-- NullifyPDF.py | 2 +- build_local.py | 4 ++-- tests/conftest.py | 10 ++++++++++ 4 files changed, 15 insertions(+), 5 deletions(-) create mode 100644 tests/conftest.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 317a846..6506e8b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,9 +7,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] -## [2.1.0-beta.1] - 2026-07-23 +## [2.1.0-beta.2] - 2026-07-23 -Prima beta pubblica della linea 2.1.0, focalizzata su privacy export avanzato, OCR completo per PDF scansionati e nuove varianti di release Lite/Full. +First public beta of the 2.1.0 line, focused on advanced privacy export, full OCR support for scanned PDFs, and new Lite/Full release variants. ### ✨ Added diff --git a/NullifyPDF.py b/NullifyPDF.py index 8844136..de638ca 100644 --- a/NullifyPDF.py +++ b/NullifyPDF.py @@ -57,7 +57,7 @@ ) __version__ = "2.1.0" -__version_prerelease__ = "beta.1" +__version_prerelease__ = "beta.2" APP_VERSION = ( f"{__version__}-{__version_prerelease__}" if __version_prerelease__ diff --git a/build_local.py b/build_local.py index a3293d5..4e7d49f 100644 --- a/build_local.py +++ b/build_local.py @@ -163,7 +163,7 @@ def build_rpm( version: Application version used for the RPM package metadata (must not contain hyphens, which the RPM Version tag disallows). file_version: Version string used for the output artifact filename - (may include a beta suffix, e.g. "2.1.0-beta.1"). + (may include a beta suffix, e.g. "2.1.0-beta.2"). executable_name: Name of compiled executable. """ print("\n[INFO] Creazione pacchetto RPM per Fedora/RHEL...") @@ -249,7 +249,7 @@ def build_deb( Args: version: Application version used for the DEB package metadata. file_version: Version string used for the output artifact filename - (may include a beta suffix, e.g. "2.1.0-beta.1"). + (may include a beta suffix, e.g. "2.1.0-beta.2"). executable_name: Name of compiled executable. """ print("\n[INFO] Creazione pacchetto DEB per Ubuntu/Debian...") diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..3e809b8 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,10 @@ +"""Pytest configuration shared by the test suite.""" + +import pathlib +import sys + + +PROJECT_ROOT = pathlib.Path(__file__).resolve().parent.parent +root_str = str(PROJECT_ROOT) +if root_str not in sys.path: + sys.path.insert(0, root_str) From 24ccf2c172bcd818045b39e4cd3fb40a5cdcf4be Mon Sep 17 00:00:00 2001 From: 0verwrite <31691645+overwrite00@users.noreply.github.com> Date: Thu, 23 Jul 2026 20:06:24 +0200 Subject: [PATCH 3/3] docs: translate automated release notes to English --- .github/workflows/beta-release.yml | 2 +- .github/workflows/release.yml | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/beta-release.yml b/.github/workflows/beta-release.yml index cdee2f3..f24e355 100644 --- a/.github/workflows/beta-release.yml +++ b/.github/workflows/beta-release.yml @@ -95,6 +95,6 @@ jobs: all-builds/NullifyPDF_v*_macOS_*.zip all-builds/NullifyPDF_v*_Ubuntu_*.deb all-builds/NullifyPDF_v*_Fedora_*.rpm - body: "Beta release NullifyPDF (build da develop). Include varianti Lite e Full OCR. Non usare in produzione. Se supera i test, questa build verra' promossa a stable senza ricompilazione." + body: "NullifyPDF beta release (build from develop). Includes Lite and Full OCR variants. Do not use in production. If validation succeeds, this build will be promoted to stable without rebuilding." env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 899714b..a737452 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -73,8 +73,8 @@ jobs: files: stable-assets/* prerelease: false body: | - Release Stabile NullifyPDF. - Promossa dalla beta `${{ steps.beta.outputs.tag }}` (build gia' verificata, nessuna ricompilazione). - Include varianti Lite e Full OCR per Windows, macOS e Linux. + NullifyPDF stable release. + Promoted from beta `${{ steps.beta.outputs.tag }}` (already verified build, no rebuild). + Includes Lite and Full OCR variants for Windows, macOS, and Linux. env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}