diff --git a/.github/workflows/python-app.yml b/.github/workflows/python-app.yml index 51d6d7b..77ef9ce 100644 --- a/.github/workflows/python-app.yml +++ b/.github/workflows/python-app.yml @@ -5,9 +5,9 @@ name: Python application on: push: - branches: [ "master", "develop"] + branches: [ "main", "develop", "CI-CD"] pull_request: - branches: [ "master", "develop"] + branches: [ "main", "develop"] permissions: contents: read @@ -19,22 +19,27 @@ jobs: steps: - uses: actions/checkout@v4 + - name: Set up Python 3.11 uses: actions/setup-python@v3 with: python-version: "3.11" + - name: Install dependencies run: | python -m pip install --upgrade pip - pip install flake8 pytest - # if [ -f requirements.txt ]; then pip install -r requirements.txt; fi - pip install . - - name: Lint with flake8 - run: | - # stop the build if there are Python syntax errors or undefined names - flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics - # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide - flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics + pip install .[test] ruff + + - name: Lint - fail on real issues + run: ruff check . --select F,E9 + + - name: Lint - style (non-blocking) + run: ruff check . --select E,B,I || true + + ## ADD LATER + # - name: Format check + # run: ruff format --check . + - name: Test with pytest run: | pytest diff --git a/.gitignore b/.gitignore index f579ca4..bd1d077 100644 --- a/.gitignore +++ b/.gitignore @@ -28,6 +28,17 @@ build/ *.tar *.zip +# Unit test / coverage reports # +################################ +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.* +*,cover + # Logs and databases # ###################### *.ipynb_checkpoints diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..bd0b11e --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,25 @@ + +default_language_version: + python: python3.11 + +repos: + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.4.4 + hooks: + - id: ruff + args: [--fix] + - id: ruff-format + + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.6.0 + hooks: + - id: trailing-whitespace + - id: end-of-file-fixer + - id: check-yaml + - id: check-toml + - id: debug-statements + + - repo: https://github.com/rhysd/actionlint + rev: v1.7.0 + hooks: + - id: actionlint diff --git a/docs/source/conf.py b/docs/source/conf.py index 39a5448..031c739 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -7,8 +7,6 @@ import sys sys.path.insert(0, os.path.abspath('../../src/perpl')) # Adjust to include your package -import sphinx_rtd_theme - # -- Project information ----------------------------------------------------- # https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information diff --git a/notebooks/make_ACTN2_Affimer_PERPL_plots.py b/notebooks/make_ACTN2_Affimer_PERPL_plots.py index fd5335d..76b250c 100644 --- a/notebooks/make_ACTN2_Affimer_PERPL_plots.py +++ b/notebooks/make_ACTN2_Affimer_PERPL_plots.py @@ -6,7 +6,7 @@ # extension: .py # format_name: light # format_version: '1.5' -# jupytext_version: 1.19.2 +# jupytext_version: 1.19.3 # kernelspec: # display_name: Python 3 (ipykernel) # language: python @@ -24,9 +24,10 @@ # ## Imports import numpy as np -import pandas as pd +# import pandas as pd import matplotlib.pyplot as plt -from perpl.modelling import modelling_general, zdisk_modelling, zdisk_plots +from perpl.modelling import zdisk_modelling, zdisk_plots +# from perpl.modelling import modelling_general from perpl.io import plotting # ## Set average estimated localisation precision for Affimer and PALM data. diff --git a/notebooks/make_Nup107_Z_PERPL_plots.py b/notebooks/make_Nup107_Z_PERPL_plots.py index 0326a8a..232cab4 100644 --- a/notebooks/make_Nup107_Z_PERPL_plots.py +++ b/notebooks/make_Nup107_Z_PERPL_plots.py @@ -6,7 +6,7 @@ # extension: .py # format_name: light # format_version: '1.5' -# jupytext_version: 1.17.1 +# jupytext_version: 1.19.3 # kernelspec: # display_name: Python 3 (ipykernel) # language: python @@ -29,7 +29,6 @@ import matplotlib.pyplot as plt from perpl.modelling import modelling_general, two_layer_fitting, background_models -from perpl.statistics import modelstats from perpl.statistics.modelstats import akaike_weights # - @@ -165,7 +164,6 @@ weights = akaike_weights([1.]) print(weights) -# + [markdown] jp-MarkdownHeadingCollapsed=true # ## Plot model components # + diff --git a/notebooks/make_mEos_LASP2andMYPN_PERPL_plots.py b/notebooks/make_mEos_LASP2andMYPN_PERPL_plots.py index b4222f8..25b77a3 100644 --- a/notebooks/make_mEos_LASP2andMYPN_PERPL_plots.py +++ b/notebooks/make_mEos_LASP2andMYPN_PERPL_plots.py @@ -6,7 +6,7 @@ # extension: .py # format_name: light # format_version: '1.5' -# jupytext_version: 1.17.1 +# jupytext_version: 1.19.3 # kernelspec: # display_name: Python 3 (ipykernel) # language: python @@ -27,7 +27,7 @@ import numpy as np import matplotlib.pyplot as plt -from perpl.modelling import modelling_general, zdisk_modelling, zdisk_plots +from perpl.modelling import zdisk_modelling, zdisk_plots from perpl.io import plotting # - diff --git a/pyproject.toml b/pyproject.toml index adf82af..dd764db 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -2,12 +2,9 @@ requires = ["setuptools>=64.0", "setuptools-scm>=8.0"] build-backend = "setuptools.build_meta" -[tool.setuptools_scm] - [project] name = "perpl" version = "1.3.0" -# dynamic = ["version"] authors = [ { name="Alistair Curd", email="a.curd@leeds.ac.uk" } ] @@ -41,4 +38,28 @@ modeldistances = "perpl.run_distance_modelling:main" Homepage = "https://github.com/AlistairCurd/PERPL-Python3" Issues = "https://github.com/AlistairCurd/PERPL-Python3/issues" -# [tool.tox], [tox] ? \ No newline at end of file + +[project.optional-dependencies] +test = [ + "pytest~=8.0", + "pytest-cov~=5.0" +] + +dev = [ + "perpl[test]", + "ruff~=0.4", + "pre-commit~=3.7" +] + +[tool.pytest.ini_options] +addopts = "-v --color=yes --cov=perpl --cov-report=term --cov-report=xml --cov-report=html" + +[tool.ruff] +line-length = 88 +target-version = "py311" + +[tool.ruff.lint] +select = ["E", "F", "B", "I"] + +[tool.ruff.format] +quote-style = "double" diff --git a/src/perpl/modelling/centriole_analysis.py b/src/perpl/modelling/centriole_analysis.py index 61a4077..a505e48 100644 --- a/src/perpl/modelling/centriole_analysis.py +++ b/src/perpl/modelling/centriole_analysis.py @@ -24,16 +24,17 @@ """ import time -from scipy.ndimage.filters import gaussian_filter -from scipy.io import loadmat -import numpy as np + import matplotlib.pyplot as plt +import numpy as np import pandas as pd +from scipy.io import loadmat +from scipy.ndimage.filters import gaussian_filter import perpl.modelling.modelling_general as models -from perpl.modelling.modelling_general import ModelWithFitSettings -from relative_positions import getdistances from perpl.modelling.background_models import pair_correlation_disk +from perpl.modelling.modelling_general import ModelWithFitSettings +from perpl.relative_positions import getdistances def get_input_data( @@ -1030,7 +1031,7 @@ def make_fit_results_table_variable_vertices_model_9fold_no_bg( # Populate the tables max_fit_distance = len(distance_histograms_list[0]) - for i, histogram in enumerate(distance_histograms_list): + for i, _ in enumerate(distance_histograms_list): # Do the fit (params_optimised, params_covar, params_1sd_error) = ( models.fit_model_to_experiment( @@ -1158,7 +1159,8 @@ def main(): np.save(outfile_dist_hist_list, distance_histograms_list) # Choose model - # model_with_info = set_up_variable_vertices_model_9fold_internal_bg_with_fit_settings() + # model_with_info = \ + # set_up_variable_vertices_model_9fold_internal_bg_with_fit_settings() model_with_info = set_up_variable_vertices_model_9fold_no_bg_with_fit_settings() # Could modify the fit parameter settings here @@ -1217,7 +1219,8 @@ def main(): ) ) # estimated_params_df.to_csv( - # 'C:/Temp/Centriole/5nm_precision_distance_histograms_normalised_parameter_estimates.csv', + # "C:/Temp/Centriole/" + # + "5nm_precision_distance_histograms_normalised_parameter_estimates.csv", # index=False # ) @@ -1258,14 +1261,14 @@ def main(): # model_with_info.vector_input_model) # For model without internal background - stdev = plot_95_ci( - axes, - x_values, - model_with_info.model_rpd, - params_optimised, - params_covar, - model_with_info.vector_input_model, - ) + # stdev = plot_95_ci( + # axes, + # x_values, + # model_with_info.model_rpd, + # params_optimised, + # params_covar, + # model_with_info.vector_input_model, + #) axes.set_title( "Centriole " + repr(i) @@ -1309,14 +1312,14 @@ def main(): fitted_curve_values = model_with_info.model_rpd(x_values, *params_optimised) axes.plot(x_values, fitted_curve_values, color="xkcd:red", lw=0.5) - stdev = plot_95_ci( - axes, - x_values, - model_with_info.model_rpd, - params_optimised, - params_covar, - model_with_info.vector_input_model, - ) + # stdev = plot_95_ci( + # axes, + # x_values, + # model_with_info.model_rpd, + # params_optimised, + # params_covar, + # model_with_info.vector_input_model, + #) axes.set_title("All centrioles") diff --git a/src/perpl/modelling/dna_paint_data_fitting.py b/src/perpl/modelling/dna_paint_data_fitting.py index 47a1bb9..b5289eb 100644 --- a/src/perpl/modelling/dna_paint_data_fitting.py +++ b/src/perpl/modelling/dna_paint_data_fitting.py @@ -32,14 +32,17 @@ """ -from scipy.optimize import curve_fit -import numpy as np import matplotlib.pyplot as plt +import numpy as np +from scipy.optimize import curve_fit + +import perpl.modelling.modelling_general as models import perpl.modelling.polyhedramodelling as poly import perpl.statistics.modelstats as stats -import perpl.modelling.modelling_general as models -from perpl.modelling.modelling_general import ModelWithFitSettings -from perpl.modelling.modelling_general import stdev_of_model +from perpl.modelling.modelling_general import ( + ModelWithFitSettings, + stdev_of_model, +) def create_default_fitting_params_dicts(): @@ -937,7 +940,7 @@ def plot_model_components_tri_prism( """ distance_values = np.arange(0, fitlength + 1, 1) - fig = plt.figure() + fig = plt.figure() # noqa: F841 # Prefer OO style, but not worth changing atm. axes = plt.subplot(111) axes.set_xlim([0, fitlength]) axes.set_xlabel(r"$\Delta$XYZ (nm)") diff --git a/src/perpl/modelling/zdisk_plots.py b/src/perpl/modelling/zdisk_plots.py index 815c1cf..3fd6dc6 100644 --- a/src/perpl/modelling/zdisk_plots.py +++ b/src/perpl/modelling/zdisk_plots.py @@ -22,27 +22,27 @@ specific language governing permissions and limitations under the License. """ +import matplotlib.pyplot as plt import numpy as np import pandas as pd -import matplotlib.pyplot as plt from scipy.signal import find_peaks import perpl.modelling.linearrepeatmodels as linmods import perpl.modelling.zdisk_modelling as zdisk_modelling -from perpl.modelling.modelling_general import kde_1nm -from perpl.modelling.modelling_general import pairwise_correlation_1d -from perpl.modelling.modelling_general import stdev_of_model from perpl.io.plotting import estimate_rpd_churchman_1d -from perpl.modelling.zdisk_modelling import read_relpos_from_pickles -from perpl.modelling.zdisk_modelling import getaxialseparations_no_smoothing -from perpl.modelling.zdisk_modelling import remove_duplicates -from perpl.modelling.zdisk_modelling import ( - set_up_model_4_variable_peaks_with_fit_settings, +from perpl.modelling.modelling_general import ( + kde_1nm, + pairwise_correlation_1d, + stdev_of_model, ) from perpl.modelling.zdisk_modelling import ( + fitmodel_to_hist, + getaxialseparations_no_smoothing, + # read_relpos_from_pickles, + remove_duplicates, + # set_up_model_4_variable_peaks_with_fit_settings, set_up_model_5_variable_peaks_with_fit_settings, ) -from perpl.modelling.zdisk_modelling import fitmodel_to_hist def plot_distance_hist( @@ -80,7 +80,7 @@ def plot_distance_hist( histaxes.set_ylim(bottom=0) histaxes.set_title("Histogram") histaxes.set_xlabel( - f"$\Delta$Axial (nm) ($\Delta$Transverse < {transverse_limit} nm)" + rf"$\Delta$Axial (nm) ($\Delta$Transverse < {transverse_limit} nm)" ) histaxes.set_ylabel("Counts") if close_plots: diff --git a/src/perpl/run_distance_modelling.py b/src/perpl/run_distance_modelling.py index af64436..4816a2e 100644 --- a/src/perpl/run_distance_modelling.py +++ b/src/perpl/run_distance_modelling.py @@ -1,13 +1,12 @@ import argparse import datetime -from itertools import product import os import sys +from itertools import product import matplotlib.pyplot as plt import numpy as np import pandas as pd - import yaml from perpl.io import plotting @@ -15,15 +14,15 @@ from perpl.modelling.modelling_general import PERPLModel - -def get_experimental_rpd(distances, rpd_type, fitlength, - bin_size, kde_kernel_size, model_config): +def get_experimental_rpd( + distances, rpd_type, fitlength, bin_size, kde_kernel_size, model_config +): """ Args: ... rpd_type (str): "distance_histogram" or "distance_kde" - + Returns: x_expt, y_expt (1D numpy arrays): Values of the distance histogram or KDE (y_expt) @@ -86,8 +85,12 @@ def model_the_data( model_name = model_file.rstrip(".yaml") x_expt, y_expt = get_experimental_rpd( - distances, rpd_type, fitlength, - bin_size, kde_kernel_size, model_config, + distances, + rpd_type, + fitlength, + bin_size, + kde_kernel_size, + model_config, ) perpl_model = PERPLModel( @@ -111,9 +114,10 @@ def model_the_data( and model_config["n_peaks"] == 0 and model_config["repeats"] is False ): - print(f"Skipping {model_name}:" - " contains no characteristic distances," - " repeated localisations or background" + print( + f"Skipping {model_name}:" + " contains no characteristic distances," + " repeated localisations or background" ) return None @@ -137,9 +141,7 @@ def model_the_data( ) figname = os.path.join( output_folder, - ( - f"{model_name}_fitlength_{fitlength}_binsize_{bin_size}_histandfit.svg" - ), + (f"{model_name}_fitlength_{fitlength}_binsize_{bin_size}_histandfit.svg"), ) elif rpd_type == "distance_kde": @@ -147,9 +149,7 @@ def model_the_data( fig = perpl_model.plot_distance_kde_and_fit(x_expt, y_expt, fitlength) figname = os.path.join( output_folder, - ( - f"{model_name}_fitlength_{fitlength}_kdeandfit.svg" - ), + (f"{model_name}_fitlength_{fitlength}_kdeandfit.svg"), ) if fig is not None: @@ -168,9 +168,7 @@ def model_the_data( elif rpd_type == "distance_kde": figname = os.path.join( output_folder, - ( - f"{model_name}_fitlength_{fitlength}_modelcomponents.svg" - ), + (f"{model_name}_fitlength_{fitlength}_modelcomponents.svg"), ) if fig2 is not None: fig2.savefig(figname) @@ -197,6 +195,7 @@ def model_the_data( perpl_model.param_names, perpl_model.params_optimised, perpl_model.params_err, + strict=True, ): f.write(f"{row[0]}: {row[1]} +- {row[2]}\n") @@ -257,10 +256,8 @@ def main(argv=None): args = parser.parse_args(argv) if not (args.fit_histograms or args.fit_kdes or args.no_fitting): - parser.error("Must specify at least one of" - " --fh, --fkde or -nofit" - ) - + parser.error("Must specify at least one of" " --fh, --fkde or -nofit") + print(f"Fit hists: {args.fit_histograms}") print(f"Fit KDEs: {args.fit_kdes}") @@ -270,7 +267,7 @@ def main(argv=None): # Collect RPD types to run run_modes = [] - + if args.fit_histograms: run_modes.append("distance_histogram") if args.fit_kdes: @@ -316,9 +313,7 @@ def main(argv=None): model_configs = [] for model_file in model_files: - with open( - os.path.join(models_folder, model_file), "r" - ) as ymlfile: + with open(os.path.join(models_folder, model_file), "r") as ymlfile: config = yaml.safe_load(ymlfile) model_configs.append(config) @@ -327,7 +322,7 @@ def main(argv=None): return # Set up output parent folder - ## Contains table of results and subdirector(ies) for fits + ## Contains table of results and subdirectory(ies) for fits parent, _ = os.path.split(args.rel_posns_file) output_modelling_folder = os.path.join( @@ -337,7 +332,6 @@ def main(argv=None): # For histoagrmmed and kde data separately for rpd_type in run_modes: - # Create output locations; if rpd_type == "distance_histogram": out_folder = "histogram_fits" @@ -345,13 +339,9 @@ def main(argv=None): elif rpd_type == "distance_kde": out_folder = "kde_fits" results_file = "results_kde.csv" - out_folder_path = os.path.join( - output_modelling_folder, out_folder - ) + out_folder_path = os.path.join(output_modelling_folder, out_folder) os.makedirs(out_folder_path) - results_path = os.path.join( - output_modelling_folder, results_file - ) + results_path = os.path.join(output_modelling_folder, results_file) print(f"Output folder: {out_folder_path}") @@ -373,8 +363,8 @@ def main(argv=None): count = 0 for bin_or_kernel, fitlength in product( - bin_or_kernel_list, - fitlength_lst, + bin_or_kernel_list, + fitlength_lst, ): if rpd_type == "distance_histogram": bin_size = bin_or_kernel @@ -382,9 +372,9 @@ def main(argv=None): else: bin_size = None kde_kernel_size = bin_or_kernel - + for model_file, model_config in zip( - model_files, model_configs + model_files, model_configs, strict=True ): model_out = model_the_data( distances, @@ -394,31 +384,37 @@ def main(argv=None): kde_kernel_size, bin_size, fitlength, - out_folder_path + out_folder_path, ) if model_out is None: continue x_expt, y_expt, fitted_model = model_out - - model_results.append({ - "Name": model_file.rstrip(".yaml"), - "Fit length": fitlength, - x_kernel_col: bin_or_kernel, - "N Peaks": model_config["n_peaks"], - "Peak ratios": model_config["peak_type"], - "Model distance ratios": model_config["characteristic_distance"], - "Repeated localisations": model_config["repeats"], - "Background model": model_config["background"], - "AICc": fitted_model.aic_corrected, - "AIC": fitted_model.aic, - "SSR": fitted_model.sum_of_squares_error, - "N params": fitted_model.n_params, - "N calculation points": len(x_expt), - "N measured distances": len(distances), - "Check: bg < 0": fitted_model.bgbelowzero, - "Check: params reached limits": fitted_model.popt_at_bound, - "Check: large param uncertainty": fitted_model.large_uncertainty - }) + + model_results.append( + { + "Name": model_file.rstrip(".yaml"), + "Fit length": fitlength, + x_kernel_col: bin_or_kernel, + "N Peaks": model_config["n_peaks"], + "Peak ratios": model_config["peak_type"], + "Model distance ratios": model_config[ + "characteristic_distance" + ], + "Repeated localisations": model_config["repeats"], + "Background model": model_config["background"], + "AICc": fitted_model.aic_corrected, + "AIC": fitted_model.aic, + "SSR": fitted_model.sum_of_squares_error, + "N params": fitted_model.n_params, + "N calculation points": len(x_expt), + "N measured distances": len(distances), + "Check: bg < 0": fitted_model.bgbelowzero, + "Check: params reached limits": fitted_model.popt_at_bound, + "Check: large param uncertainty": ( + fitted_model.large_uncertainty, + ), + } + ) if (count + 1) % 10 == 0 and count > 0: print(f"{count + 1} models run out of {len(model_files)}...")