diff --git a/FirstClassErrors.PropertyTests/ErrorCodePropertyTests.cs b/FirstClassErrors.PropertyTests/ErrorCodePropertyTests.cs new file mode 100644 index 0000000..8493033 --- /dev/null +++ b/FirstClassErrors.PropertyTests/ErrorCodePropertyTests.cs @@ -0,0 +1,63 @@ +#region Usings declarations + +using FsCheck; +using FsCheck.Fluent; + +using JetBrains.Annotations; + +#endregion + +namespace FirstClassErrors.PropertyTests; + +/// +/// Property-based tests for . They assert the invariants that must hold for every +/// valid code rather than for a handful of hand-picked examples: an error code is a verbatim, order-sensitive +/// value that round-trips through its textual representations. +/// +[TestSubject(typeof(ErrorCode))] +public sealed class ErrorCodePropertyTests { + + [Fact(DisplayName = "ErrorCode.Create preserves the code verbatim through ToString (no normalization).")] + public void CreatePreservesCodeThroughToString() { + Prop.ForAll(Generators.NonBlank().ToArbitrary(), code => ErrorCode.Create(code).ToString() == code) + .QuickCheckThrowOnFailure(); + } + + [Fact(DisplayName = "ErrorCode.Create preserves the code verbatim through the implicit string conversion.")] + public void CreatePreservesCodeThroughImplicitString() { + Prop.ForAll(Generators.NonBlank().ToArbitrary(), code => (string)ErrorCode.Create(code) == code) + .QuickCheckThrowOnFailure(); + } + + [Fact(DisplayName = "Two ErrorCodes are equal if and only if their codes are ordinally equal.")] + public void EqualityIsOrdinal() { + Gen code = Generators.NonBlank(); + var pairs = (from left in code + from right in code + select (left, right)).ToArbitrary(); + + Prop.ForAll(pairs, + pair => (ErrorCode.Create(pair.left) == ErrorCode.Create(pair.right)) + == string.Equals(pair.left, pair.right, StringComparison.Ordinal)) + .QuickCheckThrowOnFailure(); + } + + [Fact(DisplayName = "Codes built from the same string share the same hash code.")] + public void EqualCodesShareHashCode() { + Prop.ForAll(Generators.NonBlank().ToArbitrary(), + code => ErrorCode.Create(code).GetHashCode() == ErrorCode.Create(code).GetHashCode()) + .QuickCheckThrowOnFailure(); + } + + [Fact(DisplayName = "ErrorCode.Create rejects any blank code with an ArgumentException.")] + public void CreateRejectsBlankCodes() { + Prop.ForAll(Generators.Blank().ToArbitrary(), blank => Expect.Throws(() => ErrorCode.Create(blank))) + .QuickCheckThrowOnFailure(); + } + + [Fact(DisplayName = "ErrorCode.Create rejects a null code with an ArgumentException.")] + public void CreateRejectsNullCode() { + Assert.Throws(() => ErrorCode.Create(null!)); + } + +} diff --git a/FirstClassErrors.PropertyTests/ErrorDescriptionPropertyTests.cs b/FirstClassErrors.PropertyTests/ErrorDescriptionPropertyTests.cs new file mode 100644 index 0000000..031d1d9 --- /dev/null +++ b/FirstClassErrors.PropertyTests/ErrorDescriptionPropertyTests.cs @@ -0,0 +1,95 @@ +#region Usings declarations + +using FsCheck; +using FsCheck.Fluent; + +using JetBrains.Annotations; + +#endregion + +namespace FirstClassErrors.PropertyTests; + +/// +/// Property-based tests for , focusing on its normalization contract: the +/// mandatory messages are trimmed and required to be non-blank, while the optional detailed message collapses +/// to null whenever it is blank. +/// +[TestSubject(typeof(ErrorDescription))] +public sealed class ErrorDescriptionPropertyTests { + + [Fact(DisplayName = "The mandatory messages are trimmed and never keep surrounding whitespace.")] + public void MandatoryMessagesAreTrimmed() { + Gen text = Generators.NonBlank(); + var inputs = (from shortMessage in text + from diagnostic in text + select (shortMessage, diagnostic)).ToArbitrary(); + + Prop.ForAll(inputs, + input => { + ErrorDescription description = new(input.shortMessage, input.diagnostic); + + return description.ShortMessage == input.shortMessage.Trim() + && description.DiagnosticMessage == input.diagnostic.Trim() + && description.ShortMessage == description.ShortMessage.Trim() + && description.DiagnosticMessage == description.DiagnosticMessage.Trim(); + }) + .QuickCheckThrowOnFailure(); + } + + [Fact(DisplayName = "Re-describing with already-trimmed messages is idempotent.")] + public void NormalizationIsIdempotent() { + Gen text = Generators.NonBlank(); + var inputs = (from shortMessage in text + from diagnostic in text + select (shortMessage, diagnostic)).ToArbitrary(); + + Prop.ForAll(inputs, + input => { + ErrorDescription once = new(input.shortMessage, input.diagnostic); + ErrorDescription twice = new(once.ShortMessage, once.DiagnosticMessage); + + return once.ShortMessage == twice.ShortMessage && once.DiagnosticMessage == twice.DiagnosticMessage; + }) + .QuickCheckThrowOnFailure(); + } + + [Fact(DisplayName = "A blank detailed message collapses to null; a non-blank one is trimmed.")] + public void DetailedMessageCollapsesWhenBlank() { + Gen text = Generators.NonBlank(); + Gen optionalDetail = Gen.OneOf(Generators.NonBlank().Select(value => (string?)value), + Generators.Blank().Select(value => (string?)value), + Gen.Constant((string?)null)); + var inputs = (from shortMessage in text + from diagnostic in text + from detail in optionalDetail + select (shortMessage, diagnostic, detail)).ToArbitrary(); + + Prop.ForAll(inputs, + input => { + ErrorDescription description = new(input.shortMessage, input.diagnostic, input.detail); + string? expected = string.IsNullOrWhiteSpace(input.detail) ? null : input.detail.Trim(); + + return description.DetailedMessage == expected; + }) + .QuickCheckThrowOnFailure(); + } + + [Fact(DisplayName = "A blank short message is rejected with an ArgumentException.")] + public void BlankShortMessageIsRejected() { + Prop.ForAll(Generators.Blank().ToArbitrary(), blank => Expect.Throws(() => new ErrorDescription(blank, "diagnostic"))) + .QuickCheckThrowOnFailure(); + } + + [Fact(DisplayName = "A blank diagnostic message is rejected with an ArgumentException.")] + public void BlankDiagnosticMessageIsRejected() { + Prop.ForAll(Generators.Blank().ToArbitrary(), blank => Expect.Throws(() => new ErrorDescription("short", blank))) + .QuickCheckThrowOnFailure(); + } + + [Fact(DisplayName = "A null mandatory message is rejected with an ArgumentNullException.")] + public void NullMandatoryMessageIsRejected() { + Assert.Throws(() => new ErrorDescription(null!, "diagnostic")); + Assert.Throws(() => new ErrorDescription("short", null!)); + } + +} diff --git a/FirstClassErrors.PropertyTests/FirstClassErrors.PropertyTests.csproj b/FirstClassErrors.PropertyTests/FirstClassErrors.PropertyTests.csproj new file mode 100644 index 0000000..43ae738 --- /dev/null +++ b/FirstClassErrors.PropertyTests/FirstClassErrors.PropertyTests.csproj @@ -0,0 +1,35 @@ + + + + net10.0 + enable + enable + false + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + + + + + + + + + diff --git a/FirstClassErrors.PropertyTests/OutcomeMonadLawTests.cs b/FirstClassErrors.PropertyTests/OutcomeMonadLawTests.cs new file mode 100644 index 0000000..7b3ca1b --- /dev/null +++ b/FirstClassErrors.PropertyTests/OutcomeMonadLawTests.cs @@ -0,0 +1,198 @@ +#region Usings declarations + +using FsCheck; +using FsCheck.Fluent; + +using JetBrains.Annotations; + +#endregion + +namespace FirstClassErrors.PropertyTests; + +/// +/// Property-based tests asserting that obeys the monad laws and that a failure +/// propagates the original instance unchanged. These are the richest behavioural +/// invariants in the library, and the ones most worth checking against a wide range of randomly generated +/// pipelines rather than a few hand-written examples. +/// +/// +/// Two outcomes are compared with : successes by their carried value, failures by +/// reference identity of their error. Reference identity is deliberate — the library propagates the +/// very same instance through Then/To/Recover, and these tests +/// lock that in. +/// +[TestSubject(typeof(Outcome<>))] +public sealed class OutcomeMonadLawTests { + + [Fact(DisplayName = "Left identity: Success(a).Then(f) is equivalent to f(a).")] + public void LeftIdentity() { + var inputs = (from value in Ints() + from step in Steps() + select (value, step)).ToArbitrary(); + + Prop.ForAll(inputs, + input => { + Func> f = ToFunction(input.step); + + return Equivalent(Outcome.Success(input.value).Then(f), f(input.value)); + }) + .QuickCheckThrowOnFailure(); + } + + [Fact(DisplayName = "Right identity: m.Then(Success) is equivalent to m.")] + public void RightIdentity() { + Prop.ForAll(Outcomes().ToArbitrary(), outcome => Equivalent(outcome.Then(value => Outcome.Success(value)), outcome)) + .QuickCheckThrowOnFailure(); + } + + [Fact(DisplayName = "Associativity: m.Then(f).Then(g) is equivalent to m.Then(x => f(x).Then(g)).")] + public void Associativity() { + var inputs = (from outcome in Outcomes() + from first in Steps() + from second in Steps() + select (outcome, first, second)).ToArbitrary(); + + Prop.ForAll(inputs, + input => { + Func> f = ToFunction(input.first); + Func> g = ToFunction(input.second); + + return Equivalent(input.outcome.Then(f).Then(g), input.outcome.Then(value => f(value).Then(g))); + }) + .QuickCheckThrowOnFailure(); + } + + [Fact(DisplayName = "Then on a failure short-circuits and propagates the original error instance.")] + public void ThenPropagatesTheOriginalErrorOnFailure() { + Prop.ForAll(Steps().ToArbitrary(), + step => { + Error error = AnError("origin"); + Func> f = ToFunction(step); + Outcome result = Outcome.Failure(error).Then(f); + + return result.IsFailure && ReferenceEquals(result.Error, error); + }) + .QuickCheckThrowOnFailure(); + } + + [Fact(DisplayName = "Functor identity: m.To(x => x) is equivalent to m.")] + public void FunctorIdentity() { + Prop.ForAll(Outcomes().ToArbitrary(), outcome => Equivalent(outcome.To(value => value), outcome)) + .QuickCheckThrowOnFailure(); + } + + [Fact(DisplayName = "To is a special case of Then: m.To(f) equals m.Then(x => Success(f(x))).")] + public void MapIsBindWithSuccess() { + var inputs = (from outcome in Outcomes() + from delta in Ints() + select (outcome, delta)).ToArbitrary(); + + Prop.ForAll(inputs, + input => Equivalent(input.outcome.To(value => value + input.delta), + input.outcome.Then(value => Outcome.Success(value + input.delta)))) + .QuickCheckThrowOnFailure(); + } + + [Fact(DisplayName = "Recover on a success returns the same instance and never runs the fallback.")] + public void RecoverOnSuccessIsIdentity() { + Prop.ForAll(Ints().ToArbitrary(), + value => { + Outcome success = Outcome.Success(value); + Outcome recovered = success.Recover(_ => Outcome.Failure(AnError("fallback"))); + + return ReferenceEquals(recovered, success); + }) + .QuickCheckThrowOnFailure(); + } + + [Fact(DisplayName = "Recover on a failure runs the fallback with the original error.")] + public void RecoverOnFailureAppliesTheFallback() { + Prop.ForAll(Ints().ToArbitrary(), + fallbackValue => { + Error error = AnError("origin"); + Error? observed = null; + + Outcome recovered = Outcome.Failure(error) + .Recover(caught => { + observed = caught; + + return Outcome.Success(fallbackValue); + }); + + return recovered.IsSuccess && recovered.GetResultOrThrow() == fallbackValue && ReferenceEquals(observed, error); + }) + .QuickCheckThrowOnFailure(); + } + + [Fact(DisplayName = "Finally folds a success to onSuccess(value) and a failure to onFailure(error).")] + public void FinallyFoldsBothCases() { + Prop.ForAll(Outcomes().ToArbitrary(), + outcome => { + (bool handledSuccess, int value) folded = outcome.Finally(value => (true, value), _ => (false, 0)); + + return outcome.IsSuccess ? folded == (true, outcome.GetResultOrThrow()) : folded == (false, 0); + }) + .QuickCheckThrowOnFailure(); + } + + #region Statics members declarations + + /// + /// Bounded integers, kept well inside range so that the additions performed by the + /// generated steps never overflow. + /// + private static Gen Ints() { + return Gen.Choose(-1_000_000, 1_000_000); + } + + /// + /// Generates the specification of a monadic step: how much it adds, whether it fails, and a seed used to + /// build a distinct error instance when it does. + /// + private static Gen<(int Add, bool Fail, int Seed)> Steps() { + return from add in Ints() + from fail in ArbMap.Default.GeneratorFor() + from seed in Ints() + select (Add: add, Fail: fail, Seed: seed); + } + + /// + /// Generates an that is either a success carrying a value or a failure carrying + /// a distinct error instance. + /// + private static Gen> Outcomes() { + return Gen.OneOf(Ints().Select(value => Outcome.Success(value)), + Ints().Select(seed => Outcome.Failure(AnError("outcome" + seed)))); + } + + /// + /// Materializes a step specification into a function. A failing step always returns the same captured + /// error instance, so reference-identity propagation can be asserted. + /// + private static Func> ToFunction((int Add, bool Fail, int Seed) step) { + Error error = AnError("step" + step.Seed); + + return value => step.Fail ? Outcome.Failure(error) : Outcome.Success(value + step.Add); + } + + /// + /// Builds a distinct whose code embeds . + /// + private static DomainError AnError(string tag) { + return DomainError.Create(ErrorCode.Create("ERR." + tag), "diagnostic " + tag).WithPublicMessage("summary " + tag); + } + + /// + /// Structural equivalence for two outcomes: successes compare by value, failures by reference identity of + /// their error. + /// + private static bool Equivalent(Outcome left, Outcome right) { + if (left.IsSuccess && right.IsSuccess) { return left.GetResultOrThrow() == right.GetResultOrThrow(); } + if (left.IsFailure && right.IsFailure) { return ReferenceEquals(left.Error, right.Error); } + + return false; + } + + #endregion + +} diff --git a/FirstClassErrors.PropertyTests/PropertyTestSupport.cs b/FirstClassErrors.PropertyTests/PropertyTestSupport.cs new file mode 100644 index 0000000..3b3720b --- /dev/null +++ b/FirstClassErrors.PropertyTests/PropertyTestSupport.cs @@ -0,0 +1,69 @@ +#region Usings declarations + +using FsCheck; +using FsCheck.Fluent; + +#endregion + +namespace FirstClassErrors.PropertyTests; + +/// +/// Custom FsCheck generators shared by the property-based tests. They deliberately feed awkward inputs +/// (empty, whitespace-only, and whitespace-padded strings, alongside the full range of characters produced +/// by FsCheck's default string generator) into the library's validating factories, in the spirit of fuzzing. +/// +internal static class Generators { + + #region Statics members declarations + + /// + /// Generates a non-blank string: never null, always containing at least one non-whitespace + /// character, and frequently carrying leading or trailing whitespace so that trimming and normalization + /// paths are exercised. + /// + public static Gen NonBlank() { + return ArbMap.Default.GeneratorFor().Select(candidate => string.IsNullOrWhiteSpace(candidate) ? "x" + candidate : candidate); + } + + /// + /// Generates a blank string: either empty or composed solely of whitespace characters. Never null. + /// + public static Gen Blank() { + return Gen.OneOf(Gen.Constant(string.Empty), + Gen.Choose(1, 8).Select(length => new string(' ', length)), + Gen.Constant("\t"), + Gen.Constant("\n"), + Gen.Constant("\r\n"), + Gen.Constant(" \t \r\n ")); + } + + #endregion + +} + +/// +/// Small assertion helpers usable from inside an FsCheck property, where a boolean result (rather than a +/// thrown assertion) signals whether the property held for the generated input. +/// +internal static class Expect { + + #region Statics members declarations + + /// + /// Returns true when invoking throws an exception assignable to + /// ; otherwise false. + /// + public static bool Throws(Action action) + where TException : Exception { + try { + action(); + + return false; + } catch (TException) { + return true; + } + } + + #endregion + +} diff --git a/FirstClassErrors.sln b/FirstClassErrors.sln index a88b24c..7bc5950 100644 --- a/FirstClassErrors.sln +++ b/FirstClassErrors.sln @@ -39,6 +39,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "FirstClassErrors.Testing", EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "FirstClassErrors.Cli.UnitTests", "FirstClassErrors.Cli.UnitTests\FirstClassErrors.Cli.UnitTests.csproj", "{E7C09549-AD25-433A-8C56-920D26843E9D}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "FirstClassErrors.PropertyTests", "FirstClassErrors.PropertyTests\FirstClassErrors.PropertyTests.csproj", "{BB28120C-1D68-469D-928A-51AE454D2487}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -191,6 +193,18 @@ Global {E7C09549-AD25-433A-8C56-920D26843E9D}.Release|x64.Build.0 = Release|Any CPU {E7C09549-AD25-433A-8C56-920D26843E9D}.Release|x86.ActiveCfg = Release|Any CPU {E7C09549-AD25-433A-8C56-920D26843E9D}.Release|x86.Build.0 = Release|Any CPU + {BB28120C-1D68-469D-928A-51AE454D2487}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {BB28120C-1D68-469D-928A-51AE454D2487}.Debug|Any CPU.Build.0 = Debug|Any CPU + {BB28120C-1D68-469D-928A-51AE454D2487}.Debug|x64.ActiveCfg = Debug|Any CPU + {BB28120C-1D68-469D-928A-51AE454D2487}.Debug|x64.Build.0 = Debug|Any CPU + {BB28120C-1D68-469D-928A-51AE454D2487}.Debug|x86.ActiveCfg = Debug|Any CPU + {BB28120C-1D68-469D-928A-51AE454D2487}.Debug|x86.Build.0 = Debug|Any CPU + {BB28120C-1D68-469D-928A-51AE454D2487}.Release|Any CPU.ActiveCfg = Release|Any CPU + {BB28120C-1D68-469D-928A-51AE454D2487}.Release|Any CPU.Build.0 = Release|Any CPU + {BB28120C-1D68-469D-928A-51AE454D2487}.Release|x64.ActiveCfg = Release|Any CPU + {BB28120C-1D68-469D-928A-51AE454D2487}.Release|x64.Build.0 = Release|Any CPU + {BB28120C-1D68-469D-928A-51AE454D2487}.Release|x86.ActiveCfg = Release|Any CPU + {BB28120C-1D68-469D-928A-51AE454D2487}.Release|x86.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -205,9 +219,10 @@ Global {D8B4E1F6-2C7A-4F93-8B05-1E6D9A3C4B72} = {B7C3D08D-EFC5-4F5D-8DE4-5B7938354DBB} {47052422-15BC-482A-8FC0-65F9AAAB0291} = {02EA681E-C7D8-13C7-8484-4AC65E1B71E8} {2BE1F2DB-4A9E-4616-89BA-9A597B721217} = {B7C3D08D-EFC5-4F5D-8DE4-5B7938354DBB} + {0FD6C7A3-6832-491A-96C1-F33F62C56EEB} = {F75115F0-25B9-49BC-AE36-A39F2175727A} {A3966512-E54A-4C4F-BFF9-AF5B1A0B132C} = {02EA681E-C7D8-13C7-8484-4AC65E1B71E8} {E7C09549-AD25-433A-8C56-920D26843E9D} = {B7C3D08D-EFC5-4F5D-8DE4-5B7938354DBB} - {0FD6C7A3-6832-491A-96C1-F33F62C56EEB} = {F75115F0-25B9-49BC-AE36-A39F2175727A} + {BB28120C-1D68-469D-928A-51AE454D2487} = {B7C3D08D-EFC5-4F5D-8DE4-5B7938354DBB} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {4988972E-3E0D-4F48-8656-0E67ECE994BF} diff --git a/maintainers/workflows/scorecard.en.md b/maintainers/workflows/scorecard.en.md index 234ac85..bb32f21 100644 --- a/maintainers/workflows/scorecard.en.md +++ b/maintainers/workflows/scorecard.en.md @@ -67,6 +67,20 @@ that Scorecard's own Token-Permissions check rewards. not here. A low Branch-Protection sub-score, for instance, is fixed by marking checks required on `main`, not by editing this file. +## Fuzzing check + +Scorecard's **Fuzzing** check is satisfied here by the property-based tests in +`FirstClassErrors.PropertyTests` (FsCheck). Scorecard credits C# property-based +testing by scanning the source tree for `using FsCheck;`, so the fuzzers live in +that test project, not in this workflow. + +- **Mind the tooling version.** C# property-based detection landed in **scorecard + v5.5.0**; `scorecard-action@v2.4.3` — the pin in the workflow — still bundles + **v5.3.0**, which predates it. The sub-check therefore only turns green once the + scan runs on ≥ v5.5.0: when a newer `scorecard-action` (bundling ≥ v5.5.0) ships + and the pin is bumped. Until then the tests still run and still guard the code — + only the published sub-score lags. + ## Related - [`codeql`](codeql.en.md) — shares the `github/codeql-action/upload-sarif` diff --git a/maintainers/workflows/scorecard.fr.md b/maintainers/workflows/scorecard.fr.md index b3ce880..3db7222 100644 --- a/maintainers/workflows/scorecard.fr.md +++ b/maintainers/workflows/scorecard.fr.md @@ -69,6 +69,21 @@ credentials que le propre check Token-Permissions de Scorecard récompense. dépôt**, pas ici. Un sous-score Branch-Protection faible, par exemple, se corrige en marquant des checks required sur `main`, pas en éditant ce fichier. +## Check Fuzzing + +Le check **Fuzzing** de Scorecard est satisfait ici par les tests property-based +de `FirstClassErrors.PropertyTests` (FsCheck). Scorecard crédite le property-based +testing C# en cherchant `using FsCheck;` dans l'arborescence des sources : les +fuzzers vivent donc dans ce projet de tests, pas dans ce workflow. + +- **Attention à la version de l'outillage.** La détection property-based C# est + arrivée dans **scorecard v5.5.0** ; `scorecard-action@v2.4.3` — le pin du + workflow — embarque encore **v5.3.0**, antérieure. Le sous-check ne passe donc + au vert qu'une fois le scan exécuté en ≥ v5.5.0 : quand une version plus récente + de `scorecard-action` (embarquant ≥ v5.5.0) sort et que le pin est relevé. + D'ici là les tests tournent et protègent le code — seul le sous-score publié est + en retard. + ## En rapport - [`codeql`](codeql.fr.md) — partage l'action `github/codeql-action/upload-sarif`