diff --git a/docs/release-notes/.FSharp.Compiler.Service/11.0.100.md b/docs/release-notes/.FSharp.Compiler.Service/11.0.100.md index c66e2dee74e..1b8f27909dd 100644 --- a/docs/release-notes/.FSharp.Compiler.Service/11.0.100.md +++ b/docs/release-notes/.FSharp.Compiler.Service/11.0.100.md @@ -117,6 +117,7 @@ * Debug: fix if and match condition sequence points ([PR #19932](https://github.com/dotnet/fsharp/pull/19932)) * Checker: recover on checking language version ([PR ##19970](https://github.com/dotnet/fsharp/pull/19970)) * Implied argument names for function-to-delegate coercions now fall back to the delegate's `Invoke` parameter names when the function has no recoverable names (e.g. a partial application like `System.Func((+) 1)`), instead of synthetic `delegateArg0`, `delegateArg1`, … names. ([PR #20001](https://github.com/dotnet/fsharp/pull/20001)) +* Add Roslyn-format EnC CustomDebugInformation codec and portable PDB method CDI emission support to AbstractIL. ([PR #20018](https://github.com/dotnet/fsharp/pull/20018)) ### Improved diff --git a/src/Compiler/AbstractIL/EncMethodDebugInformation.fs b/src/Compiler/AbstractIL/EncMethodDebugInformation.fs new file mode 100644 index 00000000000..e605b2208a4 --- /dev/null +++ b/src/Compiler/AbstractIL/EncMethodDebugInformation.fs @@ -0,0 +1,557 @@ +/// Edit-and-Continue method debug information blobs. +/// +/// This module replicates, byte for byte, the three Portable-PDB CustomDebugInformation +/// blob formats Roslyn persists per method to support Edit and Continue +/// (roslyn/src/Compilers/Core/Portable/Emit/EditAndContinueMethodDebugInformation.cs): +/// +/// - EnC Local Slot Map (kind 755F52A8-91C5-45BE-B4B8-209571E552BD) +/// - EnC Lambda and Closure Map (kind A643004C-0240-496F-A783-30D64F4979DE) +/// - EnC State Machine State Map (kind 8B78CD68-2EDE-420B-980B-E15884B8AAA3) +/// +/// (GUIDs: roslyn/src/Dependencies/CodeAnalysis.Debugging/PortableCustomDebugInfoKinds.cs.) +/// +/// All multi-byte integers use the ECMA-335 compressed unsigned/signed encodings via +/// System.Reflection.Metadata's BlobBuilder.WriteCompressedInteger / +/// WriteCompressedSignedInteger and BlobReader.ReadCompressedInteger / +/// ReadCompressedSignedInteger, exactly as Roslyn writes/reads them. +/// +/// Every "syntax offset" slot in these blobs is an opaque, caller-defined integer key +/// (Roslyn: the syntax offset of the lambda/closure/state-machine-suspension syntax +/// node). This module does not require the key to be a source offset; it only requires +/// determinism across generations. tryEncodeOccurrenceKey/decodeOccurrenceKey provide one +/// reusable way to pack a short (depth <= 2) ordinal chain into such a key. +module internal FSharp.Compiler.AbstractIL.EncMethodDebugInformation + +#nowarn "9" // NativePtr: BlobReader only exposes a byte*-based constructor + +open System +open System.Collections.Generic +open System.Collections.Immutable +open System.IO +open System.Reflection.Metadata +open System.Reflection.Metadata.Ecma335 +open System.Runtime.InteropServices +open Microsoft.FSharp.NativeInterop + +/// Portable-PDB CustomDebugInformation kind GUIDs for the EnC blobs, copied verbatim +/// from roslyn/src/Dependencies/CodeAnalysis.Debugging/PortableCustomDebugInfoKinds.cs. +[] +module PortableCustomDebugInfoKinds = + + /// EnC Local Slot Map CDI kind. + let encLocalSlotMap = Guid("755F52A8-91C5-45BE-B4B8-209571E552BD") + + /// EnC Lambda and Closure Map CDI kind. + let encLambdaAndClosureMap = Guid("A643004C-0240-496F-A783-30D64F4979DE") + + /// EnC State Machine State Map CDI kind. + let encStateMachineStateMap = Guid("8B78CD68-2EDE-420B-980B-E15884B8AAA3") + +/// Closure ordinal of a lambda that is lowered to a static (non-capturing) method. +/// Mirrors Roslyn's LambdaDebugInfo.StaticClosureOrdinal. +[] +let StaticClosureOrdinal = -1 + +/// Closure ordinal of a lambda closed over the 'this' pointer only. +/// Mirrors Roslyn's LambdaDebugInfo.ThisOnlyClosureOrdinal. +[] +let ThisOnlyClosureOrdinal = -2 + +/// Smallest valid closure ordinal. Mirrors Roslyn's LambdaDebugInfo.MinClosureOrdinal. +[] +let MinClosureOrdinal = ThisOnlyClosureOrdinal + +/// Method ordinal of a method that has no lambda map (an empty blob decodes to this). +/// Mirrors Roslyn's DebugId.UndefinedOrdinal. +[] +let UndefinedMethodOrdinal = -1 + +/// Marker byte introducing the (optional) negative syntax-offset baseline in the +/// local-slot-map blob. Mirrors Roslyn's SyntaxOffsetBaseline = 0xFF. +[] +let private SyntaxOffsetBaselineMarker = 0xFFuy + +/// Largest synthesized-local kind serializable in the slot map: the kind is stored as +/// (kind + 1) in bits 0-5 of the leading byte (bit 6 is unused, bit 7 flags a trailing ordinal), and +/// Roslyn's reader recovers it with mask 0x3F, so only kinds 0..0x3E round-trip. +[] +let MaxSerializableLocalKind = 0x3E + +/// One slot in the EnC Local Slot Map: the local variable layout of a method body, +/// recorded so a later generation can map its locals onto the same slot indices. +[] +type EncLocalSlotInfo = + /// A short-lived lowering temp: serialized as the single byte 0x00, carrying no + /// identity (a later generation never reuses it). + | Temp + + /// A long-lived synthesized local. + /// kind: synthesized-local kind (Roslyn SynthesizedLocalKind value, 0..MaxSerializableLocalKind; + /// 0 = user-defined local). + /// syntaxOffset: caller-defined key of the declaring occurrence + /// (Roslyn: syntax offset of the local's declarator). + /// ordinal: zero-based disambiguator among slots sharing the same kind and offset (>= 0). + | Slot of kind: int * syntaxOffset: int * ordinal: int + +/// One closure scope in the EnC Lambda and Closure Map. The closure's ordinal is its +/// index in EncMethodDebugInformation.Closures; lambdas reference closures by that index. +type EncClosureInfo = + { + /// Caller-defined key (Roslyn: syntax offset of the scope owning the closure). + SyntaxOffset: int + } + +/// One lambda in the EnC Lambda and Closure Map. +type EncLambdaInfo = + { + /// Caller-defined key (Roslyn: syntax offset of the lambda body). + SyntaxOffset: int + /// Index into EncMethodDebugInformation.Closures of the closure holding the + /// lambda's captures, or StaticClosureOrdinal / ThisOnlyClosureOrdinal. + ClosureOrdinal: int + } + +/// One suspension point in the EnC State Machine State Map. +type EncStateMachineStateInfo = + { + /// State machine state number assigned to the suspension point (may be negative: + /// Roslyn uses negative numbers for increasing-iteration finalize states). + StateNumber: int + /// Caller-defined key (Roslyn: syntax offset of the await/yield syntax node). + SyntaxOffset: int + } + +/// Debugging information associated with a method, persisted by the compiler in the +/// Portable PDB to support Edit and Continue. Mirrors Roslyn's +/// EditAndContinueMethodDebugInformation. +type EncMethodDebugInformation = + { + /// Ordinal of the method within its generation (>= -1; UndefinedMethodOrdinal when absent). + MethodOrdinal: int + /// Local slot layout, in slot-index order (EnC Local Slot Map). + LocalSlots: EncLocalSlotInfo list + /// Closure scopes, in ordinal order (EnC Lambda and Closure Map). + Closures: EncClosureInfo list + /// Lambdas, in ordinal order (EnC Lambda and Closure Map). + Lambdas: EncLambdaInfo list + /// State machine suspension points (EnC State Machine State Map). + StateMachineStates: EncStateMachineStateInfo list + } + + /// An empty map (no slots, lambdas, closures or states; undefined method ordinal). + static member Empty = + { + MethodOrdinal = UndefinedMethodOrdinal + LocalSlots = [] + Closures = [] + Lambdas = [] + StateMachineStates = [] + } + +// --------------------------------------------------------------------------- +// Occurrence-key packing +// --------------------------------------------------------------------------- + +/// Maximum encodable occurrence ordinal: each chain segment is 16 bits. +[] +let private MaxOccurrenceSegment = 0xFFFF + +/// Compressed unsigned integers must lie in [0, 0x1FFFFFFF); after baseline adjustment +/// the serialized value is (key - baseline) with baseline <= -1, so keys must stay +/// strictly below 0x1FFFFFFF - 1 to be writable. Cap at 29 bits minus the adjustment. +[] +let private MaxOccurrenceKey = 0x1FFFFFFD + +/// Packs an ordinal chain (root-first enclosing ordinals, ending with the innermost +/// ordinal) into a deterministic int suitable for a "syntax offset" blob slot. Packing: +/// 16-bit segments, least-significant segment = the innermost ordinal; an enclosing +/// ordinal p is stored as (p + 1) shifted left 16 so that depth-1 keys (< 0x10000) and +/// depth-2 keys (>= 0x10000) never collide. Fails closed (None) past the limits: chains +/// deeper than 2, ordinals > 0xFFFF, or keys exceeding the compressed-integer budget — +/// callers must then treat the chain as unmappable, never truncate. +let tryEncodeOccurrenceKey (ordinalChain: int list) : int option = + match ordinalChain with + | [ ordinal ] when ordinal >= 0 && ordinal <= MaxOccurrenceSegment -> Some ordinal + | [ parent; ordinal ] when + parent >= 0 + && ordinal >= 0 + && ordinal <= MaxOccurrenceSegment + && parent < MaxOccurrenceSegment + -> + // Pack in int64: a large parent (e.g. 0xFFFE) would wrap ((parent + 1) <<< 16) negative in + // int32 and a negative key slips past the <= MaxOccurrenceKey bound, failing OPEN. + let key = ((int64 parent + 1L) <<< 16) ||| int64 ordinal + + if key <= int64 MaxOccurrenceKey then + Some(int key) + else + None + | _ -> None + +/// Unpacks an occurrence key produced by tryEncodeOccurrenceKey back into its +/// root-first ordinal chain. +let decodeOccurrenceKey (key: int) : int list = + if key < 0 then + invalidArg (nameof key) $"occurrence key must be non-negative, got %d{key}" + elif key <= MaxOccurrenceSegment then + [ key ] + else + [ (key >>> 16) - 1; key &&& MaxOccurrenceSegment ] + +// --------------------------------------------------------------------------- +// Blob helpers +// --------------------------------------------------------------------------- + +let private invalidData (blobName: string) (offset: int) = + raise (InvalidDataException $"invalid EnC %s{blobName} blob: unexpected data at offset %d{offset}") + +// Absent CDI rows arrive as null at runtime even though the parameter is non-null in the +// nullness model, so guard with box (FS3261-safe) rather than dropping the check. +let private isEmpty (blob: byte[]) = isNull (box blob) || blob.Length = 0 + +// --------------------------------------------------------------------------- +// EnC Local Slot Map +// Format (EditAndContinueMethodDebugInformation.cs, SerializeLocalSlots lines 145-191, +// UncompressSlotMap lines 92-143): optional baseline record [0xFF, compressed(-baseline)], +// then one record per slot: 0x00 for a temp, otherwise a leading byte with bits 0-5 = +// kind + 1 and bit 7 = has-ordinal flag, followed by compressed(syntaxOffset - baseline) +// and, when flagged, compressed(ordinal). +// --------------------------------------------------------------------------- + +/// Serializes the EnC Local Slot Map blob for 'info', byte-for-byte as Roslyn's +/// SerializeLocalSlots. Returns the empty array when there are no slots (no CDI row +/// should be emitted then). +let serializeLocalSlots (info: EncMethodDebugInformation) : byte[] = + match info.LocalSlots with + | [] -> Array.empty + | slots -> + let builder = BlobBuilder() + + // The baseline is the most negative syntax offset, or -1 when none is negative + // (Roslyn lines 147-160). Offsets are stored relative to it so the common + // all-non-negative case costs no baseline record. + let syntaxOffsetBaseline = + (-1, slots) + ||> List.fold (fun acc slot -> + match slot with + | EncLocalSlotInfo.Temp -> acc + | EncLocalSlotInfo.Slot(_, syntaxOffset, _) -> min acc syntaxOffset) + + if syntaxOffsetBaseline <> -1 then + builder.WriteByte SyntaxOffsetBaselineMarker + builder.WriteCompressedInteger(-syntaxOffsetBaseline) + + for slot in slots do + match slot with + | EncLocalSlotInfo.Temp -> builder.WriteByte 0uy + | EncLocalSlotInfo.Slot(kind, syntaxOffset, ordinal) -> + if kind < 0 || kind > MaxSerializableLocalKind then + invalidArg (nameof info) $"local slot kind %d{kind} is outside the serializable range 0..%d{MaxSerializableLocalKind}" + + if ordinal < 0 then + invalidArg (nameof info) $"local slot ordinal must be non-negative, got %d{ordinal}" + + let hasOrdinal = ordinal > 0 + let b = byte (kind + 1) ||| (if hasOrdinal then 0x80uy else 0uy) + builder.WriteByte b + builder.WriteCompressedInteger(syntaxOffset - syntaxOffsetBaseline) + + if hasOrdinal then + builder.WriteCompressedInteger ordinal + + builder.ToArray() + +/// Deserializes an EnC Local Slot Map blob, byte-for-byte as Roslyn's UncompressSlotMap. +/// An empty (or null) blob yields no slots. +let deserializeLocalSlots (blob: byte[]) : EncLocalSlotInfo list = + if isEmpty blob then + [] + else + let handle = GCHandle.Alloc(blob, GCHandleType.Pinned) + + try + let mutable reader = + BlobReader(NativePtr.ofNativeInt (handle.AddrOfPinnedObject()), blob.Length) + + let slots = ResizeArray() + let mutable syntaxOffsetBaseline = -1 + + try + while reader.RemainingBytes > 0 do + let b = reader.ReadByte() + + if b = SyntaxOffsetBaselineMarker then + syntaxOffsetBaseline <- -reader.ReadCompressedInteger() + elif b = 0uy then + slots.Add EncLocalSlotInfo.Temp + else + // Roslyn recovers the kind with mask 0x3F (line 126); bit 7 flags + // a trailing ordinal, bit 6 is unused by the writer. + let kind = int (b &&& 0x3Fuy) - 1 + let hasOrdinal = b &&& 0x80uy <> 0uy + let syntaxOffset = reader.ReadCompressedInteger() + syntaxOffsetBaseline + let ordinal = if hasOrdinal then reader.ReadCompressedInteger() else 0 + slots.Add(EncLocalSlotInfo.Slot(kind, syntaxOffset, ordinal)) + with :? BadImageFormatException -> + invalidData "local slot map" reader.Offset + + List.ofSeq slots + finally + handle.Free() + +// --------------------------------------------------------------------------- +// EnC Lambda and Closure Map +// Format (SerializeLambdaMap lines 261-302, UncompressLambdaMap lines 197-259): +// compressed(methodOrdinal + 1), compressed(-baseline), compressed(closureCount), +// closureCount * compressed(syntaxOffset - baseline), then until the blob ends: +// [compressed(syntaxOffset - baseline), compressed(closureOrdinal - MinClosureOrdinal)] +// per lambda. +// --------------------------------------------------------------------------- + +/// Serializes the EnC Lambda and Closure Map blob for 'info', byte-for-byte as Roslyn's +/// SerializeLambdaMap. Returns the empty array when there are no lambdas and no closures +/// (Roslyn's MetadataWriter skips the CDI row in that case; note the method ordinal is +/// then not persisted and decodes back as UndefinedMethodOrdinal). +let serializeLambdaMap (info: EncMethodDebugInformation) : byte[] = + match info.Closures, info.Lambdas with + | [], [] -> Array.empty + | closures, lambdas -> + if info.MethodOrdinal < -1 then + invalidArg (nameof info) $"method ordinal must be >= -1, got %d{info.MethodOrdinal}" + + let builder = BlobBuilder() + builder.WriteCompressedInteger(info.MethodOrdinal + 1) + + // Negative offsets are rare (Roslyn: field/property initializers), so the + // baseline is -1 unless a smaller offset exists (Roslyn lines 266-286). + let syntaxOffsetBaseline = + let closureMin = (-1, closures) ||> List.fold (fun acc c -> min acc c.SyntaxOffset) + (closureMin, lambdas) ||> List.fold (fun acc l -> min acc l.SyntaxOffset) + + builder.WriteCompressedInteger(-syntaxOffsetBaseline) + builder.WriteCompressedInteger closures.Length + + for closure in closures do + builder.WriteCompressedInteger(closure.SyntaxOffset - syntaxOffsetBaseline) + + for lambda in lambdas do + if + lambda.ClosureOrdinal < MinClosureOrdinal + || lambda.ClosureOrdinal >= closures.Length + then + invalidArg + (nameof info) + $"lambda closure ordinal %d{lambda.ClosureOrdinal} is outside [%d{MinClosureOrdinal}, %d{closures.Length})" + + builder.WriteCompressedInteger(lambda.SyntaxOffset - syntaxOffsetBaseline) + builder.WriteCompressedInteger(lambda.ClosureOrdinal - MinClosureOrdinal) + + builder.ToArray() + +/// Deserializes an EnC Lambda and Closure Map blob, byte-for-byte as Roslyn's +/// UncompressLambdaMap. An empty (or null) blob yields (UndefinedMethodOrdinal, [], []). +let deserializeLambdaMap (blob: byte[]) : int * EncClosureInfo list * EncLambdaInfo list = + if isEmpty blob then + UndefinedMethodOrdinal, [], [] + else + let handle = GCHandle.Alloc(blob, GCHandleType.Pinned) + + try + let mutable reader = + BlobReader(NativePtr.ofNativeInt (handle.AddrOfPinnedObject()), blob.Length) + + let closures = ResizeArray() + let lambdas = ResizeArray() + let mutable methodOrdinal = UndefinedMethodOrdinal + + try + methodOrdinal <- reader.ReadCompressedInteger() - 1 + let syntaxOffsetBaseline = -reader.ReadCompressedInteger() + let closureCount = reader.ReadCompressedInteger() + + for _ in 1..closureCount do + let syntaxOffset = reader.ReadCompressedInteger() + syntaxOffsetBaseline + closures.Add { SyntaxOffset = syntaxOffset } + + while reader.RemainingBytes > 0 do + let syntaxOffset = reader.ReadCompressedInteger() + syntaxOffsetBaseline + let closureOrdinal = reader.ReadCompressedInteger() + MinClosureOrdinal + + if closureOrdinal >= closureCount then + invalidData "lambda map" reader.Offset + + lambdas.Add + { + SyntaxOffset = syntaxOffset + ClosureOrdinal = closureOrdinal + } + with :? BadImageFormatException -> + invalidData "lambda map" reader.Offset + + methodOrdinal, List.ofSeq closures, List.ofSeq lambdas + finally + handle.Free() + +// --------------------------------------------------------------------------- +// EnC State Machine State Map +// Format (SerializeStateMachineStates lines 364-381, UncompressStateMachineStates +// lines 309-362): compressed(count); when count > 0: compressed(-baseline) followed by +// count * [compressedSigned(stateNumber), compressed(syntaxOffset - baseline)], entries +// ordered by syntax offset. +// --------------------------------------------------------------------------- + +/// Serializes the EnC State Machine State Map blob for 'info', byte-for-byte as +/// Roslyn's SerializeStateMachineStates: entries are sorted by syntax offset (stably, +/// preserving relative order of equal offsets, which encodes the per-offset relative +/// ordinal). Returns the empty array when there are no states (no CDI row then). +let serializeStateMachineStates (info: EncMethodDebugInformation) : byte[] = + match info.StateMachineStates with + | [] -> Array.empty + | states -> + let builder = BlobBuilder() + builder.WriteCompressedInteger states.Length + + // Unlike the other two blobs the baseline here is min(minOffset, 0) + // (Roslyn line 372). + let syntaxOffsetBaseline = + min (states |> List.map (fun s -> s.SyntaxOffset) |> List.min) 0 + + builder.WriteCompressedInteger(-syntaxOffsetBaseline) + + // Roslyn's reader rejects more than 256 entries sharing one syntax offset + // (relative ordinal must fit a byte, line 344); fail closed at write time. + for _, group in states |> List.groupBy (fun s -> s.SyntaxOffset) do + if group.Length > 256 then + invalidArg (nameof info) $"more than 256 state machine states share syntax offset %d{group.Head.SyntaxOffset}" + + for state in states |> List.sortBy (fun s -> s.SyntaxOffset) do + builder.WriteCompressedSignedInteger state.StateNumber + builder.WriteCompressedInteger(state.SyntaxOffset - syntaxOffsetBaseline) + + builder.ToArray() + +/// Deserializes an EnC State Machine State Map blob, byte-for-byte as Roslyn's +/// UncompressStateMachineStates (including the ordered-by-offset and <= 256-per-offset +/// validations). An empty (or null) blob yields no states. +let deserializeStateMachineStates (blob: byte[]) : EncStateMachineStateInfo list = + if isEmpty blob then + [] + else + let handle = GCHandle.Alloc(blob, GCHandleType.Pinned) + + try + let mutable reader = + BlobReader(NativePtr.ofNativeInt (handle.AddrOfPinnedObject()), blob.Length) + + let states = ResizeArray() + + try + let count = reader.ReadCompressedInteger() + + if count > 0 then + let syntaxOffsetBaseline = -reader.ReadCompressedInteger() + let mutable lastSyntaxOffset = Int32.MinValue + let mutable relativeOrdinal = 0 + + for _ in 1..count do + let stateNumber = reader.ReadCompressedSignedInteger() + let syntaxOffset = syntaxOffsetBaseline + reader.ReadCompressedInteger() + + // Entries must be ordered by syntax offset and at most 256 may + // share one offset (Roslyn lines 336-347). + if syntaxOffset < lastSyntaxOffset then + invalidData "state machine state map" reader.Offset + + relativeOrdinal <- + if syntaxOffset = lastSyntaxOffset then + relativeOrdinal + 1 + else + 0 + + if relativeOrdinal > 255 then + invalidData "state machine state map" reader.Offset + + states.Add + { + StateNumber = stateNumber + SyntaxOffset = syntaxOffset + } + + lastSyntaxOffset <- syntaxOffset + with :? BadImageFormatException -> + invalidData "state machine state map" reader.Offset + + List.ofSeq states + finally + handle.Free() + +/// Deserializes EnC method debug information from the three blobs (any of which may be +/// null or empty). Mirrors Roslyn's EditAndContinueMethodDebugInformation.Create. +let deserialize (slotMapBlob: byte[]) (lambdaMapBlob: byte[]) (stateMachineStateMapBlob: byte[]) : EncMethodDebugInformation = + let methodOrdinal, closures, lambdas = deserializeLambdaMap lambdaMapBlob + + { + MethodOrdinal = methodOrdinal + LocalSlots = deserializeLocalSlots slotMapBlob + Closures = closures + Lambdas = lambdas + StateMachineStates = deserializeStateMachineStates stateMachineStateMapBlob + } + +/// Decodes every method-level EnC CustomDebugInformation row of a portable PDB image into +/// per-method EnC debug information, keyed by MethodDef token (0x06xxxxxx). The CDI parent +/// of the EnC rows is always a MethodDef handle, so token keying is unambiguous. +/// Fail safe: a null/empty or non-PDB image yields the empty map, and a method whose +/// blobs do not decode is omitted rather than guessed. +let readEncMethodDebugInfoFromPortablePdb (pdbBytes: byte[]) : Map = + if isEmpty pdbBytes then + Map.empty + else + try + use provider = + MetadataReaderProvider.FromPortablePdbImage(ImmutableArray.CreateRange pdbBytes) + + let reader = provider.GetMetadataReader() + + let slotMapBlobs = Dictionary() + let lambdaMapBlobs = Dictionary() + let stateMapBlobs = Dictionary() + + for cdiHandle in reader.CustomDebugInformation do + let cdi = reader.GetCustomDebugInformation cdiHandle + + if cdi.Parent.Kind = HandleKind.MethodDefinition then + let methodToken = MetadataTokens.GetToken cdi.Parent + let kind = reader.GetGuid cdi.Kind + + if kind = PortableCustomDebugInfoKinds.encLocalSlotMap then + slotMapBlobs[methodToken] <- reader.GetBlobBytes cdi.Value + elif kind = PortableCustomDebugInfoKinds.encLambdaAndClosureMap then + lambdaMapBlobs[methodToken] <- reader.GetBlobBytes cdi.Value + elif kind = PortableCustomDebugInfoKinds.encStateMachineStateMap then + stateMapBlobs[methodToken] <- reader.GetBlobBytes cdi.Value + + let methodTokens = + Seq.concat [ slotMapBlobs.Keys :> seq; lambdaMapBlobs.Keys; stateMapBlobs.Keys ] + |> Seq.distinct + + let tryBlob (blobs: Dictionary) token = + match blobs.TryGetValue token with + | true, blob -> blob + | _ -> Array.empty + + (Map.empty, methodTokens) + ||> Seq.fold (fun acc token -> + try + let info = + deserialize (tryBlob slotMapBlobs token) (tryBlob lambdaMapBlobs token) (tryBlob stateMapBlobs token) + + Map.add token info acc + with :? InvalidDataException -> + // Fail closed per method: an undecodable blob never yields a partial + // (and so potentially mismatched) map for its method. + acc) + with :? BadImageFormatException -> + // Not a portable PDB image (or a corrupted one): callers still get an empty + // map instead of a crash. + Map.empty diff --git a/src/Compiler/AbstractIL/EncMethodDebugInformation.fsi b/src/Compiler/AbstractIL/EncMethodDebugInformation.fsi new file mode 100644 index 00000000000..1e2ba76e7c8 --- /dev/null +++ b/src/Compiler/AbstractIL/EncMethodDebugInformation.fsi @@ -0,0 +1,178 @@ +/// Edit-and-Continue method debug information blobs. +/// +/// This module replicates, byte for byte, the three Portable-PDB CustomDebugInformation +/// blob formats Roslyn persists per method to support Edit and Continue +/// (roslyn/src/Compilers/Core/Portable/Emit/EditAndContinueMethodDebugInformation.cs): +/// +/// - EnC Local Slot Map (kind 755F52A8-91C5-45BE-B4B8-209571E552BD) +/// - EnC Lambda and Closure Map (kind A643004C-0240-496F-A783-30D64F4979DE) +/// - EnC State Machine State Map (kind 8B78CD68-2EDE-420B-980B-E15884B8AAA3) +/// +/// (GUIDs: roslyn/src/Dependencies/CodeAnalysis.Debugging/PortableCustomDebugInfoKinds.cs.) +/// +/// All multi-byte integers use the ECMA-335 compressed unsigned/signed encodings via +/// System.Reflection.Metadata's BlobBuilder.WriteCompressedInteger / +/// WriteCompressedSignedInteger and BlobReader.ReadCompressedInteger / +/// ReadCompressedSignedInteger, exactly as Roslyn writes/reads them. +/// +/// Every "syntax offset" slot in these blobs is an opaque, caller-defined integer key +/// (Roslyn: the syntax offset of the lambda/closure/state-machine-suspension syntax +/// node). This module does not require the key to be a source offset; it only requires +/// determinism across generations. tryEncodeOccurrenceKey/decodeOccurrenceKey provide one +/// reusable way to pack a short (depth <= 2) ordinal chain into such a key. +module internal FSharp.Compiler.AbstractIL.EncMethodDebugInformation + +/// Portable-PDB CustomDebugInformation kind GUIDs for the EnC blobs, copied verbatim +/// from roslyn/src/Dependencies/CodeAnalysis.Debugging/PortableCustomDebugInfoKinds.cs. +[] +module PortableCustomDebugInfoKinds = + + /// EnC Local Slot Map CDI kind. + val encLocalSlotMap: System.Guid + + /// EnC Lambda and Closure Map CDI kind. + val encLambdaAndClosureMap: System.Guid + + /// EnC State Machine State Map CDI kind. + val encStateMachineStateMap: System.Guid + +/// Closure ordinal of a lambda that is lowered to a static (non-capturing) method. +/// Mirrors Roslyn's LambdaDebugInfo.StaticClosureOrdinal. +[] +val StaticClosureOrdinal: int = -1 + +/// Closure ordinal of a lambda closed over the 'this' pointer only. +/// Mirrors Roslyn's LambdaDebugInfo.ThisOnlyClosureOrdinal. +[] +val ThisOnlyClosureOrdinal: int = -2 + +/// Smallest valid closure ordinal. Mirrors Roslyn's LambdaDebugInfo.MinClosureOrdinal. +[] +val MinClosureOrdinal: int = -2 + +/// Method ordinal of a method that has no lambda map (an empty blob decodes to this). +/// Mirrors Roslyn's DebugId.UndefinedOrdinal. +[] +val UndefinedMethodOrdinal: int = -1 + +/// Largest synthesized-local kind serializable in the slot map: the kind is stored as +/// (kind + 1) in bits 0-5 of the leading byte (bit 6 is unused, bit 7 flags a trailing ordinal), and +/// Roslyn's reader recovers it with mask 0x3F, so only kinds 0..0x3E round-trip. +[] +val MaxSerializableLocalKind: int = 0x3E + +/// One slot in the EnC Local Slot Map: the local variable layout of a method body, +/// recorded so a later generation can map its locals onto the same slot indices. +[] +type EncLocalSlotInfo = + /// A short-lived lowering temp: serialized as the single byte 0x00, carrying no + /// identity (a later generation never reuses it). + | Temp + + /// A long-lived synthesized local. + /// kind: synthesized-local kind (Roslyn SynthesizedLocalKind value, 0..MaxSerializableLocalKind; + /// 0 = user-defined local). + /// syntaxOffset: caller-defined key of the declaring occurrence + /// (Roslyn: syntax offset of the local's declarator). + /// ordinal: zero-based disambiguator among slots sharing the same kind and offset (>= 0). + | Slot of kind: int * syntaxOffset: int * ordinal: int + +/// One closure scope in the EnC Lambda and Closure Map. The closure's ordinal is its +/// index in EncMethodDebugInformation.Closures; lambdas reference closures by that index. +type EncClosureInfo = + { + /// Caller-defined key (Roslyn: syntax offset of the scope owning the closure). + SyntaxOffset: int + } + +/// One lambda in the EnC Lambda and Closure Map. +type EncLambdaInfo = + { + /// Caller-defined key (Roslyn: syntax offset of the lambda body). + SyntaxOffset: int + /// Index into EncMethodDebugInformation.Closures of the closure holding the + /// lambda's captures, or StaticClosureOrdinal / ThisOnlyClosureOrdinal. + ClosureOrdinal: int + } + +/// One suspension point in the EnC State Machine State Map. +type EncStateMachineStateInfo = + { + /// State machine state number assigned to the suspension point (may be negative: + /// Roslyn uses negative numbers for increasing-iteration finalize states). + StateNumber: int + /// Caller-defined key (Roslyn: syntax offset of the await/yield syntax node). + SyntaxOffset: int + } + +/// Debugging information associated with a method, persisted by the compiler in the +/// Portable PDB to support Edit and Continue. Mirrors Roslyn's +/// EditAndContinueMethodDebugInformation. +type EncMethodDebugInformation = + { + /// Ordinal of the method within its generation (>= -1; UndefinedMethodOrdinal when absent). + MethodOrdinal: int + /// Local slot layout, in slot-index order (EnC Local Slot Map). + LocalSlots: EncLocalSlotInfo list + /// Closure scopes, in ordinal order (EnC Lambda and Closure Map). + Closures: EncClosureInfo list + /// Lambdas, in ordinal order (EnC Lambda and Closure Map). + Lambdas: EncLambdaInfo list + /// State machine suspension points (EnC State Machine State Map). + StateMachineStates: EncStateMachineStateInfo list + } + + /// An empty map (no slots, lambdas, closures or states; undefined method ordinal). + static member Empty: EncMethodDebugInformation + +/// Packs an ordinal chain (root-first enclosing ordinals, ending with the innermost +/// ordinal) into a deterministic int suitable for a "syntax offset" blob slot. Fails +/// closed (None) past the limits: chains deeper than 2, ordinals > 0xFFFF, or keys +/// exceeding the compressed-integer budget. +val tryEncodeOccurrenceKey: ordinalChain: int list -> int option + +/// Unpacks an occurrence key produced by tryEncodeOccurrenceKey back into its +/// root-first ordinal chain. +val decodeOccurrenceKey: key: int -> int list + +/// Serializes the EnC Local Slot Map blob for 'info', byte-for-byte as Roslyn's +/// SerializeLocalSlots. Returns the empty array when there are no slots (no CDI row +/// should be emitted then). +val serializeLocalSlots: info: EncMethodDebugInformation -> byte[] + +/// Deserializes an EnC Local Slot Map blob, byte-for-byte as Roslyn's UncompressSlotMap. +/// An empty (or null) blob yields no slots. +val deserializeLocalSlots: blob: byte[] -> EncLocalSlotInfo list + +/// Serializes the EnC Lambda and Closure Map blob for 'info', byte-for-byte as Roslyn's +/// SerializeLambdaMap. Returns the empty array when there are no lambdas and no closures +/// (Roslyn's MetadataWriter skips the CDI row in that case; note the method ordinal is +/// then not persisted and decodes back as UndefinedMethodOrdinal). +val serializeLambdaMap: info: EncMethodDebugInformation -> byte[] + +/// Deserializes an EnC Lambda and Closure Map blob, byte-for-byte as Roslyn's +/// UncompressLambdaMap. An empty (or null) blob yields (UndefinedMethodOrdinal, [], []). +val deserializeLambdaMap: blob: byte[] -> int * EncClosureInfo list * EncLambdaInfo list + +/// Serializes the EnC State Machine State Map blob for 'info', byte-for-byte as +/// Roslyn's SerializeStateMachineStates: entries are sorted by syntax offset (stably, +/// preserving relative order of equal offsets, which encodes the per-offset relative +/// ordinal). Returns the empty array when there are no states (no CDI row then). +val serializeStateMachineStates: info: EncMethodDebugInformation -> byte[] + +/// Deserializes an EnC State Machine State Map blob, byte-for-byte as Roslyn's +/// UncompressStateMachineStates (including the ordered-by-offset and <= 256-per-offset +/// validations). An empty (or null) blob yields no states. +val deserializeStateMachineStates: blob: byte[] -> EncStateMachineStateInfo list + +/// Deserializes EnC method debug information from the three blobs (any of which may be +/// null or empty). Mirrors Roslyn's EditAndContinueMethodDebugInformation.Create. +val deserialize: + slotMapBlob: byte[] -> lambdaMapBlob: byte[] -> stateMachineStateMapBlob: byte[] -> EncMethodDebugInformation + +/// Decodes every method-level EnC CustomDebugInformation row of a portable PDB image into +/// per-method EnC debug information, keyed by MethodDef token (0x06xxxxxx). The CDI parent +/// of the EnC rows is always a MethodDef handle, so token keying is unambiguous. +/// Fail safe: a null/empty or non-PDB image yields the empty map, and a method whose +/// blobs do not decode is omitted rather than guessed. +val readEncMethodDebugInfoFromPortablePdb: pdbBytes: byte[] -> Map diff --git a/src/Compiler/AbstractIL/ilwrite.fs b/src/Compiler/AbstractIL/ilwrite.fs index 13feeab294a..3ceb32a6212 100644 --- a/src/Compiler/AbstractIL/ilwrite.fs +++ b/src/Compiler/AbstractIL/ilwrite.fs @@ -3859,7 +3859,10 @@ type options = referenceAssemblyOnly: bool referenceAssemblyAttribOpt: ILAttribute option referenceAssemblySignatureHash : int option - pathMap: PathMap } + pathMap: PathMap + /// Per-method EnC CustomDebugInformation rows for the portable PDB writer, keyed by + /// IL method name. Empty for ordinary compiles, so flag-off output stays byte-identical. + methodCustomDebugInfoRows: Map } let writeBinaryAux (stream: Stream, options: options, modul, normalizeAssemblyRefs) = @@ -4022,7 +4025,7 @@ let writeBinaryAux (stream: Stream, options: options, modul, normalizeAssemblyRe match options.pdbfile, options.portablePDB with | Some _, true -> let pdbInfo = - generatePortablePdb options.embedAllSource options.embedSourceList options.sourceLink options.checksumAlgorithm pdbData options.pathMap + generatePortablePdb options.embedAllSource options.embedSourceList options.sourceLink options.checksumAlgorithm pdbData options.pathMap options.methodCustomDebugInfoRows if options.embeddedPDB then let uncompressedLength, contentId, stream, algorithmName, checkSum = pdbInfo diff --git a/src/Compiler/AbstractIL/ilwrite.fsi b/src/Compiler/AbstractIL/ilwrite.fsi index d074f0bc584..08321664c2f 100644 --- a/src/Compiler/AbstractIL/ilwrite.fsi +++ b/src/Compiler/AbstractIL/ilwrite.fsi @@ -9,24 +9,29 @@ open FSharp.Compiler.AbstractIL.ILPdbWriter open FSharp.Compiler.AbstractIL.StrongNameSign type options = - { ilg: ILGlobals - outfile: string - pdbfile: string option - portablePDB: bool - embeddedPDB: bool - embedAllSource: bool - embedSourceList: string list - allGivenSources: ILSourceDocument list - sourceLink: string - checksumAlgorithm: HashAlgorithm - signer: ILStrongNameSigner option - emitTailcalls: bool - deterministic: bool - dumpDebugInfo: bool - referenceAssemblyOnly: bool - referenceAssemblyAttribOpt: ILAttribute option - referenceAssemblySignatureHash: int option - pathMap: PathMap } + { + ilg: ILGlobals + outfile: string + pdbfile: string option + portablePDB: bool + embeddedPDB: bool + embedAllSource: bool + embedSourceList: string list + allGivenSources: ILSourceDocument list + sourceLink: string + checksumAlgorithm: HashAlgorithm + signer: ILStrongNameSigner option + emitTailcalls: bool + deterministic: bool + dumpDebugInfo: bool + referenceAssemblyOnly: bool + referenceAssemblyAttribOpt: ILAttribute option + referenceAssemblySignatureHash: int option + pathMap: PathMap + /// Per-method EnC CustomDebugInformation rows for the portable PDB writer, keyed by + /// IL method name. Empty for ordinary compiles, so flag-off output stays byte-identical. + methodCustomDebugInfoRows: Map + } /// Write a binary to the file system. val WriteILBinaryFile: options: options * inputModule: ILModuleDef * (ILAssemblyRef -> ILAssemblyRef) -> unit diff --git a/src/Compiler/AbstractIL/ilwritepdb.fs b/src/Compiler/AbstractIL/ilwritepdb.fs index 86a19d50c6c..70f88b471d7 100644 --- a/src/Compiler/AbstractIL/ilwritepdb.fs +++ b/src/Compiler/AbstractIL/ilwritepdb.fs @@ -118,6 +118,10 @@ type PdbMethodData = DebugPoints: PdbDebugPoint array } +/// A pre-serialized CustomDebugInformation row (kind GUID + blob) to attach to a method +/// definition row in the portable PDB. +type PdbMethodCustomDebugInfo = { KindGuid: Guid; Blob: byte[] } + module SequencePoint = let orderBySource sp1 sp2 = let c1 = compare sp1.Document sp2.Document @@ -337,7 +341,15 @@ let scopeSorter (scope1: PdbMethodScope) (scope2: PdbMethodScope) = 0 type PortablePdbGenerator - (embedAllSource: bool, embedSourceList: string list, sourceLink: string, checksumAlgorithm, info: PdbData, pathMap: PathMap) = + ( + embedAllSource: bool, + embedSourceList: string list, + sourceLink: string, + checksumAlgorithm, + info: PdbData, + pathMap: PathMap, + methodCustomDebugInfoRows: Map + ) = // Deterministic: build the Document table in a stable order by mapped file path, // but preserve the original-document-index -> handle mapping by filename. @@ -488,6 +500,27 @@ type PortablePdbGenerator let moduleImportScopeHandle = MetadataTokens.ImportScopeHandle(1) let importScopesTable = Dictionary() + // Per-method CustomDebugInformation rows keyed by IL method name. Names that match + // more than one method row (overloads, same name on different types) fail closed and + // attach nothing, so a row can never land on the wrong method. + let methodCustomDebugInfoByName = + if Map.isEmpty methodCustomDebugInfoRows then + methodCustomDebugInfoRows + else + let nameCounts = Dictionary() + + for minfo in info.Methods do + nameCounts[minfo.MethName] <- + match nameCounts.TryGetValue minfo.MethName with + | true, count -> count + 1 + | _ -> 1 + + methodCustomDebugInfoRows + |> Map.filter (fun methName _ -> + match nameCounts.TryGetValue methName with + | true, 1 -> true + | _ -> false) + let serializeImport (writer: BlobBuilder) (import: PdbImport) = match import with // We don't yet emit these kinds of imports @@ -777,6 +810,23 @@ type PortablePdbGenerator metadata.AddMethodDebugInformation(docHandle, sequencePointBlob) |> ignore + // MetadataBuilder sorts the CustomDebugInformation table by parent at serialize + // time, so adding rows in method order here is safe. + match Map.tryFind minfo.MethName methodCustomDebugInfoByName with + | Some cdiRows -> + // MethToken is the uncoded token (0x06 <<< 24 ||| rid); the handle needs the rid. + let methodHandle = + MetadataTokens.MethodDefinitionHandle(minfo.MethToken &&& 0x00FFFFFF) + + for cdiRow in cdiRows do + metadata.AddCustomDebugInformation( + MethodDefinitionHandle.op_Implicit methodHandle, + metadata.GetOrAddGuid cdiRow.KindGuid, + metadata.GetOrAddBlob cdiRow.Blob + ) + |> ignore + | None -> () + match minfo.RootScope with | None -> () | Some scope -> writeMethodScopes minfo.MethToken scope @@ -831,9 +881,10 @@ let generatePortablePdb checksumAlgorithm (info: PdbData) (pathMap: PathMap) + (methodCustomDebugInfoRows: Map) = let generator = - PortablePdbGenerator(embedAllSource, embedSourceList, sourceLink, checksumAlgorithm, info, pathMap) + PortablePdbGenerator(embedAllSource, embedSourceList, sourceLink, checksumAlgorithm, info, pathMap, methodCustomDebugInfoRows) generator.Emit() diff --git a/src/Compiler/AbstractIL/ilwritepdb.fsi b/src/Compiler/AbstractIL/ilwritepdb.fsi index 5987cc165e3..09d380e44cc 100644 --- a/src/Compiler/AbstractIL/ilwritepdb.fsi +++ b/src/Compiler/AbstractIL/ilwritepdb.fsi @@ -67,6 +67,12 @@ type PdbMethodData = DebugRange: (PdbSourceLoc * PdbSourceLoc) option DebugPoints: PdbDebugPoint[] } +/// A pre-serialized CustomDebugInformation row to attach to a method definition row in +/// the portable PDB (kind GUID + blob). Supplied by the compiler as a side channel keyed +/// by IL method name. The writer attaches the rows only when the name identifies exactly +/// one method row (fail closed on ambiguity). +type PdbMethodCustomDebugInfo = { KindGuid: System.Guid; Blob: byte[] } + [] type PdbData = { @@ -109,6 +115,7 @@ val generatePortablePdb: checksumAlgorithm: HashAlgorithm -> info: PdbData -> pathMap: PathMap -> + methodCustomDebugInfoRows: Map -> int64 * BlobContentId * MemoryStream * string * byte[] val compressPortablePdbStream: stream: MemoryStream -> MemoryStream diff --git a/src/Compiler/Driver/fsc.fs b/src/Compiler/Driver/fsc.fs index 43157660212..f5aa287b6a7 100644 --- a/src/Compiler/Driver/fsc.fs +++ b/src/Compiler/Driver/fsc.fs @@ -1149,6 +1149,7 @@ let main6 referenceAssemblyAttribOpt = referenceAssemblyAttribOpt referenceAssemblySignatureHash = refAssemblySignatureHash pathMap = tcConfig.pathMap + methodCustomDebugInfoRows = Map.empty }, ilxMainModule, normalizeAssemblyRefs @@ -1180,6 +1181,7 @@ let main6 referenceAssemblyAttribOpt = None referenceAssemblySignatureHash = None pathMap = tcConfig.pathMap + methodCustomDebugInfoRows = Map.empty }, ilxMainModule, normalizeAssemblyRefs diff --git a/src/Compiler/FSharp.Compiler.Service.fsproj b/src/Compiler/FSharp.Compiler.Service.fsproj index 5510af6b3f6..2d0b77fbf69 100644 --- a/src/Compiler/FSharp.Compiler.Service.fsproj +++ b/src/Compiler/FSharp.Compiler.Service.fsproj @@ -236,6 +236,8 @@ + + diff --git a/src/Compiler/Interactive/fsi.fs b/src/Compiler/Interactive/fsi.fs index 30801263752..5c9dc813e4b 100644 --- a/src/Compiler/Interactive/fsi.fs +++ b/src/Compiler/Interactive/fsi.fs @@ -1919,6 +1919,7 @@ type internal FsiDynamicCompiler referenceAssemblyAttribOpt = None referenceAssemblySignatureHash = None pathMap = tcConfig.pathMap + methodCustomDebugInfoRows = Map.empty } let assemblyBytes, pdbBytes = WriteILBinaryInMemory(opts, ilxMainModule, id) diff --git a/tests/FSharp.Compiler.ComponentTests/CompilerService/EncMethodDebugInformationTests.fs b/tests/FSharp.Compiler.ComponentTests/CompilerService/EncMethodDebugInformationTests.fs new file mode 100644 index 00000000000..b719c06fab9 --- /dev/null +++ b/tests/FSharp.Compiler.ComponentTests/CompilerService/EncMethodDebugInformationTests.fs @@ -0,0 +1,633 @@ +// Copyright (c) Microsoft Corporation. All Rights Reserved. See License.txt in the project root for license information. + +module internal CompilerService.EncMethodDebugInformationTests + +open System +open System.Collections.Immutable +open System.IO +open System.Reflection +open System.Reflection.Metadata +open System.Reflection.Metadata.Ecma335 +open System.Reflection.PortableExecutable +open Xunit + +open Internal.Utilities +open FSharp.Compiler.AbstractIL.IL +open FSharp.Compiler.AbstractIL.ILBinaryWriter +open FSharp.Compiler.AbstractIL.ILPdbWriter +open FSharp.Compiler.AbstractIL.EncMethodDebugInformation + +// ----------------------------------------------------------------------- +// Round-trip properties (pure codec) +// ----------------------------------------------------------------------- + +[] +let ``Empty maps serialize to empty blobs and deserialize to the empty record`` () = + let info = EncMethodDebugInformation.Empty + + Assert.Empty(serializeLocalSlots info) + Assert.Empty(serializeLambdaMap info) + Assert.Empty(serializeStateMachineStates info) + + let decoded = deserialize Array.empty Array.empty Array.empty + Assert.Equal(EncMethodDebugInformation.Empty, decoded) + + // Null blobs (absent CDI rows) behave like empty ones. + let decodedNull = deserialize null null null + Assert.Equal(EncMethodDebugInformation.Empty, decodedNull) + +[] +let ``Lambda map with a single closure round-trips`` () = + let info = + { EncMethodDebugInformation.Empty with + MethodOrdinal = 0 + Closures = [ { SyntaxOffset = 3 } ] } + + let blob = serializeLambdaMap info + let methodOrdinal, closures, lambdas = deserializeLambdaMap blob + + Assert.Equal(0, methodOrdinal) + Assert.Equal([ { SyntaxOffset = 3 } ], closures) + Assert.Empty lambdas + +[] +let ``Lambda map with several lambdas and negative-baseline offsets round-trips`` () = + // Out-of-order and negative offsets exercise the syntax-offset-baseline record; + // closure ordinals cover in-range, static (-1) and this-only (-2) lambdas. + let closures = [ { SyntaxOffset = 12 }; { SyntaxOffset = -7 }; { SyntaxOffset = 3 } ] + + let lambdas = + [ { SyntaxOffset = 30; ClosureOrdinal = 1 } + { SyntaxOffset = -7; ClosureOrdinal = StaticClosureOrdinal } + { SyntaxOffset = 0; ClosureOrdinal = ThisOnlyClosureOrdinal } + { SyntaxOffset = 5; ClosureOrdinal = 2 } ] + + let info = + { EncMethodDebugInformation.Empty with + MethodOrdinal = 5 + Closures = closures + Lambdas = lambdas } + + let blob = serializeLambdaMap info + let methodOrdinal, decodedClosures, decodedLambdas = deserializeLambdaMap blob + + Assert.Equal(5, methodOrdinal) + Assert.Equal(closures, decodedClosures) + Assert.Equal(lambdas, decodedLambdas) + +[] +let ``Lambda map golden bytes match the Roslyn encoding`` () = + // methodOrdinal 0 -> compressed(1); baseline -1 -> compressed(1); one closure at + // offset 0 -> compressed(1); lambda at offset 5 -> compressed(6) with closure + // ordinal 0 -> compressed(0 - (-2)) = compressed(2). + let info = + { EncMethodDebugInformation.Empty with + MethodOrdinal = 0 + Closures = [ { SyntaxOffset = 0 } ] + Lambdas = [ { SyntaxOffset = 5; ClosureOrdinal = 0 } ] } + + Assert.Equal([| 0x01uy; 0x01uy; 0x01uy; 0x01uy; 0x06uy; 0x02uy |], serializeLambdaMap info) + +[] +let ``Lambda map rejects closure ordinals outside the valid range`` () = + let mk ordinal = + { EncMethodDebugInformation.Empty with + MethodOrdinal = 0 + Closures = [ { SyntaxOffset = 0 } ] + Lambdas = [ { SyntaxOffset = 1; ClosureOrdinal = ordinal } ] } + + Assert.Throws(fun () -> serializeLambdaMap (mk 1) |> ignore) |> ignore + Assert.Throws(fun () -> serializeLambdaMap (mk -3) |> ignore) |> ignore + +[] +let ``Slot map with temps, ordinal-flagged slots and negative offsets round-trips`` () = + let slots = + [ EncLocalSlotInfo.Temp + EncLocalSlotInfo.Slot(0, 10, 0) + EncLocalSlotInfo.Slot(MaxSerializableLocalKind, -42, 3) + EncLocalSlotInfo.Temp + EncLocalSlotInfo.Slot(7, 0, 1) ] + + let info = + { EncMethodDebugInformation.Empty with + LocalSlots = slots } + + let blob = serializeLocalSlots info + Assert.Equal(slots, deserializeLocalSlots blob) + + // The baseline record must be present (an offset below -1 exists) and must be + // the Roslyn marker byte 0xFF followed by compressed(42). + Assert.Equal(0xFFuy, blob[0]) + +[] +let ``Slot map golden bytes match the Roslyn encoding`` () = + // No offset below -1 -> no baseline record (implicit baseline -1). + // Temp -> 0x00. + // Slot(kind 0, offset 0, ordinal 0) -> byte 0x01 (kind+1), compressed(0 - (-1)) = 0x01. + // Slot(kind 1, offset 2, ordinal 3) -> byte 0x82 (kind+1, bit 7 = has ordinal), + // compressed(3), compressed(3). + let info = + { EncMethodDebugInformation.Empty with + LocalSlots = + [ EncLocalSlotInfo.Temp + EncLocalSlotInfo.Slot(0, 0, 0) + EncLocalSlotInfo.Slot(1, 2, 3) ] } + + Assert.Equal([| 0x00uy; 0x01uy; 0x01uy; 0x82uy; 0x03uy; 0x03uy |], serializeLocalSlots info) + +[] +let ``Slot map rejects kinds outside the serializable range`` () = + let mk kind = + { EncMethodDebugInformation.Empty with + LocalSlots = [ EncLocalSlotInfo.Slot(kind, 0, 0) ] } + + Assert.Throws(fun () -> serializeLocalSlots (mk -1) |> ignore) |> ignore + + Assert.Throws(fun () -> serializeLocalSlots (mk (MaxSerializableLocalKind + 1)) |> ignore) + |> ignore + +[] +let ``State machine map with negative state numbers round-trips ordered by offset`` () = + // Input deliberately unsorted; the writer orders entries by syntax offset + // (stably, so the two entries sharing offset 20 keep their relative order). + let states = + [ { StateNumber = -4; SyntaxOffset = 20 } + { StateNumber = 0; SyntaxOffset = -5 } + { StateNumber = 3; SyntaxOffset = 20 } + { StateNumber = 1; SyntaxOffset = 7 } ] + + let info = + { EncMethodDebugInformation.Empty with + StateMachineStates = states } + + let expected = + [ { StateNumber = 0; SyntaxOffset = -5 } + { StateNumber = 1; SyntaxOffset = 7 } + { StateNumber = -4; SyntaxOffset = 20 } + { StateNumber = 3; SyntaxOffset = 20 } ] + + let blob = serializeStateMachineStates info + Assert.Equal(expected, deserializeStateMachineStates blob) + +[] +let ``Full record round-trips through the three blobs`` () = + let info = + { MethodOrdinal = 2 + LocalSlots = [ EncLocalSlotInfo.Slot(0, 4, 0); EncLocalSlotInfo.Temp ] + Closures = [ { SyntaxOffset = 0 } ] + Lambdas = [ { SyntaxOffset = 9; ClosureOrdinal = 0 } ] + StateMachineStates = [ { StateNumber = 0; SyntaxOffset = 9 } ] } + + let decoded = + deserialize (serializeLocalSlots info) (serializeLambdaMap info) (serializeStateMachineStates info) + + Assert.Equal(info, decoded) + +// ----------------------------------------------------------------------- +// Occurrence-key packing +// ----------------------------------------------------------------------- + +[] +let ``Occurrence keys pack and unpack ordinal chains`` () = + // Depth 1: the key is the ordinal itself. + Assert.Equal(Some 0, tryEncodeOccurrenceKey [ 0 ]) + Assert.Equal(Some 5, tryEncodeOccurrenceKey [ 5 ]) + Assert.Equal(Some 0xFFFF, tryEncodeOccurrenceKey [ 0xFFFF ]) + Assert.Equal([ 5 ], decodeOccurrenceKey 5) + + // Depth 2: the parent segment is stored biased by one, so [0; 0] never + // collides with the depth-1 key 0. + Assert.Equal(Some 0x10000, tryEncodeOccurrenceKey [ 0; 0 ]) + Assert.Equal([ 0; 0 ], decodeOccurrenceKey 0x10000) + Assert.Equal(Some 0x40007, tryEncodeOccurrenceKey [ 3; 7 ]) + Assert.Equal([ 3; 7 ], decodeOccurrenceKey 0x40007) + + // Every encodable chain round-trips ([0x1FFE; 0xFFFD] packs to the maximum + // key 0x1FFFFFFD that still fits the compressed-integer budget after the + // baseline adjustment). + for chain in [ [ 0 ]; [ 42 ]; [ 0xFFFF ]; [ 0; 0 ]; [ 3; 7 ]; [ 0x1FFE; 0xFFFD ] ] do + match tryEncodeOccurrenceKey chain with + | Some key -> Assert.Equal(chain, decodeOccurrenceKey key) + | None -> failwith $"expected chain %A{chain} to be encodable" + +[] +let ``Occurrence key packing fails closed past its limits`` () = + // Deeper than two segments. + Assert.Equal(None, tryEncodeOccurrenceKey [ 1; 2; 3 ]) + // Empty chain. + Assert.Equal(None, tryEncodeOccurrenceKey []) + // Ordinal past 16 bits. + Assert.Equal(None, tryEncodeOccurrenceKey [ 0x10000 ]) + Assert.Equal(None, tryEncodeOccurrenceKey [ 0; 0x10000 ]) + // Parent past the compressed-integer budget (29 bits incl. the bias). + Assert.Equal(None, tryEncodeOccurrenceKey [ 0x1FFF; 0 ]) + // Packed key past the budget even though both segments are individually valid. + Assert.Equal(None, tryEncodeOccurrenceKey [ 0x1FFE; 0xFFFF ]) + // Regression: a large in-range parent whose packed key wraps NEGATIVE in int32 + // ((0xFFFE + 1) <<< 16). The int32 packing accepted the wrapped key (negative + // <= MaxOccurrenceKey), failing open; the int64 packing must reject it. + Assert.Equal(None, tryEncodeOccurrenceKey [ 0xFFFE; 0 ]) + Assert.Equal(None, tryEncodeOccurrenceKey [ 0x7FFF; 0xFFFF ]) + // Negative ordinals. + Assert.Equal(None, tryEncodeOccurrenceKey [ -1 ]) + Assert.Equal(None, tryEncodeOccurrenceKey [ -1; 0 ]) + +// ----------------------------------------------------------------------- +// Cross-validation against Roslyn-emitted blobs +// ----------------------------------------------------------------------- + +/// C# source with nested capturing lambdas, LINQ lambdas, and an async method, so a +/// debug build emits all three EnC CDI kinds. +let private crossValidationSource = + """ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; + +namespace Scratch +{ + public class Lambdas + { + public Func MakeAdder(int x) + { + int y = x + 1; + Func inner = a => a + x + y; + return b => inner(b) + x; + } + + public int UseLinq(IEnumerable items, int threshold) + { + var filtered = items.Where(i => i > threshold).Select(i => i * 2); + return filtered.Sum(i => i + threshold); + } + + public async Task ComputeAsync(int x) + { + await Task.Delay(1); + int y = x * 2; + await Task.Yield(); + Func f = a => a + y; + return f(x); + } + } +} +""" + +/// Builds the cross-validation C# library with the repo SDK (DebugType=portable) and +/// returns the path of the produced Portable PDB. +let private buildCSharpScratchPdb () = + let workDir = + Path.Combine(Path.GetTempPath(), "fsharp-enc-cdi-" + Guid.NewGuid().ToString("N")) + + Directory.CreateDirectory workDir |> ignore + let projPath = Path.Combine(workDir, "scratch.csproj") + File.WriteAllText(Path.Combine(workDir, "Scratch.cs"), crossValidationSource) + + File.WriteAllText( + projPath, + """ + + Library + net10.0 + portable + false + true + disable + + +""" + ) + + let psi = System.Diagnostics.ProcessStartInfo() + psi.FileName <- Path.Combine(__SOURCE_DIRECTORY__, "..", "..", "..", ".dotnet", "dotnet") + // ProcessStartInfo.ArgumentList does not exist on net472, so build the quoted argument + // string by hand (projPath is the only argument that can contain spaces). + psi.Arguments <- $"build \"{projPath}\" -c Debug -p:DebugType=portable -v m" + psi.RedirectStandardOutput <- true + psi.RedirectStandardError <- true + psi.WorkingDirectory <- workDir + + use p = new System.Diagnostics.Process() + p.StartInfo <- psi + p.Start() |> ignore + let stdout = p.StandardOutput.ReadToEnd() + let stderr = p.StandardError.ReadToEnd() + p.WaitForExit() + + if p.ExitCode <> 0 then + failwith $"dotnet build of the C# scratch library failed: {stdout}\n{stderr}" + + let pdbPath = Path.Combine(workDir, "bin", "Debug", "net10.0", "scratch.pdb") + Assert.True(File.Exists pdbPath, $"expected portable PDB at {pdbPath}") + workDir, pdbPath + +/// Reads all CustomDebugInformation rows of the given kind from a portable PDB, +/// returning (parent method token, blob bytes) pairs. +let private readCdiBlobs (reader: MetadataReader) (kind: Guid) = + [ for cdiHandle in reader.CustomDebugInformation do + let cdi = reader.GetCustomDebugInformation cdiHandle + + if reader.GetGuid cdi.Kind = kind then + let parent = MetadataTokens.GetToken cdi.Parent + parent, reader.GetBlobBytes cdi.Value ] + +[] +let ``Roslyn-emitted EnC CDI blobs decode and re-encode byte-for-byte`` () = + let workDir, pdbPath = buildCSharpScratchPdb () + + try + use stream = File.OpenRead pdbPath + use provider = MetadataReaderProvider.FromPortablePdbStream stream + let reader = provider.GetMetadataReader() + + // ---- EnC Lambda and Closure Map ---- + let lambdaMaps = readCdiBlobs reader PortableCustomDebugInfoKinds.encLambdaAndClosureMap + Assert.NotEmpty lambdaMaps + + let mutable totalLambdas = 0 + let mutable totalClosures = 0 + + for _, blob in lambdaMaps do + let methodOrdinal, closures, lambdas = deserializeLambdaMap blob + + // Structural sanity: defined ordinal, at least one lambda or closure, + // closure ordinals within range. + Assert.True(methodOrdinal >= 0, "Roslyn lambda maps carry a defined method ordinal") + Assert.True(not (List.isEmpty closures) || not (List.isEmpty lambdas)) + + for lambda in lambdas do + Assert.InRange(lambda.ClosureOrdinal, MinClosureOrdinal, closures.Length - 1) + + totalLambdas <- totalLambdas + lambdas.Length + totalClosures <- totalClosures + closures.Length + + // Byte-for-byte: re-encoding the decoded map must reproduce Roslyn's blob. + let reencoded = + serializeLambdaMap + { EncMethodDebugInformation.Empty with + MethodOrdinal = methodOrdinal + Closures = closures + Lambdas = lambdas } + + Assert.Equal(blob, reencoded) + + // The source has 6 lambdas (2 in MakeAdder, 3 in UseLinq, 1 in ComputeAsync) + // and capturing closures in every method. + Assert.True(totalLambdas >= 6, $"expected at least 6 lambdas, found {totalLambdas}") + Assert.True(totalClosures >= 3, $"expected at least 3 closures, found {totalClosures}") + + // ---- EnC Local Slot Map ---- + let slotMaps = readCdiBlobs reader PortableCustomDebugInfoKinds.encLocalSlotMap + Assert.NotEmpty slotMaps + + let mutable longLivedSlots = 0 + + for _, blob in slotMaps do + let slots = deserializeLocalSlots blob + Assert.NotEmpty slots + + for slot in slots do + match slot with + | EncLocalSlotInfo.Temp -> () + | EncLocalSlotInfo.Slot(kind, _, ordinal) -> + Assert.InRange(kind, 0, MaxSerializableLocalKind) + Assert.True(ordinal >= 0) + longLivedSlots <- longLivedSlots + 1 + + let reencoded = + serializeLocalSlots + { EncMethodDebugInformation.Empty with + LocalSlots = slots } + + Assert.Equal(blob, reencoded) + + Assert.True(longLivedSlots > 0, "expected at least one long-lived local slot") + + // ---- EnC State Machine State Map ---- + let stateMaps = readCdiBlobs reader PortableCustomDebugInfoKinds.encStateMachineStateMap + Assert.NotEmpty stateMaps + + for _, blob in stateMaps do + let states = deserializeStateMachineStates blob + + // ComputeAsync has two suspension points (await Task.Delay, await + // Task.Yield); the decoder enforces monotone offsets, re-check here. + Assert.True(states.Length >= 2, $"expected at least 2 states, found {states.Length}") + + let offsets = states |> List.map (fun s -> s.SyntaxOffset) + Assert.Equal(List.sort offsets, offsets) + + let reencoded = + serializeStateMachineStates + { EncMethodDebugInformation.Empty with + StateMachineStates = states } + + Assert.Equal(blob, reencoded) + finally + try + Directory.Delete(workDir, true) + with _ -> + () + +// ----------------------------------------------------------------------- +// Synthetic plumbing: exercise the real ILBinaryWriter/PortablePdbGenerator path +// (no hot reload flag, no session machinery) with a synthetic CDI row map. +// ----------------------------------------------------------------------- + +module private Plumbing = + + // A real primary-assembly reference (this process's own corelib) so ilg.typ_Object + // resolves to an external TypeRef; the IL writer requires every type's 'extends' to + // resolve to a real System.Object, even one it never loads. + let private primaryAssemblyRef = ILAssemblyRef.FromAssemblyName(typeof.Assembly.GetName()) + + let private ilg = + mkILGlobals (ILScopeRef.Assembly primaryAssemblyRef, [], ILScopeRef.Assembly primaryAssemblyRef) + + let private mkAbstractMethod (name: string) : ILMethodDef = + // MethodBody.Abstract is the smallest body shape the IL writer accepts: it still + // gets a full PdbMethodData row (token, name), but skips code/IL-body generation + // entirely, which is all this test needs. + mkILNonGenericStaticMethod (name, ILMemberAccess.Public, [], mkILReturn ILType.Void, MethodBody.Abstract) + + let private mkType (typeName: string) (methodNames: string list) : ILTypeDef = + let methods = methodNames |> List.map mkAbstractMethod |> mkILMethods + + ILTypeDef( + typeName, + TypeAttributes.Public, + ILTypeDefLayout.Auto, + [], + [], + Some ilg.typ_Object, + methods, + mkILTypeDefs [], + mkILFields [], + emptyILMethodImpls, + mkILEvents [], + mkILProperties [], + emptyILSecurityDecls, + emptyILCustomAttrsStored + ) + + /// Builds a minimal in-memory module with one type per (typeName, methodNames) pair. + /// Two types may each declare a method of the same name: the IL writer's per-type + /// method table forbids two same-named methods of the same arity *within one type* + /// (unrelated to CDI), but the CDI name-keying this test exercises is per-assembly, + /// so cross-type name clashes are exactly the ambiguous case to cover. + let buildModuleOfTypes (types: (string * string list) list) : ILModuleDef = + let typeDefs = types |> List.map (fun (typeName, methodNames) -> mkType typeName methodNames) + + let assemblyName = "EncCdiPlumbing_" + Guid.NewGuid().ToString("N") + + mkILSimpleModule + assemblyName + assemblyName + true + (4, 0) + false + (mkILTypeDefs typeDefs) + None + None + 0 + (mkILExportedTypes []) + "v4.0.30319" // Non-empty: pins the metadata version explicitly rather than relying on primaryAssemblyRef's. + + /// Builds a minimal in-memory module with one type "T" declaring 'methodNames'. + let buildModule (methodNames: string list) : ILModuleDef = buildModuleOfTypes [ "T", methodNames ] + + /// Writes 'modul' through the same in-memory ILBinaryWriter entry point fsi.fs uses for + /// dynamic assembly emission, attaching 'methodCustomDebugInfoRows' as the CDI side + /// channel. No hot reload flag or session state is involved. + let writeInMemory (modul: ILModuleDef) (methodCustomDebugInfoRows: Map) = + let options: options = + { + ilg = ilg + outfile = "test.dll" + pdbfile = Some "test.pdb" + portablePDB = true + embeddedPDB = false + embedAllSource = false + embedSourceList = [] + allGivenSources = [] + sourceLink = "" + checksumAlgorithm = HashAlgorithm.Sha256 + signer = None + emitTailcalls = true + deterministic = false + dumpDebugInfo = false + referenceAssemblyOnly = false + referenceAssemblyAttribOpt = None + referenceAssemblySignatureHash = None + pathMap = PathMap.empty + methodCustomDebugInfoRows = methodCustomDebugInfoRows + } + + match WriteILBinaryInMemory(options, modul, id) with + | assemblyBytes, Some pdbBytes -> assemblyBytes, pdbBytes + | _, None -> failwith "expected a portable PDB to be produced" + + type CdiRow = + { + MethodName: string option + Kind: Guid + Blob: byte[] + } + + /// All method-parented CustomDebugInformation rows in the produced PDB, read back with + /// System.Reflection.Metadata (independent of this codebase's own decoders), resolving + /// each row's parent MethodDef token to its name via the companion assembly image. + let readAllCdiRows (assemblyBytes: byte[]) (pdbBytes: byte[]) : CdiRow list = + use peReader = new PEReader(ImmutableArray.CreateRange assemblyBytes) + let peMdReader = peReader.GetMetadataReader() + + use pdbProvider = MetadataReaderProvider.FromPortablePdbImage(ImmutableArray.CreateRange pdbBytes) + let pdbMdReader = pdbProvider.GetMetadataReader() + + let methodTokenToName = + [ for h: MethodDefinitionHandle in peMdReader.MethodDefinitions -> + MetadataTokens.GetToken(MethodDefinitionHandle.op_Implicit h: EntityHandle), + peMdReader.GetString(peMdReader.GetMethodDefinition(h).Name) ] + |> Map.ofList + + [ for cdiHandle in pdbMdReader.CustomDebugInformation do + let cdi = pdbMdReader.GetCustomDebugInformation cdiHandle + + if cdi.Parent.Kind = HandleKind.MethodDefinition then + { + MethodName = Map.tryFind (MetadataTokens.GetToken cdi.Parent) methodTokenToName + Kind = pdbMdReader.GetGuid cdi.Kind + Blob = pdbMdReader.GetBlobBytes cdi.Value + } ] + +[] +let ``Synthetic CustomDebugInformation row attaches to the right MethodDef`` () = + let modul = Plumbing.buildModule [ "Foo"; "Bar" ] + + let blob = + serializeLambdaMap + { EncMethodDebugInformation.Empty with + MethodOrdinal = 0 + Closures = [ { SyntaxOffset = 0 } ] + Lambdas = [ { SyntaxOffset = 5; ClosureOrdinal = 0 } ] } + + let rows = + Map.ofList [ "Foo", [ { KindGuid = PortableCustomDebugInfoKinds.encLambdaAndClosureMap; Blob = blob } ] ] + + let assemblyBytes, pdbBytes = Plumbing.writeInMemory modul rows + let cdiRows = Plumbing.readAllCdiRows assemblyBytes pdbBytes + + let row = Assert.Single cdiRows + Assert.Equal(Some "Foo", row.MethodName) + Assert.Equal(PortableCustomDebugInfoKinds.encLambdaAndClosureMap, row.Kind) + Assert.Equal(blob, row.Blob) + + // Full circle: the codec decodes exactly what was written. + let methodOrdinal, closures, lambdas = deserializeLambdaMap row.Blob + Assert.Equal(0, methodOrdinal) + Assert.Equal([ { SyntaxOffset = 0 } ], closures) + Assert.Equal([ { SyntaxOffset = 5; ClosureOrdinal = 0 } ], lambdas) + +[] +let ``Empty map produces zero CustomDebugInformation rows`` () = + let modul = Plumbing.buildModule [ "Foo" ] + let assemblyBytes, pdbBytes = Plumbing.writeInMemory modul Map.empty + Assert.Empty(Plumbing.readAllCdiRows assemblyBytes pdbBytes) + +[] +let ``A method name absent from the module attaches nothing`` () = + // Fail closed, matching the feature this codec ports from: an unresolvable name is + // silently dropped rather than raising, so it can never attach to the wrong method. + let modul = Plumbing.buildModule [ "Foo" ] + + let blob = + serializeStateMachineStates + { EncMethodDebugInformation.Empty with + StateMachineStates = [ { StateNumber = 0; SyntaxOffset = 1 } ] } + + let rows = + Map.ofList [ "DoesNotExist", [ { KindGuid = PortableCustomDebugInfoKinds.encStateMachineStateMap; Blob = blob } ] ] + + let assemblyBytes, pdbBytes = Plumbing.writeInMemory modul rows + Assert.Empty(Plumbing.readAllCdiRows assemblyBytes pdbBytes) + +[] +let ``An ambiguous method name attaches to neither method`` () = + // Two distinct types each declaring a "Dup" method: the CDI name-keying in + // PortablePdbGenerator is per-assembly (IL method name only, not qualified by + // declaring type), so this reproduces the ambiguous case without hitting the + // unrelated IL writer invariant that forbids two same-named/same-arity methods + // within a single type. + let modul = Plumbing.buildModuleOfTypes [ "T1", [ "Dup" ]; "T2", [ "Dup" ] ] + + let blob = + serializeStateMachineStates + { EncMethodDebugInformation.Empty with + StateMachineStates = [ { StateNumber = 0; SyntaxOffset = 1 } ] } + + let rows = + Map.ofList [ "Dup", [ { KindGuid = PortableCustomDebugInfoKinds.encStateMachineStateMap; Blob = blob } ] ] + + let assemblyBytes, pdbBytes = Plumbing.writeInMemory modul rows + Assert.Empty(Plumbing.readAllCdiRows assemblyBytes pdbBytes) diff --git a/tests/FSharp.Compiler.ComponentTests/FSharp.Compiler.ComponentTests.fsproj b/tests/FSharp.Compiler.ComponentTests/FSharp.Compiler.ComponentTests.fsproj index 881ae13942b..777529c1742 100644 --- a/tests/FSharp.Compiler.ComponentTests/FSharp.Compiler.ComponentTests.fsproj +++ b/tests/FSharp.Compiler.ComponentTests/FSharp.Compiler.ComponentTests.fsproj @@ -465,6 +465,7 @@ +