Handle package json lifetime

- All the package json watched are ref counted if watched
- Not watched package json locations are released
   - First of this kind are when resolution from global cache fails, we dont watch those locations so not safe to have them cached
   - If we are looking for a file and the file is not found, the package json locations looked up are not watched

Author: sheetalkamat

src/compiler/program.ts

@@ -3791,6 +3791,9 @@ export function createProgram(_rootNamesOrOptions: readonly string[] | CreatePro
             }
             (filesWithReferencesProcessed ??= new Set()).add(file.path);
         }
+        else {
+            host.onSourceFileNotCreated?.(sourceFileOptions);
+        }
         return file;
     }
 

src/compiler/resolutionCache.ts

@@ -7,6 +7,7 @@ import {
     CompilerOptions,
     createModeAwareCache,
     createModuleResolutionCache,
+    CreateSourceFileOptions,
     createTypeReferenceDirectiveResolutionCache,
     createTypeReferenceResolutionLoader,
     Debug,
@@ -53,7 +54,6 @@ import {
     noopFileWatcher,
     normalizePath,
     packageIdToString,
-    PackageJsonInfoCacheEntry,
     parseNodeModuleFromPath,
     Path,
     PathPathComponents,
@@ -111,6 +111,7 @@ export interface ResolutionCache extends Required<CompilerHostSupportingResoluti
     resolvedFileToResolution: Map<Path, Set<ResolutionWithFailedLookupLocations>>;
     resolutionsWithFailedLookups: Set<ResolutionWithFailedLookupLocations>;
     resolutionsWithOnlyAffectingLocations: Set<ResolutionWithFailedLookupLocations>;
+    packageJsonRefCount: Map<Path, number>;
     directoryWatchesOfFailedLookups: Map<Path, DirectoryWatchesOfFailedLookup>;
     fileWatchesOfAffectingLocations: Map<string, FileWatcherOfAffectingLocation>;
     packageDirWatchers: Map<Path, PackageDirWatcher>;
@@ -693,6 +694,8 @@ export function createResolutionCache(
     let resolutionsResolvedWithGlobalCache = 0;
     let resolutionsResolvedWithoutGlobalCache = 0;
 
+    const packageJsonRefCount = new Map<Path, number>();
+    let potentiallyUnwatchedPackageJsons: Set<Path> | undefined;
     const directoryWatchesOfFailedLookups = new Map<Path, DirectoryWatchesOfFailedLookup>();
     const fileWatchesOfAffectingLocations = new Map<string, FileWatcherOfAffectingLocation>();
     const rootDir = getRootDirectoryOfResolutionCache(rootDirForResolution, getCurrentDirectory);
@@ -718,6 +721,7 @@ export function createResolutionCache(
         resolvedFileToResolution,
         resolutionsWithFailedLookups,
         resolutionsWithOnlyAffectingLocations,
+        packageJsonRefCount,
         directoryWatchesOfFailedLookups,
         fileWatchesOfAffectingLocations,
         packageDirWatchers,
@@ -734,6 +738,7 @@ export function createResolutionCache(
         resolveTypeReferenceDirectiveReferences,
         onReusedModuleResolutions,
         onReusedTypeReferenceDirectiveResolutions,
+        onSourceFileNotCreated,
         resolveLibrary,
         resolveSingleModuleNameWithoutWatching,
         removeResolutionsFromProjectReferenceRedirects,
@@ -754,9 +759,11 @@ export function createResolutionCache(
     function clear() {
         potentiallyUnreferencedResolutions = undefined;
         potentiallyUnreferencedDirWatchers = undefined;
+        potentiallyUnwatchedPackageJsons = undefined;
         newUnresolvedResolutionCachePassResolutions = undefined;
         clearMap(directoryWatchesOfFailedLookups, closeFileWatcherOf);
         clearMap(fileWatchesOfAffectingLocations, closeFileWatcherOf);
+        packageJsonRefCount.clear();
         isSymlinkCache.clear();
         packageDirWatchers.clear();
         dirPathToSymlinkPackageRefCount.clear();
@@ -890,6 +897,8 @@ export function createResolutionCache(
             potentiallyUnreferencedResolutions = undefined;
         }
         hasChangedAutomaticTypeDirectiveNames = false;
+        potentiallyUnwatchedPackageJsons?.forEach(releasePotentiallyUnwatchedPackageJson);
+        potentiallyUnwatchedPackageJsons = undefined;
         if (!skipCacheCompact) compactCaches(newProgram);
         moduleResolutionCache.isReadonly = true;
         typeReferenceDirectiveResolutionCache.isReadonly = true;
@@ -954,13 +963,37 @@ export function createResolutionCache(
         }
     }
 
+    function releasePotentiallyUnwatchedPackageJson(path: Path) {
+        if (!packageJsonRefCount.has(path)) moduleResolutionCache.getPackageJsonInfoCache().getInternalMap()?.delete(path);
+    }
+
+    function releasePackageJsonCachePath(path: Path) {
+        moduleResolutionCache.getPackageJsonInfoCache().getInternalMap()?.delete(path);
+        packageJsonRefCount.delete(path);
+    }
+
+    function releasePackageJson(path: Path) {
+        const existing = packageJsonRefCount.get(path)!;
+        if (existing !== 1) packageJsonRefCount.set(path, existing - 1);
+        else releasePackageJsonCachePath(path);
+    }
+
+    function addRefToPackageJson(path: Path) {
+        packageJsonRefCount.set(path, (packageJsonRefCount.get(path) ?? 0) + 1);
+    }
+
     function closeFileWatcherOfAffectingLocation(watcher: FileWatcherOfAffectingLocation, path: string) {
         if (watcher.files === 0 && watcher.resolutions === 0 && !watcher.symlinks?.size) {
             fileWatchesOfAffectingLocations.delete(path);
+            releasePackageJson(resolutionHost.toPath(path));
             watcher.watcher.close();
         }
     }
 
+    function onSourceFileNotCreated(sourceFileOptions: CreateSourceFileOptions) {
+        sourceFileOptions.packageJsonLocations?.forEach(addToPotentiallyUnwatchedPackageJsons);
+    }
+
     function getValidResolution<T extends ResolutionWithFailedLookupLocations>(resolution: T | undefined) {
         return isInvalidatedResolution(resolution) ? undefined : resolution;
     }
@@ -1301,6 +1334,14 @@ export function createResolutionCache(
         watchFailedLookupLocationOfResolution(resolution);
         watchAffectingLocationsOfResolution(resolution);
         if (!firstTime) return;
+        if (resolution.globalCacheResolution && !resolution.globalCacheResolution.resolution.resolvedModule) {
+            // Add to potentially unreferenced resolutions
+            resolution.globalCacheResolution.resolution.failedLookupLocations?.forEach(
+                addToPotentiallyUnwatchedPackageJsonsIfPackageJson,
+            );
+            if (resolution.globalCacheResolution.resolution.alternateResult) addToPotentiallyUnwatchedPackageJsonsIfPackageJson(resolution.globalCacheResolution.resolution.alternateResult);
+            resolution.globalCacheResolution.resolution.affectingLocations?.forEach(addToPotentiallyUnwatchedPackageJsons);
+        }
         if (isResolvedWithGlobalCachePass(resolution)) resolutionsResolvedWithGlobalCache++;
         else if (isResolvedWithoutGlobalCachePass(resolution)) resolutionsResolvedWithoutGlobalCache++;
         const resolved = getResolutionWithResolvedFileName(resolution);
@@ -1312,8 +1353,17 @@ export function createResolutionCache(
         }
     }
 
+    function addToPotentiallyUnwatchedPackageJsonsIfPackageJson(location: string) {
+        if (endsWith(location, "/package.json")) addToPotentiallyUnwatchedPackageJsons(location);
+    }
+
+    function addToPotentiallyUnwatchedPackageJsons(location: string) {
+        (potentiallyUnwatchedPackageJsons ??= new Set()).add(resolutionHost.toPath(location));
+    }
+
     function watchFailedLookupLocation(failedLookupLocation: string) {
         const failedLookupLocationPath = resolutionHost.toPath(failedLookupLocation);
+        if (endsWith(failedLookupLocationPath, "/package.json")) addRefToPackageJson(failedLookupLocationPath);
         const toWatch = getDirectoryToWatchFailedLookupLocation(
             failedLookupLocation,
             failedLookupLocationPath,
@@ -1398,14 +1448,15 @@ export function createResolutionCache(
                 watcher: canWatchAffectingLocation(resolutionHost.toPath(locationToWatch)) ?
                     resolutionHost.watchAffectingFileLocation(locationToWatch, (fileName, eventKind) => {
                         cachedDirectoryStructureHost?.addOrDeleteFile(fileName, resolutionHost.toPath(locationToWatch), eventKind);
-                        invalidateAffectingFileWatcher(locationToWatch, moduleResolutionCache.getPackageJsonInfoCache().getInternalMap());
+                        invalidateAffectingFileWatcher(locationToWatch);
                         resolutionHost.scheduleInvalidateResolutionsOfFailedLookupLocations();
                     }) : noopFileWatcher,
                 resolutions: isSymlink ? 0 : resolutions,
                 files: isSymlink ? 0 : files,
                 symlinks: undefined,
             };
             fileWatchesOfAffectingLocations.set(locationToWatch, watcher);
+            addRefToPackageJson(resolutionHost.toPath(locationToWatch));
             if (isSymlink) symlinkWatcher = watcher;
         }
         if (isSymlink) {
@@ -1417,6 +1468,7 @@ export function createResolutionCache(
                         // Close symlink watcher if no ref
                         if (symlinkWatcher?.symlinks?.delete(affectingLocation) && !symlinkWatcher.symlinks.size && !symlinkWatcher.resolutions && !symlinkWatcher.files) {
                             fileWatchesOfAffectingLocations.delete(locationToWatch);
+                            releasePackageJson(resolutionHost.toPath(locationToWatch));
                             symlinkWatcher.watcher.close();
                         }
                     },
@@ -1426,16 +1478,17 @@ export function createResolutionCache(
                 symlinks: undefined,
             };
             fileWatchesOfAffectingLocations.set(affectingLocation, watcher);
+            addRefToPackageJson(resolutionHost.toPath(affectingLocation));
             (symlinkWatcher.symlinks ??= new Set()).add(affectingLocation);
         }
     }
 
-    function invalidateAffectingFileWatcher(path: string, packageJsonMap: Map<Path, PackageJsonInfoCacheEntry> | undefined) {
+    function invalidateAffectingFileWatcher(path: string) {
         const watcher = fileWatchesOfAffectingLocations.get(path);
         if (watcher?.resolutions) (affectingPathChecks ??= new Set()).add(path);
         if (watcher?.files) (affectingPathChecksForFile ??= new Set()).add(path);
-        watcher?.symlinks?.forEach(path => invalidateAffectingFileWatcher(path, packageJsonMap));
-        packageJsonMap?.delete(resolutionHost.toPath(path));
+        moduleResolutionCache.getPackageJsonInfoCache().getInternalMap()?.delete(resolutionHost.toPath(path));
+        watcher?.symlinks?.forEach(path => invalidateAffectingFileWatcher(path));
     }
 
     function createDirectoryWatcherForPackageDir(
@@ -1523,6 +1576,7 @@ export function createResolutionCache(
 
     function stopWatchFailedLookupLocation(failedLookupLocation: string) {
         const failedLookupLocationPath = resolutionHost.toPath(failedLookupLocation);
+        if (endsWith(failedLookupLocationPath, "/package.json")) releasePackageJson(failedLookupLocationPath);
         const toWatch = getDirectoryToWatchFailedLookupLocation(
             failedLookupLocation,
             failedLookupLocationPath,

src/compiler/types.ts

@@ -8199,6 +8199,7 @@ export interface CompilerHostSupportingResolutionCache {
         redirectedReference: ResolvedProjectReference | undefined,
         options: CompilerOptions,
     ): void;
+    onSourceFileNotCreated?(sourceFileOptions: CreateSourceFileOptions): void;
 }
 /** @internal */
 export interface CompilerHost extends CompilerHostSupportingResolutionCache {

src/compiler/watchPublic.ts

@@ -524,6 +524,7 @@ export function createWatchProgram<T extends BuilderProgram>(host: WatchCompiler
             getDirectoryPath(getNormalizedAbsolutePath(configFileName, currentDirectory)) :
             currentDirectory,
     );
+    compilerHost.onSourceFileNotCreated = resolutionCache.onSourceFileNotCreated.bind(resolutionCache);
     // Resolve module using host module resolution strategy if provided otherwise use resolution cache to resolve module names
     compilerHost.resolveModuleNameLiterals = maybeBind(host, host.resolveModuleNameLiterals);
     compilerHost.resolveModuleNames = maybeBind(host, host.resolveModuleNames);

src/harness/incrementalUtils.ts

@@ -435,6 +435,7 @@ export function verifyResolutionCache(
 
     // Verify that caches are same:
     forEachModuleOrTypeRefOrLibCache((cache, cacheType) => verifyModuleOrTypeResolutionCache(cache, actual[cacheType], cacheType));
+    verifyPackageJsonWatchInfo();
 
     // Stop watching resolutions to verify everything gets closed.
     expected.startCachingPerDirectoryResolution();
@@ -453,6 +454,11 @@ export function verifyResolutionCache(
     ts.Debug.assert(expected.countResolutionsResolvedWithGlobalCache() === 0, `${projectName}:: ResolutionsResolvedWithGlobalCache should be cleared`);
     ts.Debug.assert(expected.countResolutionsResolvedWithoutGlobalCache() === 0, `${projectName}:: ResolutionsResolvedWithoutGlobalCache should be cleared`);
     forEachModuleOrTypeRefOrLibCache((cache, cacheType) => verifyModuleOrTypeResolutionCacheIsEmpty(cache, cacheType, /*compacted*/ false));
+    ts.Debug.assert(expected.packageJsonRefCount.size === 0, `${projectName}:: packageJsonRefCount should be cleared`);
+    ts.Debug.assert(
+        expected.moduleResolutionCache.getPackageJsonInfoCache().getInternalMap() === undefined || expected.moduleResolutionCache.getPackageJsonInfoCache().getInternalMap()!.size === 0,
+        `${projectName}:: Shouldnt have any packageJson entries`,
+    );
 
     expected.compactCaches(/*newProgram*/ undefined);
     forEachModuleOrTypeRefOrLibCache((cache, cacheType) => verifyModuleOrTypeResolutionCacheIsEmpty(cache, cacheType, /*compacted*/ true));
@@ -803,6 +809,32 @@ export function verifyResolutionCache(
             }
         });
     }
+
+    function verifyPackageJsonWatchInfo() {
+        verifyMap(
+            expected.packageJsonRefCount,
+            actual.packageJsonRefCount,
+            (expectedRefCount, actualRefCount, caption) =>
+                ts.Debug.assert(
+                    expectedRefCount === actualRefCount,
+                    `${projectName}:: ${caption}`,
+                ),
+            "packageJsonRefCount",
+        );
+        verifyMap(
+            actual.packageJsonRefCount,
+            actual.moduleResolutionCache.getPackageJsonInfoCache().getInternalMap(),
+            (refCount, packageJsonCacheEntry, caption) => {
+                // Ok to have refcount on entry not in cache
+                if (!packageJsonCacheEntry) return;
+                ts.Debug.assert(
+                    !!refCount,
+                    `${projectName}:: ${caption}:: expected ref count on packageJson as there is entry in the cache`,
+                );
+            },
+            "Package Json cache watched",
+        );
+    }
 }
 
 function verifyMap<Key extends string, Expected, Actual>(

src/server/project.ts

@@ -24,6 +24,7 @@ import {
     createCacheableExportInfoMap,
     createLanguageService,
     createResolutionCache,
+    CreateSourceFileOptions,
     createSymlinkCache,
     Debug,
     Diagnostic,
@@ -884,6 +885,11 @@ export abstract class Project implements LanguageServiceHost, ModuleResolutionHo
         return this.resolutionCache.resolveLibrary(libraryName, resolveFrom, options, libFileName);
     }
 
+    /** @internal */
+    onSourceFileNotCreated(sourceFileOptions: CreateSourceFileOptions): void {
+        this.resolutionCache.onSourceFileNotCreated(sourceFileOptions);
+    }
+
     directoryExists(path: string): boolean {
         return this.directoryStructureHost.directoryExists!(path); // TODO: GH#18217
     }

src/services/services.ts

@@ -1773,6 +1773,7 @@ export function createLanguageService(
             onReusedModuleResolutions: maybeBind(host, host.onReusedModuleResolutions),
             onReusedTypeReferenceDirectiveResolutions: maybeBind(host, host.onReusedTypeReferenceDirectiveResolutions),
             resolveLibrary: maybeBind(host, host.resolveLibrary),
+            onSourceFileNotCreated: maybeBind(host, host.onSourceFileNotCreated),
             useSourceOfProjectReferenceRedirect: maybeBind(host, host.useSourceOfProjectReferenceRedirect),
             getParsedCommandLine,
             jsDocParsingMode: host.jsDocParsingMode,

tests/baselines/reference/tscWatch/libraryResolution/with-config.js

@@ -1984,13 +1984,13 @@ File '/home/src/workspace/projects/node_modules/@typescript/lib-dom/index.d.ts'
 Resolving real path for '/home/src/workspace/projects/node_modules/@typescript/lib-dom/index.d.ts', result '/home/src/workspace/projects/node_modules/@typescript/lib-dom/index.d.ts'.
 ======== Module name '@typescript/lib-dom' was successfully resolved to '/home/src/workspace/projects/node_modules/@typescript/lib-dom/index.d.ts'. ========
 File '/home/src/workspace/projects/node_modules/@typescript/lib-dom/package.json' does not exist according to earlier cached lookups.
-File '/home/src/workspace/projects/node_modules/@typescript/package.json' does not exist according to earlier cached lookups.
-File '/home/src/workspace/projects/node_modules/package.json' does not exist according to earlier cached lookups.
-File '/home/src/workspace/projects/package.json' does not exist according to earlier cached lookups.
-File '/home/src/workspace/package.json' does not exist according to earlier cached lookups.
-File '/home/src/package.json' does not exist according to earlier cached lookups.
-File '/home/package.json' does not exist according to earlier cached lookups.
-File '/package.json' does not exist according to earlier cached lookups.
+File '/home/src/workspace/projects/node_modules/@typescript/package.json' does not exist.
+File '/home/src/workspace/projects/node_modules/package.json' does not exist.
+File '/home/src/workspace/projects/package.json' does not exist.
+File '/home/src/workspace/package.json' does not exist.
+File '/home/src/package.json' does not exist.
+File '/home/package.json' does not exist.
+File '/package.json' does not exist.
 FileWatcher:: Added:: WatchInfo: /home/src/workspace/projects/node_modules/@typescript/lib-dom/index.d.ts 250 undefined Source file
 FileWatcher:: Close:: WatchInfo: /home/src/tslibs/TS/Lib/lib.dom.d.ts 250 undefined Source file
 FileWatcher:: Added:: WatchInfo: /home/src/workspace/projects/node_modules/@typescript/lib-dom/package.json 2000 undefined File location affecting resolution

tests/baselines/reference/tscWatch/libraryResolution/without-config.js

@@ -1108,13 +1108,13 @@ File '/home/src/workspace/projects/node_modules/@typescript/lib-webworker/index.
 Resolving real path for '/home/src/workspace/projects/node_modules/@typescript/lib-webworker/index.d.ts', result '/home/src/workspace/projects/node_modules/@typescript/lib-webworker/index.d.ts'.
 ======== Module name '@typescript/lib-webworker' was successfully resolved to '/home/src/workspace/projects/node_modules/@typescript/lib-webworker/index.d.ts'. ========
 File '/home/src/workspace/projects/node_modules/@typescript/lib-webworker/package.json' does not exist according to earlier cached lookups.
-File '/home/src/workspace/projects/node_modules/@typescript/package.json' does not exist according to earlier cached lookups.
-File '/home/src/workspace/projects/node_modules/package.json' does not exist according to earlier cached lookups.
-File '/home/src/workspace/projects/package.json' does not exist according to earlier cached lookups.
-File '/home/src/workspace/package.json' does not exist according to earlier cached lookups.
-File '/home/src/package.json' does not exist according to earlier cached lookups.
-File '/home/package.json' does not exist according to earlier cached lookups.
-File '/package.json' does not exist according to earlier cached lookups.
+File '/home/src/workspace/projects/node_modules/@typescript/package.json' does not exist.
+File '/home/src/workspace/projects/node_modules/package.json' does not exist.
+File '/home/src/workspace/projects/package.json' does not exist.
+File '/home/src/workspace/package.json' does not exist.
+File '/home/src/package.json' does not exist.
+File '/home/package.json' does not exist.
+File '/package.json' does not exist.
 FileWatcher:: Added:: WatchInfo: /home/src/workspace/projects/node_modules/@typescript/lib-webworker/index.d.ts 250 undefined Source file
 Reusing resolution of module '@typescript/lib-scripthost' from '/home/src/workspace/projects/__lib_node_modules_lookup_lib.scripthost.d.ts__.ts' of old program, it was not resolved.
 Reusing resolution of module '@typescript/lib-es5' from '/home/src/workspace/projects/__lib_node_modules_lookup_lib.es5.d.ts__.ts' of old program, it was not resolved.