Skip to content

Commit cd198b0

Browse files
zhhyuzhhyu
authored
Typo squatting checklist retrieval (#6361)
* Typo squatting checklist retrieval * refactor and delete PackagesCheckListService * update codes based on the review * Change linq and Delete concurrent dictionary for normalized Ids * Update index migration files and Fix issues in reviews * include ID in the index * Delete unnecessary codes in Facts * Fix threshold
1 parent f938ec7 commit cd198b0

7 files changed

Lines changed: 316 additions & 91 deletions

‎src/NuGetGallery/Migrations/201808312302101_AddIsVerifiedDownloadCountIdIndexForPackageRegistrationsTable.Designer.cs‎

Lines changed: 29 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎src/NuGetGallery/Migrations/201808312302101_AddIsVerifiedDownloadCountIdIndexForPackageRegistrationsTable.cs‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
namespace NuGetGallery.Migrations
2+
{
3+
using System;
4+
using System.Data.Entity.Migrations;
5+
6+
public partial class AddIsVerifiedDownloadCountIdIndexForPackageRegistrationsTable : DbMigration
7+
{
8+
public override void Up()
9+
{
10+
// Used for getting checklist for typosquatting
11+
Sql("CREATE NONCLUSTERED INDEX [IX_PackageRegistration_IsVerified_DownloadCount] ON [dbo].[PackageRegistrations] ([IsVerified], [DownloadCount]) INCLUDE ([Id])");
12+
}
13+
14+
public override void Down()
15+
{
16+
DropIndex("PackageRegistrations", name: "IX_PackageRegistration_IsVerified_DownloadCount");
17+
}
18+
}
19+
}

‎src/NuGetGallery/Migrations/201808312302101_AddIsVerifiedDownloadCountIdIndexForPackageRegistrationsTable.resx‎

Lines changed: 126 additions & 0 deletions
Large diffs are not rendered by default.

‎src/NuGetGallery/NuGetGallery.csproj‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -362,6 +362,10 @@
362362
<Compile Include="Migrations\201808032317064_FixSymbolCreatedColumnEFIssue.Designer.cs">
363363
<DependentUpon>201808032317064_FixSymbolCreatedColumnEFIssue.cs</DependentUpon>
364364
</Compile>
365+
<Compile Include="Migrations\201808312302101_AddIsVerifiedDownloadCountIdIndexForPackageRegistrationsTable.cs" />
366+
<Compile Include="Migrations\201808312302101_AddIsVerifiedDownloadCountIdIndexForPackageRegistrationsTable.Designer.cs">
367+
<DependentUpon>201808312302101_AddIsVerifiedDownloadCountIdIndexForPackageRegistrationsTable.cs</DependentUpon>
368+
</Compile>
365369
<Compile Include="RequestModels\DeleteAccountRequest.cs" />
366370
<Compile Include="Migrations\201710301857232_Organizations.cs" />
367371
<Compile Include="Migrations\201710301857232_Organizations.Designer.cs">
@@ -392,6 +396,8 @@
392396
<Compile Include="Services\ISymbolPackageUploadService.cs" />
393397
<Compile Include="Services\ISymbolPackageFileService.cs" />
394398
<Compile Include="Services\ISymbolsConfiguration.cs" />
399+
<Compile Include="Services\ITyposquattingCheckService.cs" />
400+
<Compile Include="Services\ITyposquattingUserService.cs" />
395401
<Compile Include="Services\SymbolPackageUploadService.cs" />
396402
<Compile Include="Services\SymbolPackageFileService.cs" />
397403
<Compile Include="Services\SymbolPackageService.cs" />
@@ -500,6 +506,10 @@
500506
<Compile Include="Services\ReservedNamespaceService.cs" />
501507
<Compile Include="Services\ReadMeService.cs" />
502508
<Compile Include="Services\TelemetryClientWrapper.cs" />
509+
<Compile Include="Services\TyposquattingCheckService.cs" />
510+
<Compile Include="Services\TyposquattingDistanceCalculation.cs" />
511+
<Compile Include="Services\TyposquattingStringNormalization.cs" />
512+
<Compile Include="Services\TyposquattingUserService.cs" />
503513
<Compile Include="Services\ValidationService.cs" />
504514
<Compile Include="Telemetry\ClientInformationTelemetryEnricher.cs" />
505515
<Compile Include="Telemetry\ClientTelemetryPIIProcessor.cs" />
@@ -1558,6 +1568,9 @@
15581568
<EmbeddedResource Include="Migrations\201808032317064_FixSymbolCreatedColumnEFIssue.resx">
15591569
<DependentUpon>201808032317064_FixSymbolCreatedColumnEFIssue.cs</DependentUpon>
15601570
</EmbeddedResource>
1571+
<EmbeddedResource Include="Migrations\201808312302101_AddIsVerifiedDownloadCountIdIndexForPackageRegistrationsTable.resx">
1572+
<DependentUpon>201808312302101_AddIsVerifiedDownloadCountIdIndexForPackageRegistrationsTable.cs</DependentUpon>
1573+
</EmbeddedResource>
15611574
<EmbeddedResource Include="OData\QueryAllowed\Data\apiv1packages.json" />
15621575
<EmbeddedResource Include="OData\QueryAllowed\Data\apiv1search.json" />
15631576
<EmbeddedResource Include="OData\QueryAllowed\Data\apiv2getupdates.json" />

‎src/NuGetGallery/Services/TyposquattingCheckService.cs‎

Lines changed: 43 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -2,34 +2,37 @@
22
// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
33

44
using System;
5+
using System.Collections.Concurrent;
56
using System.Collections.Generic;
6-
using System.Threading;
77
using System.Threading.Tasks;
8+
using System.Linq;
89

910
namespace NuGetGallery
1011
{
1112
public class TyposquattingCheckService : ITyposquattingCheckService
1213
{
14+
// TODO: Length of checklist will be saved in the configuration file.
15+
// https://github.com/NuGet/Engineering/issues/1645
16+
private const int TyposquattingCheckListLength = 20000;
17+
1318
// TODO: Threshold parameters will be saved in the configuration file.
1419
// https://github.com/NuGet/Engineering/issues/1645
15-
private static List<ThresholdInfo> _thresholdsList = new List<ThresholdInfo>
20+
private static readonly IReadOnlyList<ThresholdInfo> ThresholdsList = new List<ThresholdInfo>
1621
{
17-
new ThresholdInfo { LowerBound = 0, UpperBound = 30, Threshold = 0 },
18-
new ThresholdInfo { LowerBound = 30, UpperBound = 50, Threshold = 1 },
19-
new ThresholdInfo { LowerBound = 50, UpperBound = 120, Threshold = 2 }
22+
new ThresholdInfo (lowerBound: 0, upperBound: 30, threshold: 0),
23+
new ThresholdInfo (lowerBound: 30, upperBound: 50, threshold: 1),
24+
new ThresholdInfo (lowerBound: 50, upperBound: 121, threshold: 2)
2025
};
21-
22-
// TODO: popular packages checklist will be implemented
23-
// https://github.com/NuGet/Engineering/issues/1624
24-
public static List<PackageInfo> PackagesCheckList { get; set; }
2526

2627
private readonly ITyposquattingUserService _userTyposquattingService;
28+
private readonly IEntityRepository<PackageRegistration> _packageRegistrationRepository;
2729

28-
public TyposquattingCheckService(ITyposquattingUserService typosquattingUserService)
30+
public TyposquattingCheckService(ITyposquattingUserService typosquattingUserService, IEntityRepository<PackageRegistration> packageRegistrationRepository)
2931
{
3032
_userTyposquattingService = typosquattingUserService ?? throw new ArgumentNullException(nameof(typosquattingUserService));
33+
_packageRegistrationRepository = packageRegistrationRepository ?? throw new ArgumentNullException(nameof(packageRegistrationRepository));
3134
}
32-
35+
3336
public bool IsUploadedPackageIdTyposquatting(string uploadedPackageId, User uploadedPackageOwner)
3437
{
3538
if (uploadedPackageId == null)
@@ -42,38 +45,42 @@ public bool IsUploadedPackageIdTyposquatting(string uploadedPackageId, User uplo
4245
throw new ArgumentNullException(nameof(uploadedPackageOwner));
4346
}
4447

48+
var packagesCheckList = _packageRegistrationRepository.GetAll()
49+
.OrderByDescending(pr => pr.IsVerified)
50+
.ThenByDescending(pr => pr.DownloadCount)
51+
.Select(pr => pr.Id)
52+
.Take(TyposquattingCheckListLength)
53+
.ToList();
54+
4555
var threshold = GetThreshold(uploadedPackageId);
4656
uploadedPackageId = TyposquattingStringNormalization.NormalizeString(uploadedPackageId);
4757

48-
var countCollision = 0;
49-
Parallel.ForEach(PackagesCheckList, (package, loopState) =>
58+
var collisionPackageIds = new ConcurrentBag<string>();
59+
Parallel.ForEach(packagesCheckList, (packageId, loopState) =>
5060
{
5161
// TODO: handle the package which is owned by an organization.
5262
// https://github.com/NuGet/Engineering/issues/1656
53-
if (package.Owners.Contains(uploadedPackageOwner.Username))
63+
string normalizedPackageId = TyposquattingStringNormalization.NormalizeString(packageId);
64+
if (TyposquattingDistanceCalculation.IsDistanceLessThanThreshold(uploadedPackageId, normalizedPackageId, threshold))
5465
{
55-
return;
66+
collisionPackageIds.Add(packageId);
5667
}
68+
});
5769

58-
if (TyposquattingDistanceCalculation.IsDistanceLessThanThreshold(uploadedPackageId, package.Id, threshold))
70+
foreach (var packageId in collisionPackageIds)
71+
{
72+
if (!_userTyposquattingService.CanUserTyposquat(packageId, uploadedPackageOwner.Username))
5973
{
60-
// Double check the owners list in the latest DB.
61-
if (_userTyposquattingService.CanUserTyposquat(package.Id, uploadedPackageOwner.Username))
62-
{
63-
return;
64-
}
65-
66-
Interlocked.Increment(ref countCollision);
67-
loopState.Stop();
74+
return true;
6875
}
69-
});
76+
}
7077

71-
return countCollision != 0;
78+
return false;
7279
}
7380

7481
private static int GetThreshold(string packageId)
7582
{
76-
foreach (var thresholdInfo in _thresholdsList)
83+
foreach (var thresholdInfo in ThresholdsList)
7784
{
7885
if (packageId.Length >= thresholdInfo.LowerBound && packageId.Length < thresholdInfo.UpperBound)
7986
{
@@ -85,16 +92,16 @@ private static int GetThreshold(string packageId)
8592
}
8693
}
8794

88-
public class PackageInfo
89-
{
90-
public string Id { get; set; }
91-
public HashSet<string> Owners { get; set; }
92-
}
93-
9495
public class ThresholdInfo
9596
{
96-
public int LowerBound { get; set; }
97-
public int UpperBound { get; set; }
98-
public int Threshold { get; set; }
97+
public int LowerBound { get; }
98+
public int UpperBound { get; }
99+
public int Threshold { get; }
100+
public ThresholdInfo(int lowerBound, int upperBound, int threshold)
101+
{
102+
LowerBound = lowerBound;
103+
UpperBound = upperBound;
104+
Threshold = threshold;
105+
}
99106
}
100107
}

‎tests/NuGetGallery.Facts/NuGetGallery.Facts.csproj‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -129,6 +129,7 @@
129129
<Compile Include="Security\RequirePackageMetadataCompliancePolicyFacts.cs" />
130130
<Compile Include="Services\ActionRequiringEntityPermissionsFacts.cs" />
131131
<Compile Include="Services\CertificatesConfigurationFacts.cs" />
132+
<Compile Include="Services\TyposquattingCheckServiceFacts.cs" />
132133
<Compile Include="Services\CloudDownloadCountServiceFacts.cs" />
133134
<Compile Include="Services\CertificateServiceFacts.cs" />
134135
<Compile Include="Services\CertificateValidatorFacts.cs" />
@@ -169,6 +170,7 @@
169170
<Compile Include="Services\TelemetryServiceFacts.cs" />
170171
<Compile Include="Services\TestableActionRequiringEntityPermissions.cs" />
171172
<Compile Include="Services\TestablePermissionsEntity.cs" />
173+
<Compile Include="Services\TyposquattingUserServiceFacts.cs" />
172174
<Compile Include="Services\ValidationServiceFacts.cs" />
173175
<Compile Include="Telemetry\ClientInformationTelemetryEnricherTests.cs" />
174176
<Compile Include="Telemetry\ClientTelemetryPIIProcessorTests.cs" />

0 commit comments

Comments
 (0)