Skip to content

Commit aeaf0c2

Browse files
zhhyuzhhyu
authored
[ST] 4. Add minIntervalBeforeToReadUpdate to cursor.json (#10733)
* Add * update
1 parent b117683 commit aeaf0c2

8 files changed

Lines changed: 92 additions & 24 deletions

File tree

‎src/Catalog/DurableCursorWithUpdates.cs‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,9 +19,10 @@ public class DurableCursorWithUpdates : DurableCursor
1919

2020
private readonly int _maxNumberOfUpdatesToKeep;
2121
private readonly TimeSpan _minIntervalBetweenTwoUpdates;
22+
private readonly TimeSpan _minIntervalBeforeToReadUpdate;
2223

2324
public DurableCursorWithUpdates(Uri address, Persistence.Storage storage, DateTime defaultValue, ILogger logger,
24-
int maxNumberOfUpdatesToKeep, TimeSpan minIntervalBetweenTwoUpdates)
25+
int maxNumberOfUpdatesToKeep, TimeSpan minIntervalBetweenTwoUpdates, TimeSpan minIntervalBeforeToReadUpdate)
2526
: base(address, storage, defaultValue)
2627
{
2728
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
@@ -36,8 +37,14 @@ public DurableCursorWithUpdates(Uri address, Persistence.Storage storage, DateTi
3637
throw new ArgumentOutOfRangeException(nameof(minIntervalBetweenTwoUpdates), $"{nameof(minIntervalBetweenTwoUpdates)} must be equal or larger than 0.");
3738
}
3839

40+
if (minIntervalBeforeToReadUpdate < TimeSpan.Zero)
41+
{
42+
throw new ArgumentOutOfRangeException(nameof(minIntervalBeforeToReadUpdate), $"{nameof(minIntervalBeforeToReadUpdate)} must be equal or larger than 0.");
43+
}
44+
3945
_maxNumberOfUpdatesToKeep = maxNumberOfUpdatesToKeep;
4046
_minIntervalBetweenTwoUpdates = minIntervalBetweenTwoUpdates;
47+
_minIntervalBeforeToReadUpdate = minIntervalBeforeToReadUpdate;
4148
}
4249

4350
public override async Task SaveAsync(CancellationToken cancellationToken)
@@ -54,6 +61,7 @@ public override async Task SaveAsync(CancellationToken cancellationToken)
5461
}
5562

5663
cursorValueWithUpdates.Value = Value.ToString("O");
64+
cursorValueWithUpdates.MinIntervalBeforeToReadUpdate = _minIntervalBeforeToReadUpdate;
5765
if (storageContent != null)
5866
{
5967
cursorValueWithUpdates.Updates = GetUpdates(cursorValueWithUpdates, storageContent.StorageDateTimeInUtc);

‎src/Catalog/Helpers/CursorValueWithUpdates.cs‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,12 @@ public class CursorValueWithUpdates
1616

1717
[JsonProperty("value")]
1818
public string Value { get; set; }
19+
20+
// This is for the cursor reader to determine which update (in the list of updates) of the cursor value to read.
21+
// The timestamp of the update to read should be at least before the current timestamp minus this interval.
22+
[JsonProperty("minIntervalBeforeToReadUpdate")]
23+
public TimeSpan MinIntervalBeforeToReadUpdate { get; set; }
24+
1925
[JsonProperty("updates")]
2026
public IList<CursorValueUpdate> Updates { get; set; } = new List<CursorValueUpdate>();
2127
}
@@ -30,6 +36,7 @@ public CursorValueUpdate(DateTime timeStamp, string value)
3036

3137
[JsonProperty("timeStamp")]
3238
public DateTime TimeStamp { get; set; }
39+
3340
[JsonProperty("value")]
3441
public string Value { get; set; }
3542
}

‎src/Catalog/HttpReadCursorWithUpdates.cs‎

Lines changed: 8 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -13,14 +13,11 @@ namespace NuGet.Services.Metadata.Catalog
1313
public class HttpReadCursorWithUpdates : HttpReadCursor
1414
{
1515
private readonly ILogger _logger;
16-
private readonly TimeSpan _minIntervalBeforeToReadCursorUpdateValue;
1716

18-
public HttpReadCursorWithUpdates(TimeSpan minIntervalBeforeToReadCursorUpdateValue, Uri address, ILogger logger,
19-
Func<HttpMessageHandler> handlerFunc = null)
17+
public HttpReadCursorWithUpdates(Uri address, ILogger logger, Func<HttpMessageHandler> handlerFunc = null)
2018
: base(address, handlerFunc)
2119
{
2220
_logger = logger ?? throw new ArgumentNullException(nameof(logger));
23-
_minIntervalBeforeToReadCursorUpdateValue = minIntervalBeforeToReadCursorUpdateValue;
2421
}
2522

2623
public override async Task<string> GetValueInJsonAsync(HttpResponseMessage response)
@@ -45,20 +42,22 @@ private async Task<CursorValueUpdate> GetUpdate(HttpResponseMessage response, Da
4542
}
4643

4744
var content = await response.Content.ReadAsStringAsync();
45+
4846
var cursorValueWithUpdates = JsonConvert.DeserializeObject<CursorValueWithUpdates>(content, CursorValueWithUpdates.SerializerSettings);
47+
var minIntervalBeforeToReadUpdate = cursorValueWithUpdates.MinIntervalBeforeToReadUpdate;
4948
var updates = cursorValueWithUpdates.Updates.OrderByDescending(u => u.TimeStamp).ToList();
5049

5150
foreach (var update in updates)
5251
{
53-
if (update.TimeStamp <= storageDateTimeInUtc.Value - _minIntervalBeforeToReadCursorUpdateValue)
52+
if (update.TimeStamp <= storageDateTimeInUtc.Value - minIntervalBeforeToReadUpdate)
5453
{
5554
_logger.LogInformation("Read the cursor update with timeStamp: {TimeStamp} and value: {UpdateValue}, at {Address}. " +
56-
"(Storage DateTime: {StorageDateTime}, MinIntervalBeforeToReadCursorUpdateValue: {MinIntervalBeforeToReadCursorUpdateValue})",
55+
"(Storage DateTime: {StorageDateTime}, MinIntervalBeforeToReadUpdate: {MinIntervalBeforeToReadUpdate})",
5756
update.TimeStamp.ToString(CursorValueWithUpdates.SerializerSettings.DateFormatString),
5857
update.Value,
5958
_address.AbsoluteUri,
6059
storageDateTimeInUtc.Value.ToString(CursorValueWithUpdates.SerializerSettings.DateFormatString),
61-
_minIntervalBeforeToReadCursorUpdateValue);
60+
minIntervalBeforeToReadUpdate);
6261

6362
return update;
6463
}
@@ -67,11 +66,11 @@ private async Task<CursorValueUpdate> GetUpdate(HttpResponseMessage response, Da
6766
if (updates.Count > 0)
6867
{
6968
_logger.LogWarning("Unable to find the cursor update and the oldest cursor update has timeStamp: {TimeStamp}, at {Address}. " +
70-
"(Storage DateTime: {StorageDateTime}, MinIntervalBeforeToReadCursorUpdateValue: {MinIntervalBeforeToReadCursorUpdateValue})",
69+
"(Storage DateTime: {StorageDateTime}, MinIntervalBeforeToReadUpdate: {MinIntervalBeforeToReadUpdate})",
7170
updates.Last().TimeStamp.ToString(CursorValueWithUpdates.SerializerSettings.DateFormatString),
7271
_address.AbsoluteUri,
7372
storageDateTimeInUtc.Value.ToString(CursorValueWithUpdates.SerializerSettings.DateFormatString),
74-
_minIntervalBeforeToReadCursorUpdateValue);
73+
minIntervalBeforeToReadUpdate);
7574
}
7675
else
7776
{

‎src/Ng/Jobs/Catalog2DnxJob.cs‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -102,7 +102,8 @@ protected override void Init(IDictionary<string, string> arguments, Cancellation
102102

103103
var storage = storageFactory.Create();
104104
_front = new DurableCursorWithUpdates(storage.ResolveUri("cursor.json"), storage, MemoryCursor.MinValue, Logger,
105-
DnxConstants.MaxNumberOfUpdatesToKeepOfFrontCursor, DnxConstants.MinIntervalBetweenTwoUpdatesOfFrontCursor);
105+
DnxConstants.MaxNumberOfUpdatesToKeepOfFrontCursor, DnxConstants.MinIntervalBetweenTwoUpdatesOfFrontCursor,
106+
minIntervalBeforeToReadUpdate: DnxConstants.CacheDurationOfPackageVersionIndex + TimeSpan.FromSeconds(1));
106107
_back = MemoryCursor.CreateMax();
107108

108109
_destination = storageFactory.BaseAddress;

‎src/NuGet.Jobs.Catalog2Registration/Catalog2RegistrationCommand.cs‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -59,9 +59,7 @@ private async Task ExecuteAsync(CancellationToken token)
5959
_logger.LogInformation("Depending on cursors: {DependencyCursorUrls}", _options.Value.DependencyCursorUrls);
6060
backCursor = new AggregateCursor(_options
6161
.Value
62-
.DependencyCursorUrls.Select(r => new HttpReadCursorWithUpdates(
63-
minIntervalBeforeToReadCursorUpdateValue: DnxConstants.CacheDurationOfPackageVersionIndex + TimeSpan.FromSeconds(1),
64-
new Uri(r), _logger, _handlerFunc)));
62+
.DependencyCursorUrls.Select(r => new HttpReadCursorWithUpdates(new Uri(r), _logger, _handlerFunc)));
6563
}
6664
else
6765
{

‎tests/CatalogTests/DurableCursorWithUpdatesTests.cs‎

Lines changed: 48 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -31,15 +31,15 @@ public DurableCursorWithUpdatesTests()
3131
.Callback<Uri, StorageContent, CancellationToken>((u, sc, c) => _savedStorageContent = sc);
3232

3333
_cursor = new DurableCursorWithUpdates(_address, _storage.Object, _defaultValue, Mock.Of<ILogger>(),
34-
maxNumberOfUpdatesToKeep: 2, minIntervalBetweenTwoUpdates: TimeSpan.FromSeconds(60));
34+
maxNumberOfUpdatesToKeep: 2, minIntervalBetweenTwoUpdates: TimeSpan.FromSeconds(60), minIntervalBeforeToReadUpdate: TimeSpan.FromSeconds(1));
3535
_cursor.Value = new DateTime(2026, 1, 1, 1, 0, 0, DateTimeKind.Unspecified);
3636
}
3737

3838
[Fact]
3939
public void ThrowArgumentOutOfRangeException_maxNumberOfUpdatesToKeep()
4040
{
4141
var exception = Assert.Throws<ArgumentOutOfRangeException>(() => new DurableCursorWithUpdates(_address, It.IsAny<Storage>(), _defaultValue, Mock.Of<ILogger>(),
42-
maxNumberOfUpdatesToKeep: -1, minIntervalBetweenTwoUpdates: TimeSpan.FromSeconds(60)));
42+
maxNumberOfUpdatesToKeep: -1, minIntervalBetweenTwoUpdates: TimeSpan.FromSeconds(60), minIntervalBeforeToReadUpdate: TimeSpan.FromSeconds(1)));
4343

4444
Assert.Equal("maxNumberOfUpdatesToKeep", exception.ParamName);
4545
Assert.Equal("maxNumberOfUpdatesToKeep must be equal or larger than 0.\r\nParameter name: maxNumberOfUpdatesToKeep", exception.Message);
@@ -49,12 +49,22 @@ public void ThrowArgumentOutOfRangeException_maxNumberOfUpdatesToKeep()
4949
public void ThrowArgumentOutOfRangeException_minIntervalBetweenTwoUpdates()
5050
{
5151
var exception = Assert.Throws<ArgumentOutOfRangeException>(() => new DurableCursorWithUpdates(_address, It.IsAny<Storage>(), _defaultValue, Mock.Of<ILogger>(),
52-
maxNumberOfUpdatesToKeep: 2, minIntervalBetweenTwoUpdates: TimeSpan.FromSeconds(-1)));
52+
maxNumberOfUpdatesToKeep: 2, minIntervalBetweenTwoUpdates: TimeSpan.FromSeconds(-1), minIntervalBeforeToReadUpdate: TimeSpan.FromSeconds(1)));
5353

5454
Assert.Equal("minIntervalBetweenTwoUpdates", exception.ParamName);
5555
Assert.Equal("minIntervalBetweenTwoUpdates must be equal or larger than 0.\r\nParameter name: minIntervalBetweenTwoUpdates", exception.Message);
5656
}
5757

58+
[Fact]
59+
public void ThrowArgumentOutOfRangeException_minIntervalBeforeToReadUpdate()
60+
{
61+
var exception = Assert.Throws<ArgumentOutOfRangeException>(() => new DurableCursorWithUpdates(_address, It.IsAny<Storage>(), _defaultValue, Mock.Of<ILogger>(),
62+
maxNumberOfUpdatesToKeep: 2, minIntervalBetweenTwoUpdates: TimeSpan.FromSeconds(60), minIntervalBeforeToReadUpdate: TimeSpan.FromSeconds(-1)));
63+
64+
Assert.Equal("minIntervalBeforeToReadUpdate", exception.ParamName);
65+
Assert.Equal("minIntervalBeforeToReadUpdate must be equal or larger than 0.\r\nParameter name: minIntervalBeforeToReadUpdate", exception.Message);
66+
}
67+
5868
[Fact]
5969
public async Task SaveAsync_WithDoesNotExistInStorage()
6070
{
@@ -63,15 +73,18 @@ public async Task SaveAsync_WithDoesNotExistInStorage()
6373

6474
Assert.NotNull(_savedStorageContent);
6575
Assert.IsType<StringStorageContent>(_savedStorageContent);
66-
Assert.Equal("{\"value\":\"2026-01-01T01:00:00.0000000\",\"updates\":[]}", (_savedStorageContent as StringStorageContent).Content);
76+
Assert.Equal("{\"value\":\"2026-01-01T01:00:00.0000000\",\"minIntervalBeforeToReadUpdate\":\"00:00:01\",\"updates\":[]}",
77+
(_savedStorageContent as StringStorageContent).Content);
6778

6879
_storage.Verify(s => s.LoadStringStorageContentAsync(It.IsAny<Uri>(), It.IsAny<CancellationToken>()), Times.Once);
6980
_storage.Verify(s => s.SaveAsync(It.IsAny<Uri>(), It.IsAny<StorageContent>(), It.IsAny<CancellationToken>()), Times.Once);
7081
}
7182

7283
[Theory]
7384
[InlineData("{\"value\":\"2026-01-01T00:59:00.0000000\"}")]
85+
[InlineData("{\"value\":\"2026-01-01T00:59:00.0000000\",\"minIntervalBeforeToReadUpdate\":\"00:00:01\"}")]
7486
[InlineData("{\"value\":\"2026-01-01T00:59:00.0000000\",\"updates\":[]}")]
87+
[InlineData("{\"value\":\"2026-01-01T00:59:00.0000000\",\"minIntervalBeforeToReadUpdate\":\"00:00:01\",\"updates\":[]}")]
7588
public async Task SaveAsync_WithEmptyUpdatesInStorage(string content)
7689
{
7790
_storageContent = new StringStorageContent(content, storageDateTimeInUtc: new DateTime(2026, 1, 1, 1, 0, 30, DateTimeKind.Utc));
@@ -80,6 +93,25 @@ public async Task SaveAsync_WithEmptyUpdatesInStorage(string content)
8093
Assert.NotNull(_savedStorageContent);
8194
Assert.IsType<StringStorageContent>(_savedStorageContent);
8295
Assert.Equal("{\"value\":\"2026-01-01T01:00:00.0000000\"," +
96+
"\"minIntervalBeforeToReadUpdate\":\"00:00:01\"," +
97+
"\"updates\":[{\"timeStamp\":\"2026-01-01T01:00:30.0000000Z\",\"value\":\"2026-01-01T01:00:00.0000000\"}]}",
98+
(_savedStorageContent as StringStorageContent).Content);
99+
100+
_storage.Verify(s => s.LoadStringStorageContentAsync(It.IsAny<Uri>(), It.IsAny<CancellationToken>()), Times.Once);
101+
_storage.Verify(s => s.SaveAsync(It.IsAny<Uri>(), It.IsAny<StorageContent>(), It.IsAny<CancellationToken>()), Times.Once);
102+
}
103+
104+
[Fact]
105+
public async Task SaveAsync_WithDifferentMinInterval()
106+
{
107+
_storageContent = new StringStorageContent("{\"value\":\"2026-01-01T00:59:00.0000000\",\"minIntervalBeforeToReadUpdate\":\"01:01:01\"}",
108+
storageDateTimeInUtc: new DateTime(2026, 1, 1, 1, 0, 30, DateTimeKind.Utc));
109+
await _cursor.SaveAsync(CancellationToken.None);
110+
111+
Assert.NotNull(_savedStorageContent);
112+
Assert.IsType<StringStorageContent>(_savedStorageContent);
113+
Assert.Equal("{\"value\":\"2026-01-01T01:00:00.0000000\"," +
114+
"\"minIntervalBeforeToReadUpdate\":\"00:00:01\"," +
83115
"\"updates\":[{\"timeStamp\":\"2026-01-01T01:00:30.0000000Z\",\"value\":\"2026-01-01T01:00:00.0000000\"}]}",
84116
(_savedStorageContent as StringStorageContent).Content);
85117

@@ -89,41 +121,53 @@ public async Task SaveAsync_WithEmptyUpdatesInStorage(string content)
89121

90122
[Theory]
91123
[InlineData("{\"value\":\"2026-01-01T00:59:00.0000000\"," +
124+
"\"minIntervalBeforeToReadUpdate\":\"00:00:01\"," +
92125
"\"updates\":[{\"timeStamp\":\"2026-01-01T00:59:30.0000000Z\",\"value\":\"2026-01-01T00:59:00.0000000\"}]}",
93126
"{\"value\":\"2026-01-01T01:00:00.0000000\"," +
127+
"\"minIntervalBeforeToReadUpdate\":\"00:00:01\"," +
94128
"\"updates\":[{\"timeStamp\":\"2026-01-01T01:00:30.0000000Z\",\"value\":\"2026-01-01T01:00:00.0000000\"}," +
95129
"{\"timeStamp\":\"2026-01-01T00:59:30.0000000Z\",\"value\":\"2026-01-01T00:59:00.0000000\"}]}")]
96130
[InlineData("{\"value\":\"2026-01-01T00:59:00.0000000\"," +
131+
"\"minIntervalBeforeToReadUpdate\":\"00:00:01\"," +
97132
"\"updates\":[{\"timeStamp\":\"2026-01-01T00:59:30.0000000Z\",\"value\":\"2026-01-01T00:59:00.0000000\"}," +
98133
"{\"timeStamp\":\"2026-01-01T00:58:30.0000000Z\",\"value\":\"2026-01-01T00:58:00.0000000\"}]}",
99134
"{\"value\":\"2026-01-01T01:00:00.0000000\"," +
135+
"\"minIntervalBeforeToReadUpdate\":\"00:00:01\"," +
100136
"\"updates\":[{\"timeStamp\":\"2026-01-01T01:00:30.0000000Z\",\"value\":\"2026-01-01T01:00:00.0000000\"}," +
101137
"{\"timeStamp\":\"2026-01-01T00:59:30.0000000Z\",\"value\":\"2026-01-01T00:59:00.0000000\"}]}")]
102138
[InlineData("{\"value\":\"2026-01-01T00:59:00.0000000\"," +
139+
"\"minIntervalBeforeToReadUpdate\":\"00:00:01\"," +
103140
"\"updates\":[{\"timeStamp\":\"2026-01-01T00:59:30.0000000Z\",\"value\":\"2026-01-01T00:59:00.0000000\"}," +
104141
"{\"timeStamp\":\"2026-01-01T00:58:30.0000000Z\",\"value\":\"2026-01-01T00:58:00.0000000\"}," +
105142
"{\"timeStamp\":\"2026-01-01T00:57:30.0000000Z\",\"value\":\"2026-01-01T00:57:00.0000000\"}]}",
106143
"{\"value\":\"2026-01-01T01:00:00.0000000\"," +
144+
"\"minIntervalBeforeToReadUpdate\":\"00:00:01\"," +
107145
"\"updates\":[{\"timeStamp\":\"2026-01-01T01:00:30.0000000Z\",\"value\":\"2026-01-01T01:00:00.0000000\"}," +
108146
"{\"timeStamp\":\"2026-01-01T00:59:30.0000000Z\",\"value\":\"2026-01-01T00:59:00.0000000\"}]}")]
109147
[InlineData("{\"value\":\"2026-01-01T00:59:00.0000000\"," +
148+
"\"minIntervalBeforeToReadUpdate\":\"00:00:01\"," +
110149
"\"updates\":[{\"timeStamp\":\"2026-01-01T00:58:30.0000000Z\",\"value\":\"2026-01-01T00:58:00.0000000\"}," +
111150
"{\"timeStamp\":\"2026-01-01T00:57:30.0000000Z\",\"value\":\"2026-01-01T00:57:00.0000000\"}," +
112151
"{\"timeStamp\":\"2026-01-01T00:59:30.0000000Z\",\"value\":\"2026-01-01T00:59:00.0000000\"}]}",
113152
"{\"value\":\"2026-01-01T01:00:00.0000000\"," +
153+
"\"minIntervalBeforeToReadUpdate\":\"00:00:01\"," +
114154
"\"updates\":[{\"timeStamp\":\"2026-01-01T01:00:30.0000000Z\",\"value\":\"2026-01-01T01:00:00.0000000\"}," +
115155
"{\"timeStamp\":\"2026-01-01T00:59:30.0000000Z\",\"value\":\"2026-01-01T00:59:00.0000000\"}]}")]
116156
[InlineData("{\"value\":\"2026-01-01T00:59:00.0000000\"," +
157+
"\"minIntervalBeforeToReadUpdate\":\"00:00:01\"," +
117158
"\"updates\":[{\"timeStamp\":\"2026-01-01T00:59:31.0000000Z\",\"value\":\"2026-01-01T00:59:00.0000000\"}," +
118159
"{\"timeStamp\":\"2026-01-01T00:58:30.0000000Z\",\"value\":\"2026-01-01T00:58:00.0000000\"}]}",
119160
"{\"value\":\"2026-01-01T01:00:00.0000000\"," +
161+
"\"minIntervalBeforeToReadUpdate\":\"00:00:01\"," +
120162
"\"updates\":[{\"timeStamp\":\"2026-01-01T00:59:31.0000000Z\",\"value\":\"2026-01-01T00:59:00.0000000\"}," +
121163
"{\"timeStamp\":\"2026-01-01T00:58:30.0000000Z\",\"value\":\"2026-01-01T00:58:00.0000000\"}]}")]
122164
[InlineData("{\"value\":\"2026-01-01T00:59:00.0000000\"," +
165+
"\"minIntervalBeforeToReadUpdate\":\"00:00:01\"," +
123166
"\"updates\":[{\"timeStamp\":\"2026-01-01T00:59:31.0000000Z\",\"value\":\"2026-01-01T00:59:00.0000000\"}," +
124167
"{\"timeStamp\":\"2026-01-01T00:58:30.0000000Z\",\"value\":\"2026-01-01T00:58:00.0000000\"}," +
125168
"{\"timeStamp\":\"2026-01-01T00:57:30.0000000Z\",\"value\":\"2026-01-01T00:57:00.0000000\"}]}",
126169
"{\"value\":\"2026-01-01T01:00:00.0000000\"," +
170+
"\"minIntervalBeforeToReadUpdate\":\"00:00:01\"," +
127171
"\"updates\":[{\"timeStamp\":\"2026-01-01T00:59:31.0000000Z\",\"value\":\"2026-01-01T00:59:00.0000000\"}," +
128172
"{\"timeStamp\":\"2026-01-01T00:58:30.0000000Z\",\"value\":\"2026-01-01T00:58:00.0000000\"}]}")]
129173
public async Task SaveAsync(string content, string expectedContentAfterSave)

0 commit comments

Comments
 (0)