Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Native ETag Support #675

Open
wants to merge 42 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from 24 commits
Commits
Show all changes
42 commits
Select commit Hold shift + click to select a range
ce7d671
init etag impl
hamdaankhalidmsft Sep 18, 2024
74619d0
WIP tests
hamdaankhalidmsft Sep 19, 2024
2dbab85
complete test coverage and bugfixes
hamdaankhalidmsft Sep 24, 2024
683ead2
add edge case handling for etag override set
hamdaankhalidmsft Sep 24, 2024
7078ec3
Add documentation
hamdaankhalidmsft Sep 25, 2024
0e766f5
fix etag clearing
hamdaankhalidmsft Sep 25, 2024
1f7226d
Format
hamdaankhalidmsft Sep 25, 2024
c7cb0b5
fix build warnings
hamdaankhalidmsft Sep 25, 2024
3631d20
Update test
hamdaankhalidmsft Sep 25, 2024
a079c0f
Update tests for badrish reqs
hamdaankhalidmsft Sep 30, 2024
41bdb41
Update mainstore ops fixing rename bug and remove unnecessary GetForE…
hamdaankhalidmsft Oct 1, 2024
f6e2fb3
add read nil test
hamdaankhalidmsft Oct 1, 2024
e89477d
fmt
hamdaankhalidmsft Oct 1, 2024
ae5dd43
Update docs and add more tests for edgecase
hamdaankhalidmsft Oct 1, 2024
98e4ec3
fix renamenx and etag compat
hamdaankhalidmsft Oct 1, 2024
da48417
Add more etag edge case tests
hamdaankhalidmsft Oct 1, 2024
d581562
remove shifting in network buffer in setifmatch
hamdaankhalidmsft Oct 2, 2024
355bef9
Fix named parameters for readability
hamdaankhalidmsft Oct 2, 2024
8b18e43
use constants for etag size
hamdaankhalidmsft Oct 2, 2024
0c340c8
update read nil to be a faster
hamdaankhalidmsft Oct 2, 2024
141c870
format
hamdaankhalidmsft Oct 2, 2024
a7756c8
pr feedback
hamdaankhalidmsft Oct 8, 2024
b6ffd7a
Merge branch 'main' into hkhalid/etag-impl
hamdaankhalidmsft Oct 8, 2024
8014242
remove redundant ternary
hamdaankhalidmsft Oct 8, 2024
abc6d8f
update tests for transaction
hamdaankhalidmsft Oct 9, 2024
802ff73
Add test to make sure etag data works with recovery scenarios
hamdaankhalidmsft Oct 10, 2024
d11a1e8
Merge branch 'main' into hkhalid/etag-impl
hamdaankhalidmsft Oct 11, 2024
8bda79a
WIP
hamdaankhalidmsft Oct 24, 2024
1309833
Merge branch 'main' into hkhalid/etag-impl
hamdaankhalidmsft Nov 15, 2024
8f09c7e
WIP
hamdaankhalidmsft Nov 16, 2024
ca4caaf
Merge branch 'main' into hkhalid/etag-impl
hamdaankhalidmsft Dec 12, 2024
8861f75
unit tests working
hamdaankhalidmsft Dec 16, 2024
d7beeb6
Merge branch 'main' into hkhalid/etag-impl
hamdaankhalidmsft Dec 16, 2024
c832f71
fmt
hamdaankhalidmsft Dec 16, 2024
e7a90c8
Fix boundary condition for bitmanager
hamdaankhalidmsft Dec 16, 2024
56e305c
fix rename edgecase
hamdaankhalidmsft Dec 16, 2024
fd062d8
Merge branch 'main' into hkhalid/etag-impl
hamdaankhalid Dec 16, 2024
ddf63ca
Add custom txn, proc, and cmd handling
hamdaankhalidmsft Dec 17, 2024
2e2c6b1
Merge branch 'hkhalid/etag-impl' of https://github.com/hamdaankhalid/…
hamdaankhalidmsft Dec 17, 2024
4e327b4
update docs
hamdaankhalidmsft Dec 17, 2024
1d05e69
Finish website and nits
hamdaankhalidmsft Dec 17, 2024
c3b2dbd
nit
hamdaankhalidmsft Dec 17, 2024
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
42 changes: 42 additions & 0 deletions libs/common/RespReadUtils.cs
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@

using System;
using System.Buffers.Text;
using System.Diagnostics;
using System.Runtime.CompilerServices;
using System.Runtime.InteropServices;
using System.Text;
Expand Down Expand Up @@ -377,6 +378,47 @@ public static bool TryRead64Int(out long number, ref byte* ptr, byte* end, out b
return true;
}

/// <summary>
/// Given a buffer check if the value is nil ($-1\r\n)
/// If the value is nil it advances the buffer forward
/// </summary>
/// <param name="ptr">The starting position in the RESP string. Will be advanced if parsing is successful.</param>
/// <param name="end">The current end of the RESP string.</param>
/// <param name="unexpectedToken"></param>
/// <returns>True if value is nil on the buffer, false if the value on buffer is not nil</returns>
public static bool ReadNil(ref byte* ptr, byte* end, out byte? unexpectedToken)
{
unexpectedToken = null;
if (end - ptr < 5)
{
return false;
}

ReadOnlySpan<byte> expectedNilRepr = "$-1\r\n"u8;

if (*(uint*)ptr != MemoryMarshal.Read<uint>(expectedNilRepr.Slice(0, 4)) || *(ptr + 4) != expectedNilRepr[4])
{
ReadOnlySpan<byte> ptrNext5Bytes = new ReadOnlySpan<byte>(ptr, 5);
for (int i = 0; i < 5; i++)
{
// first place where the sequence differs we have found the unexpected token
if (expectedNilRepr[i] != ptrNext5Bytes[i])
{
// move the pointer to the unexpected token
ptr += i;
unexpectedToken = ptrNext5Bytes[i];
return false;
}
}
// If the sequence is not equal we shouldn't even reach this because atleast one byte should have mismatched
Debug.Assert(false);
return false;
}

ptr += 5;
return true;
}

/// <summary>
/// Tries to read a RESP array length header from the given ASCII-encoded RESP string
/// and, if successful, moves the given ptr to the end of the length header.
Expand Down
56 changes: 56 additions & 0 deletions libs/resources/RespCommandsInfo.json
Original file line number Diff line number Diff line change
Expand Up @@ -1351,6 +1351,20 @@
}
]
},
{
"Command": "GETIFNOTMATCH",
"Name": "GETIFNOTMATCH",
"IsInternal": false,
"Arity": 2,
"Flags": "NONE",
"FirstKey": 1,
"LastKey": 1,
"Step": 1,
"AclCategories": "Fast, String, Read",
"Tips": null,
"KeySpecifications": null,
"SubCommands": null
},
{
"Command": "GETRANGE",
"Name": "GETRANGE",
Expand All @@ -1376,6 +1390,20 @@
}
]
},
{
"Command": "GETWITHETAG",
"Name": "GETWITHETAG",
"IsInternal": false,
"Arity": 1,
"Flags": "NONE",
"FirstKey": 1,
"LastKey": 1,
"Step": 1,
"AclCategories": "Fast, String, Read",
"Tips": null,
"KeySpecifications": null,
"SubCommands": null
},
{
"Command": "HDEL",
"Name": "HDEL",
Expand Down Expand Up @@ -3212,6 +3240,20 @@
}
]
},
{
"Command": "SETIFMATCH",
"Name": "SETIFMATCH",
"IsInternal": false,
"Arity": 3,
"Flags": "NONE",
"FirstKey": 1,
"LastKey": 1,
"Step": 1,
"AclCategories": "Fast, String, Write",
"Tips": null,
"KeySpecifications": null,
"SubCommands": null
},
{
"Command": "SETRANGE",
"Name": "SETRANGE",
Expand All @@ -3237,6 +3279,20 @@
}
]
},
{
"Command": "SETWITHETAG",
"Name": "SETWITHETAG",
"IsInternal": false,
"Arity": 2,
"Flags": "NONE",
"FirstKey": 1,
"LastKey": 1,
"Step": 1,
"AclCategories": "Fast, String, Write",
"Tips": null,
"KeySpecifications": null,
"SubCommands": null
},
{
"Command": "SINTER",
"Name": "SINTER",
Expand Down
4 changes: 2 additions & 2 deletions libs/server/API/GarnetApi.cs
Original file line number Diff line number Diff line change
Expand Up @@ -115,8 +115,8 @@ public GarnetStatus SET_Conditional(ref SpanByte key, ref SpanByte input)
=> storageSession.SET_Conditional(ref key, ref input, ref context);

/// <inheritdoc />
public GarnetStatus SET_Conditional(ref SpanByte key, ref SpanByte input, ref SpanByteAndMemory output)
=> storageSession.SET_Conditional(ref key, ref input, ref output, ref context);
public GarnetStatus SET_Conditional(ref SpanByte key, ref SpanByte input, ref SpanByteAndMemory output, RespCommand cmd)
=> storageSession.SET_Conditional(ref key, ref input, ref output, ref context, cmd);

/// <inheritdoc />
public GarnetStatus SET(ArgSlice key, Memory<byte> value)
Expand Down
6 changes: 5 additions & 1 deletion libs/server/API/GarnetStatus.cs
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,10 @@ public enum GarnetStatus : byte
/// <summary>
/// Wrong type
/// </summary>
WRONGTYPE
WRONGTYPE,
/// <summary>
/// ETAG mismatch result for an etag based command
/// </summary>
ETAGMISMATCH,
}
}
2 changes: 1 addition & 1 deletion libs/server/API/IGarnetApi.cs
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ public interface IGarnetApi : IGarnetReadApi, IGarnetAdvancedApi
/// <summary>
/// SET Conditional
/// </summary>
GarnetStatus SET_Conditional(ref SpanByte key, ref SpanByte input, ref SpanByteAndMemory output);
GarnetStatus SET_Conditional(ref SpanByte key, ref SpanByte input, ref SpanByteAndMemory output, RespCommand cmd);

/// <summary>
/// SET
Expand Down
11 changes: 11 additions & 0 deletions libs/server/Constants.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT license.


namespace Garnet.server
{
internal static class Constants
{
public const int EtagSize = sizeof(long);
}
}
18 changes: 18 additions & 0 deletions libs/server/InputHeader.cs
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,13 @@ enum RespInputFlags : byte
/// Expired
/// </summary>
Expired = 128,

/// <summary>
/// Flag indicating if a SET operation should retain the etag of the previous value if it exists.
/// This is used for conditional setting.
/// </summary>
RetainEtag = 129,

}

/// <summary>
Expand Down Expand Up @@ -100,6 +107,17 @@ internal ListOperation ListOp
/// </summary>
internal unsafe void SetSetGetFlag() => flags |= RespInputFlags.SetGet;

/// <summary>
/// Set "RetainEtag" flag, used to update the old etag of a key after conditionally setting it
/// </summary>
internal unsafe void SetRetainEtagFlag() => flags |= RespInputFlags.RetainEtag;

/// <summary>
/// Check if the RetainEtagFlag is set
/// </summary>
/// <returns></returns>
internal unsafe bool CheckRetainEtagFlag() => (flags & RespInputFlags.RetainEtag) != 0;

/// <summary>
/// Check if record is expired, either deterministically during log replay,
/// or based on current time in normal operation.
Expand Down
Loading