Skip to content

Commit 6ed8309

Browse files
committed
Update based on comments from @CGodiksen
1 parent d1d7685 commit 6ed8309

15 files changed

Lines changed: 105 additions & 68 deletions

File tree

Cargo.lock

Lines changed: 18 additions & 19 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

Cargo.toml

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,16 @@ http = "1.4.0"
3636
http-body-util = "0.1.3"
3737
itertools = "0.14.0"
3838
log = "0.4.29"
39+
modelardb_auth = { path = "crates/modelardb_auth"}
40+
modelardb_bulkloader = { path = "crates/modelardb_bulkloader"}
41+
modelardb_client = { path = "crates/modelardb_client"}
42+
modelardb_compression = { path = "crates/modelardb_compression"}
43+
modelardb_embedded = { path = "crates/modelardb_embedded"}
44+
modelardb_macros = { path = "crates/modelardb_macros"}
45+
modelardb_server = { path = "crates/modelardb_server"}
46+
modelardb_storage = { path = "crates/modelardb_storage"}
47+
modelardb_test = { path = "crates/modelardb_test"}
48+
modelardb_types = { path = "crates/modelardb_types"}
3949
object_store = "0.13.2"
4050
proptest = "1.11.0"
4151
prost = "0.14.3"

crates/modelardb_bulkloader/Cargo.toml

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -30,9 +30,9 @@ datafusion.workspace = true
3030
delta_kernel.workspace = true
3131
deltalake.workspace = true
3232
futures.workspace = true
33-
modelardb_compression = { path = "../modelardb_compression" }
34-
modelardb_embedded = { path = "../modelardb_embedded" }
35-
modelardb_storage = { path = "../modelardb_storage" }
36-
modelardb_types = { path = "../modelardb_types" }
33+
modelardb_compression.workspace = true
34+
modelardb_embedded.workspace = true
35+
modelardb_storage.workspace = true
36+
modelardb_types.workspace = true
3737
sysinfo.workspace = true
3838
tokio.workspace = true

crates/modelardb_client/Cargo.toml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,7 @@ arrow = { workspace = true, features = ["prettyprint"] }
2929
bytes.workspace = true
3030
clap.workspace = true
3131
dirs.workspace = true
32-
modelardb_auth = { path = "../modelardb_auth" }
32+
modelardb_auth.workspace = true
3333
rustyline.workspace = true
3434
tokio.workspace = true
3535
tonic.workspace = true

crates/modelardb_compression/Cargo.toml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,8 +21,8 @@ authors = ["Soeren Kejser Jensen <[email protected]>"]
2121

2222
[dependencies]
2323
arrow.workspace = true
24-
modelardb_types = { path = "../modelardb_types" }
24+
modelardb_types.workspace = true
2525

2626
[dev-dependencies]
27-
modelardb_test = { path = "../modelardb_test" }
27+
modelardb_test.workspace = true
2828
proptest.workspace = true

crates/modelardb_embedded/Cargo.toml

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -31,10 +31,10 @@ async-trait.workspace = true
3131
datafusion.workspace = true
3232
deltalake = { workspace = true, features = ["datafusion", "s3"] }
3333
futures.workspace = true
34-
modelardb_auth = { path = "../modelardb_auth" }
35-
modelardb_compression = { path = "../modelardb_compression" }
36-
modelardb_storage = { path = "../modelardb_storage" }
37-
modelardb_types = { path = "../modelardb_types" }
34+
modelardb_auth.workspace = true
35+
modelardb_compression.workspace = true
36+
modelardb_storage.workspace = true
37+
modelardb_types.workspace = true
3838
tokio.workspace = true
3939
tonic.workspace = true
4040

crates/modelardb_macros/Cargo.toml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,4 +24,4 @@ proc-macro = true
2424

2525
[dependencies]
2626
itertools.workspace = true
27-
modelardb_test = { path = "../modelardb_test" }
27+
modelardb_test.workspace = true

crates/modelardb_macros/src/lib.rs

Lines changed: 47 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,7 @@ impl Display for ParameterType {
4747
}
4848
}
4949

50-
/// An function that will create an argument to be passed to a function as a borrow.
50+
/// A function that will create an argument to be passed to a function as a borrow.
5151
struct BorrowedArgument {
5252
/// Name of the function.
5353
name: String,
@@ -74,6 +74,16 @@ impl BorrowedArgument {
7474
/// The macro must be placed on an `async` function without `#[test]` or `#[tokio::test]` that only
7575
/// has `&DataFolder` parameters. It will generate one `#[tokio::test]` function for each
7676
/// permutation with replacement of `DataFolder` configurations that call the annotated function.
77+
///
78+
/// ```ignore
79+
/// // This doc test is not tested as procedural macros cannot be used in their own crates.
80+
/// use modelardb_macros::data_folder_test;
81+
///
82+
/// #[data_folder_test]
83+
/// async fn test_data_folder_drop_table(data_folder: &DataFolder) {
84+
/// data_folder.drop_table("table_name").await.unwrap();
85+
/// }
86+
/// ```.
7787
#[proc_macro_attribute]
7888
pub fn data_folder_test(
7989
_args: proc_macro::TokenStream,
@@ -91,10 +101,11 @@ pub fn data_folder_test(
91101
// for the generated function start with the name of the annotated function followed by the name
92102
// of each data folder configuration used with each part of the name separated by two
93103
// underscores. By using the annotated function name as a prefix, all the generated tests can be
94-
// run with cargo test annotated function name. The names of the data folder configurations used
95-
// are included to make it simple to identify which data folder configurations causes a test to
96-
// fail. Finally, each part of the name is separated by two underscores to make it more readable
97-
// and to avoid conflicts with user code as function names should not use two underscores.
104+
// run with `cargo test annotated_function_name`. The names of the data folder configurations
105+
// used are included to make it simple to identify which data folder configurations causes a
106+
// test to fail. Finally, each part of the name is separated by two underscores to make it more
107+
// readable and to avoid conflicts with user code as function names should not use two
108+
// underscores.
98109
let data_folders = &[
99110
BorrowedArgument::new(
100111
"in_memory_data_folder",
@@ -108,12 +119,7 @@ pub fn data_folder_test(
108119
true,
109120
true,
110121
),
111-
BorrowedArgument::new(
112-
"aws3_data_folder",
113-
"modelardb_test::data_folder",
114-
false,
115-
true,
116-
),
122+
BorrowedArgument::new("s3_data_folder", "modelardb_test::data_folder", false, true),
117123
BorrowedArgument::new(
118124
"azure_data_folder",
119125
"modelardb_test::data_folder",
@@ -122,6 +128,12 @@ pub fn data_folder_test(
122128
),
123129
];
124130

131+
// Create an iterate that produce all permutations with replacements of the items in
132+
// data_folders. First the code creates an iterator that repeats the data_folders iterator
133+
// data_folder_parameter_count times. Then these iterators are crossed together to produce each
134+
// permutation with replacements. This should be the same as data_folder_parameter_count nested
135+
// loops iterating over data_folders without the need to know the number of loops required
136+
// beforehand since the value of data_folder_parameter_count is not known at development time.
125137
let data_folder_permutations_with_replacements =
126138
itertools::repeat_n(data_folders.iter(), data_folder_parameter_count as usize)
127139
.multi_cartesian_product();
@@ -142,12 +154,28 @@ pub fn data_folder_test(
142154
/// stores. The macro must be placed on an `async` function without `#[test]` or `#[tokio::test]`
143155
/// that only has `&dyn ObjectStore` parameters. It will generate one `#[tokio::test]` function for
144156
/// each permutation with replacement of supported object stores that call the annotated function.
157+
///
158+
/// ```ignore
159+
/// // This doc test is not tested as procedural macros cannot be used in their own crates.
160+
/// use modelardb_macros::object_store_test;
161+
///
162+
/// use futures::StreamExt;
163+
/// use object_store::ObjectStore;
164+
///
165+
/// #[object_store_test]
166+
/// async fn test_object_store_list(object_store: &dyn ObjectStore) {
167+
/// let mut files = object_store.list(None);
168+
/// while let Some(f) = files.next().await {
169+
/// f.unwrap();
170+
/// }
171+
/// }
172+
/// ```.
145173
#[proc_macro_attribute]
146174
pub fn object_store_test(
147175
_args: proc_macro::TokenStream,
148176
input: proc_macro::TokenStream,
149177
) -> proc_macro::TokenStream {
150-
// input is cloned as it is read as part of this macro and extended with the generated code.
178+
// See the comments in data_folder_test as it follows the same structure as object_store_test.
151179
let (function_name, object_store_parameter_count) =
152180
function_name_and_checked_parameter_count(input.clone(), ParameterType::ObjectStore);
153181

@@ -165,7 +193,7 @@ pub fn object_store_test(
165193
false,
166194
),
167195
BorrowedArgument::new(
168-
"aws3_object_store",
196+
"s3_object_store",
169197
"modelardb_test::object_store",
170198
false,
171199
false,
@@ -229,10 +257,10 @@ fn next_ident_and_group(input: TokenStream) -> Option<(IdentStruct, GroupStruct)
229257
None
230258
}
231259

232-
/// Returns the number of `parameter_type` parameters in `function_parameter_group`. `The return
233-
/// type is `u16` as `rustc` returns an error if a function or method have more than 65,535
234-
/// parameters at the time of writing. An [`ModelarDbMacrosError`] is returned if
235-
/// `function_parameter_group` contain anything but multiple instances of `parameter_type`.
260+
/// Returns the number of `parameter_type` parameters in `function_parameter_group`. The return type
261+
/// is `u16` as `rustc` returns an error if a function or method have more than 65,535 parameters at
262+
/// the time of writing. A [`ModelarDbMacrosError`] is returned if `function_parameter_group`
263+
/// contain anything but multiple instances of `parameter_type`.
236264
fn expect_parameter_type_and_count(
237265
function_parameter_group: GroupStruct,
238266
parameter_type: ParameterType,
@@ -263,7 +291,7 @@ fn expect_parameter_type(
263291
token_iterator: &mut impl Iterator<Item = TokenTree>,
264292
parameter_type: ParameterType,
265293
) -> Result<()> {
266-
// The contents of the first Ident token cannot checked as it is the parameter name.
294+
// The contents of the first Ident token cannot be checked as it is the parameter name.
267295
expect_ident_without_contents(token_iterator)?;
268296
expect_punct_with_contents(token_iterator, ':')?;
269297
expect_punct_with_contents(token_iterator, '&')?;
@@ -279,7 +307,7 @@ fn expect_parameter_type(
279307
Ok(())
280308
}
281309

282-
/// Return [`Ok`] if the next [`TokenTree`] from `token_iterator` is an [`Ident`], otherwise a
310+
/// Return [`Ok`] if the next [`TokenTree`] from `token_iterator` is a [`Ident`], otherwise a
283311
/// [`ModelarDbMacrosError`] is returned.
284312
fn expect_ident_without_contents(
285313
token_iterator: &mut impl Iterator<Item = TokenTree>,

crates/modelardb_server/Cargo.toml

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -36,10 +36,10 @@ deltalake = { workspace = true, features = ["datafusion"] }
3636
futures.workspace = true
3737
http.workspace = true
3838
http-body-util.workspace = true
39-
modelardb_auth = { path = "../modelardb_auth" }
40-
modelardb_compression = { path = "../modelardb_compression" }
41-
modelardb_storage = { path = "../modelardb_storage" }
42-
modelardb_types = { path = "../modelardb_types" }
39+
modelardb_auth.workspace = true
40+
modelardb_compression.workspace = true
41+
modelardb_storage.workspace = true
42+
modelardb_types.workspace = true
4343
object_store = { workspace = true, features = ["aws", "azure"] }
4444
prost.workspace = true
4545
rand.workspace = true
@@ -58,8 +58,8 @@ tracing = { workspace = true, features = ["max_level_debug", "release_max_level_
5858
tracing-subscriber.workspace = true
5959

6060
[dev-dependencies]
61-
modelardb_test = { path = "../modelardb_test" }
62-
modelardb_auth = { path = "../modelardb_auth", features = ["testing"] }
61+
modelardb_test.workspace = true
62+
modelardb_auth = { workspace = true, features = ["testing"] }
6363
proptest.workspace = true
6464
tempfile.workspace = true
6565

crates/modelardb_storage/Cargo.toml

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -30,8 +30,8 @@ datafusion-proto.workspace = true
3030
delta_kernel.workspace = true
3131
deltalake = { workspace = true, features = ["datafusion", "s3"] }
3232
futures.workspace = true
33-
modelardb_compression = { path = "../modelardb_compression" }
34-
modelardb_types = { path = "../modelardb_types" }
33+
modelardb_compression.workspace = true
34+
modelardb_types.workspace = true
3535
object_store = { workspace = true, features = ["aws", "azure"] }
3636
serde_json.workspace = true
3737
sqlparser.workspace = true
@@ -42,6 +42,6 @@ uuid.workspace = true
4242

4343
[dev-dependencies]
4444
futures.workspace = true
45-
modelardb_test = { path = "../modelardb_test" }
45+
modelardb_test.workspace = true
4646
tempfile.workspace = true
4747
tokio.workspace = true

0 commit comments

Comments
 (0)