From cb8f879b928eca5e9b3d2dabffe85d6b8ceaf309 Mon Sep 17 00:00:00 2001 From: Cyphersnake Date: Mon, 24 Oct 2022 17:25:10 +0400 Subject: [PATCH 1/5] Add `parse_env` attribute to `confique::Config` macro Since there is no established format for representing collections within env variables, to enable their support, the feature with a custom parser attribute added. Add helpers for parsing env variables into collections Improve rustdoc for `env_utils` functions Improve `parse_env` example --- CHANGELOG.md | 3 +- Cargo.toml | 1 + examples/parse_env.rs | 83 ++++++++++++++++++++++++++++++++++ macro/src/gen/mod.rs | 34 ++++++++------ macro/src/ir.rs | 1 + macro/src/parse.rs | 52 ++++++++++++--------- src/env.rs | 103 +++++++++++++++++++++++++++++++++++++++++- src/internal.rs | 26 +++++++++-- src/lib.rs | 4 ++ tests/general.rs | 25 +++++++++- 10 files changed, 290 insertions(+), 42 deletions(-) create mode 100644 examples/parse_env.rs diff --git a/CHANGELOG.md b/CHANGELOG.md index 8b14ad1..c352fb5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,8 @@ All notable changes to this project will be documented in this file. ## [Unreleased] - +### Added +- `parse_env` attribute which is using for parse env variable in arbitrary type ## [0.2.0] - 2022-10-21 ### Added diff --git a/Cargo.toml b/Cargo.toml index 88988a0..9581475 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -28,6 +28,7 @@ yaml = ["serde_yaml"] [dependencies] confique-macro = { version = "=0.0.5", path = "macro" } json5 = { version = "0.4.1", optional = true } +paste = "1.0.9" serde = { version = "1", features = ["derive"] } serde_yaml = { version = "0.8", optional = true } toml = { version = "0.5", optional = true } diff --git a/examples/parse_env.rs b/examples/parse_env.rs new file mode 100644 index 0000000..e9d56ff --- /dev/null +++ b/examples/parse_env.rs @@ -0,0 +1,83 @@ +#![allow(dead_code)] + +use confique::{ + env_utils::{ + to_collection_by_char_separator, to_collection_by_comma, to_collection_by_semicolon, + }, + Config, +}; +use std::{collections::HashSet, num::NonZeroU64, path::PathBuf, str::FromStr}; + +#[derive(Debug, Config)] +struct Conf { + #[config( + env = "PATHS", + parse_env = to_collection_by_comma + )] + paths: HashSet, + #[config( + env = "PORTS", + parse_env = to_collection_by_semicolon + )] + ports: Vec, + #[config( + env = "NAMES", + parse_env = to_collection_by_char_separator::<'|', _, _> + )] + names: Vec, + #[config( + env = "TIMEOUT", + parse_env = NonZeroU64::from_str, + )] + timeout_seconds: NonZeroU64, + #[config( + env = "FORMATS", + parse_env = parse_formats, + )] + formats: Vec, +} + +#[derive(Debug, serde::Deserialize)] +enum Format { + Env, + Toml, + Json5, + Yaml, +} + +#[derive(Debug)] +enum Error {} + +fn parse_formats(input: &str) -> Result, Error> { + let mut result = Vec::new(); + + if input.contains("toml") { + result.push(Format::Toml); + } + if input.contains("env") { + result.push(Format::Env); + } + if input.contains("yaml") { + result.push(Format::Yaml); + } + if input.contains("json5") { + result.push(Format::Json5); + } + + Ok(result) +} + +fn main() -> Result<(), Box> { + std::env::set_var("PATHS", "/bin/ls,/usr/local/bin,/usr/bin/ls"); + std::env::set_var("PORTS", "8080;8888;8000"); + std::env::set_var( + "NAMES", + "Mervinc Harmon|Alfreda Valenzuela|Arlen Cabrera|Damon Rice|Willie Schwartz", + ); + std::env::set_var("TIMEOUT", "100"); + std::env::set_var("FORMATS", "json5,yaml;.env"); + + println!("{:#?}", Conf::builder().env().load()?); + + Ok(()) +} diff --git a/macro/src/gen/mod.rs b/macro/src/gen/mod.rs index 55303d2..de3b9f6 100644 --- a/macro/src/gen/mod.rs +++ b/macro/src/gen/mod.rs @@ -147,22 +147,28 @@ fn gen_partial_mod(input: &ir::Input) -> TokenStream { } }); - let from_env_fields = input.fields.iter().map(|f| { - match &f.kind { - FieldKind::Leaf { env: Some(key), deserialize_with, .. } => { - let field = format!("{}::{}", input.name, f.name); - match deserialize_with { - None => quote! { - confique::internal::from_env(#key, #field)? - }, - Some(d) => quote! { - confique::internal::from_env_with(#key, #field, #d)? - }, - } + let from_env_fields = input.fields.iter().map(|f| match &f.kind { + FieldKind::Leaf { + env: Some(key), + deserialize_with, + parse_env, + .. + } => { + let field = format!("{}::{}", input.name, f.name); + match (parse_env, deserialize_with) { + (None, None) => quote! { + confique::internal::from_env(#key, #field)? + }, + (None, Some(deserialize_with)) => quote! { + confique::internal::deserialize_from_env_with(#key, #field, #deserialize_with)? + }, + (Some(parse_env), None) | (Some(parse_env), Some(_)) => quote! { + confique::internal::parse_from_env_with(#key, #field, #parse_env)? + }, } - FieldKind::Leaf { .. } => quote! { None }, - FieldKind::Nested { .. } => quote! { confique::Partial::from_env()? }, } + FieldKind::Leaf { .. } => quote! { None }, + FieldKind::Nested { .. } => quote! { confique::Partial::from_env()? }, }); let fallbacks = input.fields.iter().map(|f| { diff --git a/macro/src/ir.rs b/macro/src/ir.rs index f04708f..5694450 100644 --- a/macro/src/ir.rs +++ b/macro/src/ir.rs @@ -24,6 +24,7 @@ pub(crate) enum FieldKind { Leaf { env: Option, deserialize_with: Option, + parse_env: Option, kind: LeafKind, }, diff --git a/macro/src/parse.rs b/macro/src/parse.rs index 6167785..bce7cdb 100644 --- a/macro/src/parse.rs +++ b/macro/src/parse.rs @@ -65,32 +65,27 @@ impl Field { FieldKind::Nested { ty: field.ty } } else { - match unwrap_option(&field.ty) { - None => FieldKind::Leaf { - env: attrs.env, - deserialize_with: attrs.deserialize_with, - kind: LeafKind::Required { - default: attrs.default, - ty: field.ty, - }, - }, - Some(inner) => { - if attrs.default.is_some() { + if attrs.env.is_none() && attrs.parse_env.is_some() { + return Err(Error::new( + field.ident.span(), + "A `parse_env` attribute, cannot be provided without the `env` attribute", + )); + } + FieldKind::Leaf { + env: attrs.env, + deserialize_with: attrs.deserialize_with, + parse_env: attrs.parse_env, + kind: match unwrap_option(&field.ty) { + Some(_inner) if attrs.default.is_some() => { return Err(Error::new( field.ident.span(), "optional fields (type `Option<_>`) cannot have default \ values (`#[config(default = ...)]`)", )); - } - - FieldKind::Leaf { - env: attrs.env, - deserialize_with: attrs.deserialize_with, - kind: LeafKind::Optional { - inner_ty: inner.clone(), - }, - } - } + }, + Some(inner) => LeafKind::Optional { inner_ty: inner.clone() }, + None => LeafKind::Required { default: attrs.default, ty: field.ty }, + }, } }; @@ -242,6 +237,10 @@ fn extract_internal_attrs( duplicate_if!(out.env.is_some()); out.env = Some(key); } + InternalAttr::ParseEnv(path) => { + duplicate_if!(out.parse_env.is_some()); + out.parse_env = Some(path); + } InternalAttr::DeserializeWith(path) => { duplicate_if!(out.deserialize_with.is_some()); out.deserialize_with = Some(path); @@ -259,6 +258,7 @@ struct InternalAttrs { default: Option, env: Option, deserialize_with: Option, + parse_env: Option, } enum InternalAttr { @@ -266,6 +266,7 @@ enum InternalAttr { Default(Expr), Env(String), DeserializeWith(syn::Path), + ParseEnv(syn::Path), } impl InternalAttr { @@ -274,6 +275,7 @@ impl InternalAttr { Self::Nested => "nested", Self::Default(_) => "default", Self::Env(_) => "env", + Self::ParseEnv(_) => "parse_env", Self::DeserializeWith(_) => "deserialize_with", } } @@ -310,6 +312,14 @@ impl Parse for InternalAttr { } } + "parse_env" => { + let _: Token![=] = input.parse()?; + let path: syn::Path = input.parse()?; + assert_empty_or_comma(input)?; + + Ok(Self::ParseEnv(path)) + } + "deserialize_with" => { let _: Token![=] = input.parse()?; let path: syn::Path = input.parse()?; diff --git a/src/env.rs b/src/env.rs index 5b12e05..8060a8f 100644 --- a/src/env.rs +++ b/src/env.rs @@ -128,7 +128,6 @@ impl<'de> serde::Deserializer<'de> for Deserializer { } } - #[cfg(test)] mod tests { use super::*; @@ -166,3 +165,105 @@ mod tests { assert_eq!(de("-123.456"), Ok(-123.456f64)); } } + +/// This module contains helper methods to simplify configuration via environment variables +pub mod env_utils { + use std::str::FromStr; + + /// Helper function to parse any type that implements [`std::str::FromStr`] + /// into a collection that implements [`std::iter::FromIterator`], spliting original string by + /// const char separator. + /// + /// # To Vec + /// ``` + /// use crate::confique::Config; + /// #[derive(Debug, confique::Config)] + /// struct Conf { + /// #[config( + /// env = "PORTS", + /// parse_env = confique::env_utils::to_collection_by_char_separator::<',', _, _> + /// )] + /// ports: Vec + /// } + /// + /// std::env::set_var("PORTS", "8080,8000,8888"); + /// println!("{:?}", Conf::builder().env().load().unwrap()) + /// ``` + /// + /// # To HashSet + /// ``` + /// use crate::confique::Config; + /// #[derive(Debug, confique::Config)] + /// struct Conf { + /// #[config( + /// env = "PATHS", + /// parse_env = confique::env_utils::to_collection_by_char_separator::<';', _, _> + /// )] + /// paths: std::collections::HashSet, + /// } + /// + /// std::env::set_var("PATHS", "/bin;/user/bin;/home/user/.cargo/bin"); + /// println!("{:?}", Conf::builder().env().load().unwrap()) + /// ``` + pub fn to_collection_by_char_separator< + const SEPARATOR: char, + T: FromStr, + C: FromIterator::Err>>, + >( + input: &str, + ) -> C { + input.split(SEPARATOR.to_owned()).map(T::from_str).collect() + } + + macro_rules! specify_fn_wrapper { + ($symbol_name:ident, $symbol:tt) => { + ::paste::paste! { + /// Helper function to parse any type that implements [`std::str::FromStr`] + /// into a collection that implements [`std::iter::FromIterator`], spliting original string by + #[doc = stringify!($symbol_name)] + /// . + /// + /// # To Vec + /// ``` + /// use crate::confique::Config; + /// #[derive(Debug, confique::Config)] + /// struct Conf { + /// #[config( + /// env = "PORTS", + #[doc = concat!(" parse_env = confique::env_utils::", stringify!([],))] + /// )] + /// ports: Vec + /// } + /// + #[doc = concat!("std::env::set_var(\"PORTS\", \"8080", $symbol, "8000", $symbol, "8888", "\");")] + /// println!("{:#?}", Conf::builder().env().load().unwrap()) + /// ``` + /// + /// # To HashSet + /// ``` + /// use crate::confique::Config; + /// #[derive(Debug, confique::Config)] + /// struct Conf { + /// #[config( + /// env = "PATHS", + #[doc = concat!(" parse_env = confique::env_utils::", stringify!([],))] + /// )] + /// paths: std::collections::HashSet, + /// } + /// + #[doc = concat!("std::env::set_var(\"PATHS\", \"/bin", $symbol, "/user/bin", $symbol, "/home/user/.cargo/bin", "\");")] + /// println!("{:#?}", Conf::builder().env().load().unwrap()) + /// ``` + pub fn []::Err>>>( + input: &str, + ) -> C { + to_collection_by_char_separator::<$symbol, _, _>(input) + } + } + } + } + + specify_fn_wrapper!(comma, ','); + specify_fn_wrapper!(semicolon, ';'); + specify_fn_wrapper!(space, ' '); +} diff --git a/src/internal.rs b/src/internal.rs index ae68f54..c0b4856 100644 --- a/src/internal.rs +++ b/src/internal.rs @@ -2,8 +2,9 @@ //! intended to be used directly. None of this is covered by semver! Do not use //! any of this directly. -use crate::{error::ErrorInner, Error}; +use std::fmt::Debug; +use crate::{error::ErrorInner, Error}; pub fn deserialize_default(src: I) -> Result where @@ -41,10 +42,29 @@ pub fn from_env<'de, T: serde::Deserialize<'de>>( key: &str, field: &str, ) -> Result, Error> { - from_env_with(key, field, |de| T::deserialize(de)) + deserialize_from_env_with(key, field, |de| T::deserialize(de)) } -pub fn from_env_with( +pub fn parse_from_env_with( + key: &str, + field: &str, + parse: fn(&str) -> Result, +) -> Result, Error> { + from_env::(key, field)? + .as_deref() + .map(parse) + .transpose() + .map_err(|err| { + ErrorInner::EnvDeserialization { + field: field.to_owned(), + key: key.to_owned(), + msg: format!("Error while parse: {:?}", err), + } + .into() + }) +} + +pub fn deserialize_from_env_with( key: &str, field: &str, deserialize: fn(crate::env::Deserializer) -> Result, diff --git a/src/lib.rs b/src/lib.rs index 0f63300..31cdce9 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -176,6 +176,7 @@ pub mod internal; mod builder; mod env; +pub use env::env_utils; mod error; pub mod meta; @@ -322,6 +323,9 @@ pub use crate::{ /// /// [serde-deser]: https://serde.rs/field-attrs.html#deserialize_with /// +/// - **`#[config(from_env = path::to::function, env = "KEY")]`**: like +/// deserialize_with` attribute, but only for fields from an environment variable. +/// Can only be present if the `env` attribute is present /// /// ## Special types for leaf fields /// diff --git a/tests/general.rs b/tests/general.rs index 878b0bc..d33b36b 100644 --- a/tests/general.rs +++ b/tests/general.rs @@ -1,5 +1,7 @@ use std::{collections::HashMap, net::IpAddr, path::PathBuf}; + use pretty_assertions::assert_eq; +use serde::Deserialize; use confique::{meta, Config, Partial}; @@ -110,6 +112,9 @@ mod full { #[config(env = "ENV_TEST_FULL_3")] optional: Option, + + #[config(env = "ENV_TEST_FULL_4", parse_env = parse_dummy_collection)] + env_collection: DummyCollection<','>, } } @@ -247,6 +252,14 @@ fn full() { kind: meta::LeafKind::Optional, }, }, + meta::Field { + name: "env_collection", + doc: &[], + kind: meta::FieldKind::Leaf { + env: Some("ENV_TEST_FULL_4"), + kind: meta::LeafKind::Required { default: None }, + }, + }, ], }, }, @@ -266,6 +279,7 @@ fn full() { assert_eq!(def.env.required, None); assert_eq!(def.env.with_default, Some(8080)); assert_eq!(def.env.optional, None); + assert_eq!(def.env.env_collection, None); } #[derive(Debug, PartialEq)] @@ -275,12 +289,19 @@ pub(crate) fn deserialize_dummy<'de, D>(deserializer: D) -> Result, { - use serde::Deserialize; - let s = String::deserialize(deserializer)?; Ok(Dummy(format!("dummy {s}"))) } +#[derive(Debug, PartialEq, Deserialize)] +struct DummyCollection(Vec); + +pub(crate) fn parse_dummy_collection(input: &str) -> Result, String> { + Ok(DummyCollection( + input.split(SEPARATOR).map(ToString::to_string).collect(), + )) +} + // This only makes sure this compiles and doesn't result in any "cannot infer // type" problems. #[test] From 1e74330f00cae047b8fd762adba077efd7f6d787 Mon Sep 17 00:00:00 2001 From: Lukas Kalbertodt Date: Sun, 6 Nov 2022 10:37:58 +0100 Subject: [PATCH 2/5] Improve formatting slightly --- macro/src/gen/mod.rs | 37 +++++++++++++++++-------------------- macro/src/parse.rs | 25 ++++++++++++++----------- src/env.rs | 1 + 3 files changed, 32 insertions(+), 31 deletions(-) diff --git a/macro/src/gen/mod.rs b/macro/src/gen/mod.rs index de3b9f6..d51c698 100644 --- a/macro/src/gen/mod.rs +++ b/macro/src/gen/mod.rs @@ -147,28 +147,25 @@ fn gen_partial_mod(input: &ir::Input) -> TokenStream { } }); - let from_env_fields = input.fields.iter().map(|f| match &f.kind { - FieldKind::Leaf { - env: Some(key), - deserialize_with, - parse_env, - .. - } => { - let field = format!("{}::{}", input.name, f.name); - match (parse_env, deserialize_with) { - (None, None) => quote! { - confique::internal::from_env(#key, #field)? - }, - (None, Some(deserialize_with)) => quote! { - confique::internal::deserialize_from_env_with(#key, #field, #deserialize_with)? - }, - (Some(parse_env), None) | (Some(parse_env), Some(_)) => quote! { - confique::internal::parse_from_env_with(#key, #field, #parse_env)? - }, + let from_env_fields = input.fields.iter().map(|f| { + match &f.kind { + FieldKind::Leaf { env: Some(key), deserialize_with, parse_env, .. } => { + let field = format!("{}::{}", input.name, f.name); + match (parse_env, deserialize_with) { + (None, None) => quote! { + confique::internal::from_env(#key, #field)? + }, + (None, Some(deserialize_with)) => quote! { + confique::internal::deserialize_from_env_with(#key, #field, #deserialize_with)? + }, + (Some(parse_env), None) | (Some(parse_env), Some(_)) => quote! { + confique::internal::parse_from_env_with(#key, #field, #parse_env)? + }, + } } + FieldKind::Leaf { .. } => quote! { None }, + FieldKind::Nested { .. } => quote! { confique::Partial::from_env()? }, } - FieldKind::Leaf { .. } => quote! { None }, - FieldKind::Nested { .. } => quote! { confique::Partial::from_env()? }, }); let fallbacks = input.fields.iter().map(|f| { diff --git a/macro/src/parse.rs b/macro/src/parse.rs index bce7cdb..5c0fb99 100644 --- a/macro/src/parse.rs +++ b/macro/src/parse.rs @@ -71,21 +71,24 @@ impl Field { "A `parse_env` attribute, cannot be provided without the `env` attribute", )); } + + let kind = match unwrap_option(&field.ty) { + Some(_) if attrs.default.is_some() => { + return Err(Error::new( + field.ident.span(), + "optional fields (type `Option<_>`) cannot have default \ + values (`#[config(default = ...)]`)", + )); + }, + Some(inner) => LeafKind::Optional { inner_ty: inner.clone() }, + None => LeafKind::Required { default: attrs.default, ty: field.ty }, + }; + FieldKind::Leaf { env: attrs.env, deserialize_with: attrs.deserialize_with, parse_env: attrs.parse_env, - kind: match unwrap_option(&field.ty) { - Some(_inner) if attrs.default.is_some() => { - return Err(Error::new( - field.ident.span(), - "optional fields (type `Option<_>`) cannot have default \ - values (`#[config(default = ...)]`)", - )); - }, - Some(inner) => LeafKind::Optional { inner_ty: inner.clone() }, - None => LeafKind::Required { default: attrs.default, ty: field.ty }, - }, + kind, } }; diff --git a/src/env.rs b/src/env.rs index 8060a8f..36c6648 100644 --- a/src/env.rs +++ b/src/env.rs @@ -128,6 +128,7 @@ impl<'de> serde::Deserializer<'de> for Deserializer { } } + #[cfg(test)] mod tests { use super::*; From e2dded17fabe3eb6ab8d6652a089aa1cafb5f763 Mon Sep 17 00:00:00 2001 From: Lukas Kalbertodt Date: Sun, 6 Nov 2022 11:23:21 +0100 Subject: [PATCH 3/5] Change `parse_env` error type from `Debug` to `impl std::error::Error` This is the more appropriate trait I think and should work well for most real world use cases. --- examples/parse_env.rs | 7 ++---- macro/src/gen/mod.rs | 7 +++--- src/error.rs | 12 ++++++++++ src/internal.rs | 51 +++++++++++++++++++++++-------------------- tests/general.rs | 10 ++++----- 5 files changed, 50 insertions(+), 37 deletions(-) diff --git a/examples/parse_env.rs b/examples/parse_env.rs index e9d56ff..266a493 100644 --- a/examples/parse_env.rs +++ b/examples/parse_env.rs @@ -6,7 +6,7 @@ use confique::{ }, Config, }; -use std::{collections::HashSet, num::NonZeroU64, path::PathBuf, str::FromStr}; +use std::{collections::HashSet, num::NonZeroU64, path::PathBuf, str::FromStr, convert::Infallible}; #[derive(Debug, Config)] struct Conf { @@ -45,10 +45,7 @@ enum Format { Yaml, } -#[derive(Debug)] -enum Error {} - -fn parse_formats(input: &str) -> Result, Error> { +fn parse_formats(input: &str) -> Result, Infallible> { let mut result = Vec::new(); if input.contains("toml") { diff --git a/macro/src/gen/mod.rs b/macro/src/gen/mod.rs index d51c698..93f5b33 100644 --- a/macro/src/gen/mod.rs +++ b/macro/src/gen/mod.rs @@ -156,10 +156,11 @@ fn gen_partial_mod(input: &ir::Input) -> TokenStream { confique::internal::from_env(#key, #field)? }, (None, Some(deserialize_with)) => quote! { - confique::internal::deserialize_from_env_with(#key, #field, #deserialize_with)? + confique::internal::from_env_with_deserializer( + #key, #field, #deserialize_with)? }, - (Some(parse_env), None) | (Some(parse_env), Some(_)) => quote! { - confique::internal::parse_from_env_with(#key, #field, #parse_env)? + (Some(parse_env), _) => quote! { + confique::internal::from_env_with_parser(#key, #field, #parse_env)? }, } } diff --git a/src/error.rs b/src/error.rs index f3e7e2e..f46fa02 100644 --- a/src/error.rs +++ b/src/error.rs @@ -50,6 +50,13 @@ pub(crate) enum ErrorInner { msg: String, }, + /// When a custom `parse_env` function fails. + EnvParseError { + field: String, + key: String, + err: Box, + }, + /// Returned by the [`Source`] impls for `Path` and `PathBuf` if the file /// extension is not supported by confique or if the corresponding Cargo /// feature of confique was not enabled. @@ -71,6 +78,7 @@ impl std::error::Error for Error { ErrorInner::MissingValue(_) => None, ErrorInner::EnvNotUnicode { .. } => None, ErrorInner::EnvDeserialization { .. } => None, + ErrorInner::EnvParseError { err, .. } => Some(&**err), ErrorInner::UnsupportedFileFormat { .. } => None, ErrorInner::MissingFileExtension { .. } => None, ErrorInner::MissingRequiredFile { .. } => None, @@ -107,6 +115,10 @@ impl fmt::Display for Error { std::write!(f, "failed to deserialize value `{field}` from \ environment variable `{key}`: {msg}") } + ErrorInner::EnvParseError { field, key, err } => { + std::write!(f, "failed to parse environment variable `{key}` into \ + field `{field}`: {err}") + } ErrorInner::UnsupportedFileFormat { path } => { std::write!(f, "unknown configuration file format/extension: '{}'", diff --git a/src/internal.rs b/src/internal.rs index c0b4856..3ad175f 100644 --- a/src/internal.rs +++ b/src/internal.rs @@ -2,8 +2,6 @@ //! intended to be used directly. None of this is covered by semver! Do not use //! any of this directly. -use std::fmt::Debug; - use crate::{error::ErrorInner, Error}; pub fn deserialize_default(src: I) -> Result @@ -38,48 +36,53 @@ pub fn map_err_prefix_path(res: Result, prefix: &str) -> Result { + match std::env::var($key) { + Err(std::env::VarError::NotPresent) => return Ok(None), + Err(std::env::VarError::NotUnicode(_)) => { + let err = ErrorInner::EnvNotUnicode { + key: $key.into(), + field: $field.into(), + }; + return Err(err.into()); + } + Ok(s) => s, + } + }; +} + pub fn from_env<'de, T: serde::Deserialize<'de>>( key: &str, field: &str, ) -> Result, Error> { - deserialize_from_env_with(key, field, |de| T::deserialize(de)) + from_env_with_deserializer(key, field, |de| T::deserialize(de)) } -pub fn parse_from_env_with( +pub fn from_env_with_parser( key: &str, field: &str, parse: fn(&str) -> Result, ) -> Result, Error> { - from_env::(key, field)? - .as_deref() - .map(parse) - .transpose() + let v = get_env_var!(key, field); + parse(&v) + .map(Some) .map_err(|err| { - ErrorInner::EnvDeserialization { + ErrorInner::EnvParseError { field: field.to_owned(), key: key.to_owned(), - msg: format!("Error while parse: {:?}", err), - } - .into() + err: Box::new(err), + }.into() }) } -pub fn deserialize_from_env_with( +pub fn from_env_with_deserializer( key: &str, field: &str, deserialize: fn(crate::env::Deserializer) -> Result, ) -> Result, Error> { - let s = match std::env::var(key) { - Err(std::env::VarError::NotPresent) => return Ok(None), - Err(std::env::VarError::NotUnicode(_)) => { - let err = ErrorInner::EnvNotUnicode { - key: key.into(), - field: field.into(), - }; - return Err(err.into()); - } - Ok(s) => s, - }; + let s = get_env_var!(key, field); match deserialize(crate::env::Deserializer::new(s)) { Ok(v) => Ok(Some(v)), diff --git a/tests/general.rs b/tests/general.rs index d33b36b..493f689 100644 --- a/tests/general.rs +++ b/tests/general.rs @@ -1,4 +1,4 @@ -use std::{collections::HashMap, net::IpAddr, path::PathBuf}; +use std::{collections::HashMap, net::IpAddr, path::PathBuf, convert::Infallible}; use pretty_assertions::assert_eq; use serde::Deserialize; @@ -114,7 +114,7 @@ mod full { optional: Option, #[config(env = "ENV_TEST_FULL_4", parse_env = parse_dummy_collection)] - env_collection: DummyCollection<','>, + env_collection: DummyCollection, } } @@ -294,11 +294,11 @@ where } #[derive(Debug, PartialEq, Deserialize)] -struct DummyCollection(Vec); +struct DummyCollection(Vec); -pub(crate) fn parse_dummy_collection(input: &str) -> Result, String> { +pub(crate) fn parse_dummy_collection(input: &str) -> Result { Ok(DummyCollection( - input.split(SEPARATOR).map(ToString::to_string).collect(), + input.split(',').map(ToString::to_string).collect(), )) } From 49828fc2e338395bd0a8f2f7290ac1b6543239de Mon Sep 17 00:00:00 2001 From: Lukas Kalbertodt Date: Sun, 6 Nov 2022 12:14:47 +0100 Subject: [PATCH 4/5] Change API of `parse_env` functions I think this is an overall improvement. A few things done here: - They are available as `::env::parse` instead of `::env_utils`. The `env` module gets functions of its own soon enough. - Renamed functions to be shorter: `list_by_sep`, `list_by_comma`, ... - The docs were adjusted. One example is enough. And the functions with a fixed separator don't need the full docs again. That way we can avoid the `paste` dependency. - Functions now always return `Result`. While the previous version was slightly more flexible, I don't think anyone would benefit from that flexibility (as `parse_env` requires `Result<_, _>` anyway) and this way it's a bit clearer what the function does, especially for those who don't know the nifty `FromIterator for Result` impl. - The example was adjusted accordingly. I also changed the names to obviously dummy names as I didn't know the existing names and don't want to spend time investigating whether I want their names in my code base :D --- Cargo.toml | 1 - examples/parse_env.rs | 45 +++++----------- src/env.rs | 122 ++++++++++++++---------------------------- src/lib.rs | 3 +- 4 files changed, 55 insertions(+), 116 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 9581475..88988a0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -28,7 +28,6 @@ yaml = ["serde_yaml"] [dependencies] confique-macro = { version = "=0.0.5", path = "macro" } json5 = { version = "0.4.1", optional = true } -paste = "1.0.9" serde = { version = "1", features = ["derive"] } serde_yaml = { version = "0.8", optional = true } toml = { version = "0.5", optional = true } diff --git a/examples/parse_env.rs b/examples/parse_env.rs index 266a493..11af4b0 100644 --- a/examples/parse_env.rs +++ b/examples/parse_env.rs @@ -1,39 +1,24 @@ #![allow(dead_code)] -use confique::{ - env_utils::{ - to_collection_by_char_separator, to_collection_by_comma, to_collection_by_semicolon, - }, - Config, -}; +use confique::Config; use std::{collections::HashSet, num::NonZeroU64, path::PathBuf, str::FromStr, convert::Infallible}; + #[derive(Debug, Config)] struct Conf { - #[config( - env = "PATHS", - parse_env = to_collection_by_comma - )] + #[config(env = "PATHS", parse_env = confique::env::parse::list_by_colon)] paths: HashSet, - #[config( - env = "PORTS", - parse_env = to_collection_by_semicolon - )] + + #[config(env = "PORTS", parse_env = confique::env::parse::list_by_comma)] ports: Vec, - #[config( - env = "NAMES", - parse_env = to_collection_by_char_separator::<'|', _, _> - )] + + #[config(env = "NAMES", parse_env = confique::env::parse::list_by_sep::<'|', _, _>)] names: Vec, - #[config( - env = "TIMEOUT", - parse_env = NonZeroU64::from_str, - )] + + #[config(env = "TIMEOUT", parse_env = NonZeroU64::from_str)] timeout_seconds: NonZeroU64, - #[config( - env = "FORMATS", - parse_env = parse_formats, - )] + + #[config(env = "FORMATS", parse_env = parse_formats)] formats: Vec, } @@ -45,6 +30,7 @@ enum Format { Yaml, } +/// Example custom parser. fn parse_formats(input: &str) -> Result, Infallible> { let mut result = Vec::new(); @@ -66,11 +52,8 @@ fn parse_formats(input: &str) -> Result, Infallible> { fn main() -> Result<(), Box> { std::env::set_var("PATHS", "/bin/ls,/usr/local/bin,/usr/bin/ls"); - std::env::set_var("PORTS", "8080;8888;8000"); - std::env::set_var( - "NAMES", - "Mervinc Harmon|Alfreda Valenzuela|Arlen Cabrera|Damon Rice|Willie Schwartz", - ); + std::env::set_var("PORTS", "8080,8888,8000"); + std::env::set_var("NAMES", "Alex|Peter|Mary"); std::env::set_var("TIMEOUT", "100"); std::env::set_var("FORMATS", "json5,yaml;.env"); diff --git a/src/env.rs b/src/env.rs index 36c6648..4580661 100644 --- a/src/env.rs +++ b/src/env.rs @@ -11,6 +11,7 @@ use serde::de::IntoDeserializer; /// module. Gets converted into `ErrorKind::EnvDeserialization` before reaching /// the real public API. #[derive(PartialEq, Eq)] +#[doc(hidden)] pub struct DeError(pub(crate) String); impl std::error::Error for DeError {} @@ -38,6 +39,7 @@ impl serde::de::Error for DeError { /// Deserializer type. Semantically private (see `DeError`). +#[doc(hidden)] pub struct Deserializer { value: String, } @@ -167,104 +169,60 @@ mod tests { } } -/// This module contains helper methods to simplify configuration via environment variables -pub mod env_utils { +/// Functions for the `#[config(parse_env = ...)]` attribute. +pub mod parse { use std::str::FromStr; - /// Helper function to parse any type that implements [`std::str::FromStr`] - /// into a collection that implements [`std::iter::FromIterator`], spliting original string by - /// const char separator. + /// Splits the environment variable by separator `SEP`, parses each element + /// with [`FromStr`] and collects everything via [`FromIterator`]. + /// + /// To avoid having to specify the separator via `::<>` syntax, see the + /// other functions in this module. + /// + /// [`FromStr`]: std::str::FromStr + /// [`FromIterator`]: std::iter::FromIterator + /// + /// + /// # Example /// - /// # To Vec /// ``` - /// use crate::confique::Config; + /// use confique::Config; + /// /// #[derive(Debug, confique::Config)] /// struct Conf { - /// #[config( - /// env = "PORTS", - /// parse_env = confique::env_utils::to_collection_by_char_separator::<',', _, _> - /// )] - /// ports: Vec + /// #[config(env = "PORTS", parse_env = confique::env::parse::list_by_sep::<',', _, _>)] + /// ports: Vec, /// } /// /// std::env::set_var("PORTS", "8080,8000,8888"); - /// println!("{:?}", Conf::builder().env().load().unwrap()) + /// let conf = Conf::builder().env().load()?; + /// assert_eq!(conf.ports, vec![8080, 8000, 8888]); + /// # Ok::<_, confique::Error>(()) /// ``` - /// - /// # To HashSet - /// ``` - /// use crate::confique::Config; - /// #[derive(Debug, confique::Config)] - /// struct Conf { - /// #[config( - /// env = "PATHS", - /// parse_env = confique::env_utils::to_collection_by_char_separator::<';', _, _> - /// )] - /// paths: std::collections::HashSet, - /// } - /// - /// std::env::set_var("PATHS", "/bin;/user/bin;/home/user/.cargo/bin"); - /// println!("{:?}", Conf::builder().env().load().unwrap()) - /// ``` - pub fn to_collection_by_char_separator< - const SEPARATOR: char, + pub fn list_by_sep(input: &str) -> Result::Err> + where T: FromStr, - C: FromIterator::Err>>, - >( - input: &str, - ) -> C { - input.split(SEPARATOR.to_owned()).map(T::from_str).collect() + C: FromIterator, + { + input.split(SEP).map(T::from_str).collect() } + macro_rules! specify_fn_wrapper { - ($symbol_name:ident, $symbol:tt) => { - ::paste::paste! { - /// Helper function to parse any type that implements [`std::str::FromStr`] - /// into a collection that implements [`std::iter::FromIterator`], spliting original string by - #[doc = stringify!($symbol_name)] - /// . - /// - /// # To Vec - /// ``` - /// use crate::confique::Config; - /// #[derive(Debug, confique::Config)] - /// struct Conf { - /// #[config( - /// env = "PORTS", - #[doc = concat!(" parse_env = confique::env_utils::", stringify!([],))] - /// )] - /// ports: Vec - /// } - /// - #[doc = concat!("std::env::set_var(\"PORTS\", \"8080", $symbol, "8000", $symbol, "8888", "\");")] - /// println!("{:#?}", Conf::builder().env().load().unwrap()) - /// ``` - /// - /// # To HashSet - /// ``` - /// use crate::confique::Config; - /// #[derive(Debug, confique::Config)] - /// struct Conf { - /// #[config( - /// env = "PATHS", - #[doc = concat!(" parse_env = confique::env_utils::", stringify!([],))] - /// )] - /// paths: std::collections::HashSet, - /// } - /// - #[doc = concat!("std::env::set_var(\"PATHS\", \"/bin", $symbol, "/user/bin", $symbol, "/home/user/.cargo/bin", "\");")] - /// println!("{:#?}", Conf::builder().env().load().unwrap()) - /// ``` - pub fn []::Err>>>( - input: &str, - ) -> C { - to_collection_by_char_separator::<$symbol, _, _>(input) - } + ($fn_name:ident, $sep:literal) => { + #[doc = concat!("Like [`list_by_sep`] with `", $sep, "` separator.")] + pub fn $fn_name(input: &str) -> Result::Err> + where + T: FromStr, + C: FromIterator, + { + list_by_sep::<$sep, _, _>(input) } } } - specify_fn_wrapper!(comma, ','); - specify_fn_wrapper!(semicolon, ';'); - specify_fn_wrapper!(space, ' '); + specify_fn_wrapper!(list_by_comma, ','); + specify_fn_wrapper!(list_by_semicolon, ';'); + specify_fn_wrapper!(list_by_colon, ':'); + specify_fn_wrapper!(list_by_space, ' '); } diff --git a/src/lib.rs b/src/lib.rs index 31cdce9..f09873d 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -175,8 +175,7 @@ use serde::Deserialize; pub mod internal; mod builder; -mod env; -pub use env::env_utils; +pub mod env; mod error; pub mod meta; From 9d57b048ce24ba840ea386426055148b140eaa3b Mon Sep 17 00:00:00 2001 From: Lukas Kalbertodt Date: Sun, 6 Nov 2022 12:24:10 +0100 Subject: [PATCH 5/5] Improve documentation --- CHANGELOG.md | 3 ++- src/lib.rs | 11 +++++++---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c352fb5..1b37b01 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,8 @@ All notable changes to this project will be documented in this file. ## [Unreleased] ### Added -- `parse_env` attribute which is using for parse env variable in arbitrary type +- `parse_env` attribute for custom parsing of environment variables (allows you + to load lists and other complex objects from env vars). ## [0.2.0] - 2022-10-21 ### Added diff --git a/src/lib.rs b/src/lib.rs index f09873d..da44c22 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -320,11 +320,14 @@ pub use crate::{ /// - **`#[config(deserialize_with = path::to::function)]`**: like /// [serde's `deserialize_with` attribute][serde-deser]. /// -/// [serde-deser]: https://serde.rs/field-attrs.html#deserialize_with +/// - **`#[config(parse_env = path::to::function)]`**: function used to parse +/// environment variables. Mostly useful if you need to parse lists or +/// other complex objects from env vars. Function needs signature +/// `fn(&str) -> Result` where `T` is the type of +/// the field.. Can only be present if the `env` attribute is present. Also +/// see [`env::parse`]. /// -/// - **`#[config(from_env = path::to::function, env = "KEY")]`**: like -/// deserialize_with` attribute, but only for fields from an environment variable. -/// Can only be present if the `env` attribute is present +/// [serde-deser]: https://serde.rs/field-attrs.html#deserialize_with /// /// ## Special types for leaf fields ///