Thanks to visit codestin.com
Credit goes to github.com

Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

25 changes: 25 additions & 0 deletions rust/boil/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -31,3 +31,28 @@ boil image list
# Display a list of versions of the image located in the 'airflow' folder
boil image list airflow
```

## Advanced Building Options

### Use Remote Cache

> [!NOTE]
> The default builder (with the `docker` driver) doesn't support the registry cache storage backend. You must create
> a new builder using the `docker-container` driver and either set this new builder as the default or pass
> `-- --builder <NAME>` to use it:
>
> ```shell
> docker builder create --name container --driver=docker-container
> boil build airflow --cache-registry oci.example.org -- --builder container
> ```

boil offers to option to automatically pull from and push to a remote cache. This feature can be
enabled by using the `--cache-registry` (and the optional `--cache-namespace`) argument:

```shell
# This will use `oci.example.org/<NAMESPACE>-cache/airflow` to store and retrieve cached layers
boil build airflow --cache-registry oci.example.org

# This will use `oci.example.org/foo/airflow` to store and retrieve cached layers
boil build airflow --cache-registry oci.example.org --cache-namespace foo
Comment thread
NickLarsenNZ marked this conversation as resolved.
```
6 changes: 6 additions & 0 deletions rust/boil/src/cli/build.rs
Original file line number Diff line number Diff line change
Expand Up @@ -118,6 +118,12 @@ pub struct BuildArguments {
#[deprecated(since = "0.1.7", note = "Use -- --load instead")]
pub load: bool,

#[arg(long, help_heading = "Build Options", group = "cache")]
pub cache_registry: Option<HostPort>,

#[arg(long, help_heading = "Build Options", requires = "cache")]
pub cache_namespace: Option<String>,

/// Dry run. This does not build the image(s) but instead prints out the bakefile.
#[arg(short, long, alias = "dry")]
pub dry_run: bool,
Expand Down
118 changes: 107 additions & 11 deletions rust/boil/src/core/bakefile.rs
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
use std::{
collections::{BTreeMap, btree_map::Entry},
fmt::Debug,
fmt::{Debug, Display},
ops::{Deref, DerefMut},
path::PathBuf,
};
Expand Down Expand Up @@ -340,26 +340,45 @@ impl Bakefile {

for (image_name, (image_config, is_entry)) in targets.into_iter() {
for (image_version, image_options) in image_config.versions {
let image_repository_uri = utils::format_image_repository_uri(
let (
image_repository_uri,
image_index_manifest_tag,
image_manifest_tag,
image_manifest_uri,
) = utils::format_image_tag_parts(
image_registry,
&cli_args.registry_namespace,
&image_name,
);

let image_index_manifest_tag = utils::format_image_index_manifest_tag(
&image_version,
&metadata.vendor_tag_prefix,
&cli_args.image_version,
);

let image_manifest_tag = utils::format_image_manifest_tag(
&image_index_manifest_tag,
cli_args.target_platform.architecture(),
cli_args.strip_architecture,
);

let image_manifest_uri =
utils::format_image_manifest_uri(&image_repository_uri, &image_manifest_tag);
let cache_image_manifest_uri =
cli_args.cache_registry.as_ref().map(|cache_registry| {
let uri = utils::format_image_cache_repository_uri(
cache_registry,
cli_args.cache_namespace.as_deref(),
&cli_args.registry_namespace,
&image_name,
);

let cache_image_index_manifest_tag = utils::format_image_index_manifest_tag(
&image_version,
&metadata.vendor_tag_prefix,
&cli_args.image_version,
);

let cache_image_manifest_tag = utils::format_image_manifest_tag(
&cache_image_index_manifest_tag,
cli_args.target_platform.architecture(),
cli_args.strip_architecture,
);

utils::format_image_manifest_uri(&uri, &cache_image_manifest_tag)
});

// TODO (@Techassi): Clean this up
// TODO (@Techassi): Move the arg formatting into functions
Expand Down Expand Up @@ -449,6 +468,23 @@ impl Bakefile {
&cli_args.image_version,
);

let cache_to =
cache_image_manifest_uri
.clone()
.map_or_else(Vec::new, |reference| {
vec![CacheStorageBackend::Registry {
reference,
mode: Some(CacheMode::Max),
}]
});

let cache_from = cache_image_manifest_uri.map_or_else(Vec::new, |reference| {
vec![CacheStorageBackend::Registry {
reference,
mode: None,
}]
});

let target = BakefileTarget {
tags: vec![image_manifest_uri],
arguments: build_arguments,
Expand All @@ -459,6 +495,8 @@ impl Bakefile {
inherits: vec![COMMON_TARGET_NAME.to_owned()],
annotations,
contexts,
cache_from,
cache_to,
..Default::default()
};

Expand Down Expand Up @@ -529,6 +567,7 @@ impl Bakefile {
// TODO (@Techassi): Figure out of we can use borrowed data in here. This would avoid a whole bunch
// of cloning.
#[derive(Debug, Default, Serialize)]
#[serde(rename_all = "kebab-case")]
pub struct BakefileTarget {
/// Defines build arguments for the target.
#[serde(
Expand Down Expand Up @@ -577,6 +616,12 @@ pub struct BakefileTarget {
/// Image names and tags to use for the build target.
#[serde(skip_serializing_if = "Vec::is_empty")]
pub tags: Vec<String>,

#[serde(skip_serializing_if = "Vec::is_empty")]
pub cache_from: Vec<CacheStorageBackend>,

#[serde(skip_serializing_if = "Vec::is_empty")]
pub cache_to: Vec<CacheStorageBackend>,
}

impl BakefileTarget {
Expand Down Expand Up @@ -668,3 +713,54 @@ impl BakefileTarget {
pub struct BakefileGroup {
targets: Vec<String>,
}

#[derive(Clone, Debug)]
pub enum CacheStorageBackend {
Registry {
/// Full name of the cache image to import.
reference: String,

/// Specifies which layers to cache
mode: Option<CacheMode>,
},
}

impl Display for CacheStorageBackend {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
CacheStorageBackend::Registry { reference, mode } => {
f.write_str("type=registry")?;
f.write_fmt(format_args!(",ref={reference}"))?;

if let Some(mode) = mode {
f.write_fmt(format_args!(",mode={mode}"))?;
}

Ok(())
}
}
}
}

impl Serialize for CacheStorageBackend {
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
where
S: serde::Serializer,
{
serializer.serialize_str(&self.to_string())
}
}

#[derive(Clone, Copy, Debug)]
pub enum CacheMode {
// We currently only support max, because we want to cache every layer
Max,
}

impl Display for CacheMode {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
CacheMode::Max => write!(f, "max"),
}
}
}
47 changes: 47 additions & 0 deletions rust/boil/src/utils.rs
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,24 @@ pub fn format_image_repository_uri(
format!("{image_registry}/{registry_namespace}/{image_name}")
}

pub fn format_image_cache_repository_uri(
image_cache_registry: &HostPort,
cache_registry_namespace: Option<&str>,
registry_namespace: &str,
image_name: &str,
) -> String {
// We don't use .map here because we are unable to borrow the formatted string long enough
if let Some(cache_registry_namespace) = cache_registry_namespace {
format_image_repository_uri(image_cache_registry, cache_registry_namespace, image_name)
} else {
format_image_repository_uri(
image_cache_registry,
&format!("{}-cache", registry_namespace),
image_name,
)
}
}

/// Formats and returns the image manifest URI, eg. `oci.stackable.tech/sdp/opa:1.4.2-stackable25.7.0-amd64`.
pub fn format_image_manifest_uri(image_repository_uri: &str, image_manifest_tag: &str) -> String {
format!("{image_repository_uri}:{image_manifest_tag}")
Expand Down Expand Up @@ -41,6 +59,35 @@ pub fn format_image_manifest_tag(
}
}

// TODO (@Techassi): Can we design this better? Maybe add a new struct/type which implements a bunch
// of associated functions.
#[allow(clippy::too_many_arguments)]
pub fn format_image_tag_parts(
image_registry: &HostPort,
registry_namespace: &str,
image_name: &str,
image_version: &str,
vendor_tag_prefix: &str,
vendor_image_version: &str,
architecture: &Architecture,
strip_architecture: bool,
) -> (String, String, String, String) {
let image_repository_uri =
format_image_repository_uri(image_registry, registry_namespace, image_name);
let image_index_manifest_tag =
format_image_index_manifest_tag(image_version, vendor_tag_prefix, vendor_image_version);
let image_manifest_tag =
format_image_manifest_tag(&image_index_manifest_tag, architecture, strip_architecture);
let image_manifest_uri = format_image_manifest_uri(&image_repository_uri, &image_manifest_tag);

(
image_repository_uri,
image_index_manifest_tag,
image_manifest_tag,
image_manifest_uri,
)
}

/// Formats and returns the registry-specific env var name, eg. `BOIL_REGISTRY_TOKEN_OCI_STACKABLE_TECH`.
pub fn format_registry_token_env_var_name(registry_uri: &str) -> String {
format!(
Expand Down
Loading