Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 10 additions & 1 deletion crates/turborepo-scm/src/ls_tree.rs
Original file line number Diff line number Diff line change
Expand Up @@ -180,9 +180,18 @@ impl GitRepo {
Ok(state.paths)
}

pub(crate) fn git_ls_files_cached_with_unsupported(&self) -> Result<GitPathList, Error> {
self.git_ls_files(&["--cached"])
}

pub(crate) fn git_ls_files_untracked_with_unsupported(&self) -> Result<GitPathList, Error> {
self.git_ls_files(&["--others", "--exclude-standard"])
}

fn git_ls_files(&self, args: &[&str]) -> Result<GitPathList, Error> {
let mut git = Command::new(self.bin.as_std_path())
.args(["ls-files", "--others", "--exclude-standard", "-z"])
.args(["ls-files", "-z"])
.args(args)
.env("GIT_OPTIONAL_LOCKS", "0")
.current_dir(&self.root)
.stdout(Stdio::piped())
Expand Down
148 changes: 148 additions & 0 deletions crates/turborepo-scm/src/package_deps.rs
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,83 @@ impl SCM {
}
}

/// Hash tracked files selected by task outputs, without feeding generated
/// (untracked) outputs back into the task's cache key. Paths are returned
/// relative to the package, just like `get_package_file_hashes`.
pub fn get_tracked_output_hashes(
&self,
turbo_root: &AbsoluteSystemPath,
package_path: &AnchoredSystemPath,
inclusions: &[String],
exclusions: &[String],
repo_index: Option<&RepoGitIndex>,
) -> Result<GitHashes, Error> {
let SCM::Git(git) = self else {
return Ok(GitHashes::new());
};
if inclusions.is_empty() {
return Ok(GitHashes::new());
}
let package_unix_path = package_path.to_unix();
let resolve = |glob: &String| {
let glob = if package_unix_path.as_str().is_empty() {
glob.clone()
} else {
format!("{package_unix_path}/{glob}")
};
glob.parse::<ValidatedGlob>()
};
let inclusions = inclusions
.iter()
.map(resolve)
.collect::<Result<Vec<_>, _>>()?;
let exclusions = exclusions
.iter()
.map(resolve)
.collect::<Result<Vec<_>, _>>()?;
let files = globwalk::globwalk(
turbo_root,
&inclusions,
&exclusions,
globwalk::WalkType::Files,
)?;
if files.is_empty() {
return Ok(GitHashes::new());
}
let tracked = if repo_index.is_none() {
// The output walk already validated selected paths. Unsupported
// tracked paths elsewhere in the repository must not fail this query.
git.git_ls_files_cached_with_unsupported()?
.paths
.into_iter()
.collect()
} else {
std::collections::HashSet::new()
};
let mut candidates = Vec::new();
for file in files {
let path = git.root.anchor(&file)?.to_unix();
let is_tracked = match repo_index {
Some(index) => index.is_tracked(&path),
None => tracked.contains(&path),
};
if is_tracked {
candidates.push(path);
}
}
let full_pkg_path = turbo_root.resolve(package_path);
let pkg_prefix = git.root.anchor(&full_pkg_path)?.to_unix();
let mut hashes = GitHashes::new();
git.extend_hashes_from_candidates(
&full_pkg_path,
&pkg_prefix,
repo_index,
candidates,
&mut hashes,
)?;
Ok(hashes)
}

pub fn hash_files(
&self,
turbo_root: &AbsoluteSystemPath,
Expand Down Expand Up @@ -599,6 +676,77 @@ mod tests {
}
}

#[test]
fn test_tracked_outputs_with_nested_turbo_root_and_no_git_index() {
let (_tmp, root) = tmp_dir();
setup_repository(&root);
let turbo_root = root.join_component("project");
let package_path = AnchoredSystemPath::new("packages/app").unwrap();
let license = turbo_root.join_components(&["packages", "app", "dist", "LICENSE"]);
license.ensure_dir().unwrap();
license.create_with_contents("license v1").unwrap();
let inclusions = vec!["dist/**".to_owned()];
let scm = SCM::new(&turbo_root);
// An unborn repository may have no index yet; generated outputs are not inputs.
assert!(
scm.get_tracked_output_hashes(&turbo_root, package_path, &inclusions, &[], None)
.unwrap()
.is_empty()
);
assert!(
SCM::Manual
.get_tracked_output_hashes(&turbo_root, package_path, &inclusions, &[], None)
.unwrap()
.is_empty()
);
require_git_cmd(&root, &["add", "."]);
for indexed in [false, true] {
let index = indexed.then(|| scm.build_repo_index_eager().unwrap());
let hashes = scm
.get_tracked_output_hashes(
&turbo_root,
package_path,
&inclusions,
&[],
index.as_ref(),
)
.unwrap();
assert_eq!(hashes.len(), 1);
assert!(hashes.contains_key(&RelativeUnixPathBuf::new("dist/LICENSE").unwrap()));
}
}

#[cfg(unix)]
#[test]
fn test_tracked_outputs_ignore_unrelated_non_utf8_paths() {
use std::{ffi::OsString, os::unix::ffi::OsStringExt};

let (_tmp, root) = tmp_dir();
setup_repository(&root);
let license = root.join_components(&["dist", "LICENSE"]);
license.ensure_dir().unwrap();
license.create_with_contents("license").unwrap();
std::fs::write(
root.as_std_path()
.join(OsString::from_vec(b"unrelated-\xff".to_vec())),
"unrelated",
)
.unwrap();
commit_all(&root);
let scm = SCM::new(&root);
let index = scm.build_repo_index_eager().unwrap();
let package = AnchoredSystemPath::new("").unwrap();
let outputs = vec!["dist/**".to_owned()];
let fallback = scm
.get_tracked_output_hashes(&root, package, &outputs, &[], None)
.unwrap();
let indexed = scm
.get_tracked_output_hashes(&root, package, &outputs, &[], Some(&index))
.unwrap();
assert_eq!(fallback, indexed);
assert_eq!(fallback.len(), 1);
}

#[test]
fn test_hash_symlink() {
let (_, tmp_root) = tmp_dir();
Expand Down
13 changes: 13 additions & 0 deletions crates/turborepo-scm/src/repo_index.rs
Original file line number Diff line number Diff line change
Expand Up @@ -706,6 +706,19 @@ impl RepoGitIndex {
Ok((hashes, to_hash))
}

pub(crate) fn is_tracked(&self, path: &RelativeUnixPathBuf) -> bool {
if let Ok(idx) = self
.status_entries
.binary_search_by(|entry| entry.path.cmp(path))
{
let entry = &self.status_entries[idx];
return !entry.is_untracked && !entry.is_delete;
}
self.ls_tree_hashes
.binary_search_by(|(entry_path, _)| entry_path.cmp(path))
.is_ok()
}

/// Partition a set of existing git-root-relative file paths into:
/// - clean tracked files whose blob OIDs can be reused immediately
/// - files that still need content hashing because they are dirty,
Expand Down
68 changes: 49 additions & 19 deletions crates/turborepo-task-hash/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ use turborepo_task_id::TaskId;
use turborepo_telemetry::events::task::PackageTaskEventBuilder;
use turborepo_types::{
EnvMode, HashTrackerCacheHitMetadata, HashTrackerDetailedMap, HashTrackerInfo, RunOptsHashInfo,
TaskCommandOverride, TaskDefinitionHashInfo, TaskInputs,
TaskCommandOverride, TaskDefinitionHashInfo, TaskInputs, TaskOutputs,
};

fn env_var_names_for_debug_log(env_vars: &EnvironmentVariableMap) -> Vec<String> {
Expand Down Expand Up @@ -160,13 +160,14 @@ impl PackageInputsHashes {
}
};

// Phase 1: Collect task metadata and group by (package_path, inputs) for dedup.
// Multiple tasks in the same package with identical inputs produce the same
// file hashes — no need to globwalk and hash the same files repeatedly.
// Phase 1: Group by package, inputs, and outputs: tracked outputs are
// implicit inputs, so different output selections can produce different
// file hashes even when the configured inputs are identical.
struct TaskInfo<'b> {
task_id: TaskId<'static>,
package_path: &'b AnchoredSystemPath,
inputs: &'b TaskInputs,
outputs: &'b TaskOutputs,
}

let collect_span = tracing::info_span!("collect_task_hash_keys").entered();
Expand All @@ -188,11 +189,19 @@ impl PackageInputsHashes {
task_id: task_id.clone(),
package_path,
inputs,
outputs: task_definition.outputs(),
});
}

// Build dedup key: (package_path_str, globs, default, eager)
type HashKey = (AnchoredSystemPathBuf, Vec<String>, bool, bool);
// Include both output inclusions and exclusions in the dedup key.
type HashKey = (
AnchoredSystemPathBuf,
Vec<String>,
bool,
bool,
Vec<String>,
Vec<String>,
);
let mut unique_keys: Vec<HashKey> = Vec::new();
let mut key_indices: HashMap<HashKey, usize> = HashMap::new();
let mut task_key_map: Vec<usize> = Vec::with_capacity(task_infos.len());
Expand All @@ -203,6 +212,8 @@ impl PackageInputsHashes {
info.inputs.globs.clone(),
info.inputs.default,
info.inputs.eager,
info.outputs.inclusions.clone(),
info.outputs.exclusions.clone(),
);
let idx = match key_indices.entry(key) {
std::collections::hash_map::Entry::Occupied(e) => *e.get(),
Expand Down Expand Up @@ -233,22 +244,39 @@ impl PackageInputsHashes {
let hash_span = tracing::info_span!("hash_unique_inputs").entered();
let file_hash_results: Vec<Result<(Arc<FileHashes>, String), Error>> = unique_keys
.into_par_iter()
.map(|(package_path, globs, default, eager)| {
let file_hashes = if !eager {
Arc::new(FileHashes(Vec::new()))
} else {
file_hashes_for_inputs(
scm,
.map(
|(package_path, globs, default, eager, inclusions, exclusions)| {
let mut file_hashes = if !eager {
Arc::new(FileHashes(Vec::new()))
} else {
file_hashes_for_inputs(
scm,
repo_root,
&package_path,
&globs,
default,
repo_index,
)?
};
// Tracked outputs are startup inputs even for JIT-only tasks.
// Deferred hashing carries these forward through expanded_hashes.
let tracked_outputs = scm.get_tracked_output_hashes(
repo_root,
&package_path,
&globs,
default,
&inclusions,
&exclusions,
repo_index,
)?
};
let hash = file_hashes.as_ref().hash();
Ok((file_hashes, hash))
})
)?;
if !tracked_outputs.is_empty() {
file_hashes = combine_file_hashes(
&file_hashes,
&FileHashes(tracked_outputs.into_iter().collect()),
);
}
let hash = file_hashes.as_ref().hash();
Ok((file_hashes, hash))
},
)
.collect();

let file_hash_results: Vec<(Arc<FileHashes>, String)> = file_hash_results
Expand Down Expand Up @@ -1147,6 +1175,8 @@ mod test {
mod jit_inputs;
#[path = "structured_inputs_test.rs"]
mod structured_inputs;
#[path = "tracked_outputs_test.rs"]
mod tracked_outputs;

struct TestRunOpts {
single_package: bool,
Expand Down
Loading
Loading