Fix upload to skip hidden files/dirs in submodules

Previously, the upload command only skipped files starting with "."
which correctly skipped root hidden files (.git/, .datalad/, etc.) but
not hidden files/dirs in submodules (e.g., derivatives/.git/annex/objects/
or derivatives/.datalad/config).

This could cause the CLI to upload duplicate annex objects or internal
datalad files when uploading datasets containing submodules.

The fix adds a check for any hidden path component (starting with .)
anywhere in the relative path, while still correctly following symlinks
that point to .git/annex/objects/.

Also adds unit tests for the new shouldIncludeFile() function.

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>

Author: mvdoc

cli/src/commands/upload.test.ts

@@ -0,0 +1,68 @@
+import { assertEquals } from "@std/assert/equals"
+import { hiddenPathPattern, shouldIncludeFile } from "./upload.ts"
+
+Deno.test("hiddenPathPattern matches hidden files/dirs in subpaths", () => {
+  assertEquals(hiddenPathPattern.test("derivatives/.git/config"), true)
+  assertEquals(hiddenPathPattern.test("derivatives/.git/annex/objects/xx/yy/key"), true)
+  assertEquals(hiddenPathPattern.test("derivatives/.datalad/config"), true)
+  assertEquals(hiddenPathPattern.test("a/.git/b"), true)
+  assertEquals(hiddenPathPattern.test("sub/.gitignore"), true)
+  // Should not match dotfiles at root (no leading slash)
+  assertEquals(hiddenPathPattern.test(".git/config"), false)
+  assertEquals(hiddenPathPattern.test(".bidsignore"), false)
+  // Should not match files that just contain a dot in name
+  assertEquals(hiddenPathPattern.test("my.gitconfig"), false)
+  assertEquals(hiddenPathPattern.test("file.nii.gz"), false)
+})
+
+Deno.test("shouldIncludeFile() includes regular dataset files", () => {
+  assertEquals(shouldIncludeFile("dataset_description.json"), true)
+  assertEquals(shouldIncludeFile("sub-01/anat/sub-01_T1w.nii.gz"), true)
+  assertEquals(shouldIncludeFile("derivatives/sub-01/func/bold.nii.gz"), true)
+})
+
+Deno.test("shouldIncludeFile() includes .bidsignore", () => {
+  assertEquals(shouldIncludeFile(".bidsignore"), true)
+})
+
+Deno.test("shouldIncludeFile() skips root hidden files and directories", () => {
+  assertEquals(shouldIncludeFile(".git/config"), false)
+  assertEquals(shouldIncludeFile(".git/annex/objects/abc/def/SHA256--xyz"), false)
+  assertEquals(shouldIncludeFile(".datalad/config"), false)
+  assertEquals(shouldIncludeFile(".gitattributes"), false)
+})
+
+Deno.test("shouldIncludeFile() skips hidden files/dirs in submodules", () => {
+  // .git directories
+  assertEquals(
+    shouldIncludeFile("derivatives/.git/annex/objects/abc/def/SHA256--xyz"),
+    false,
+  )
+  assertEquals(shouldIncludeFile("derivatives/.git/config"), false)
+  assertEquals(shouldIncludeFile("sourcedata/.git/HEAD"), false)
+  assertEquals(
+    shouldIncludeFile("derivatives/preprocessing/.git/annex/objects/xx/yy/key"),
+    false,
+  )
+  // Deeper nested paths
+  assertEquals(
+    shouldIncludeFile("derivatives/qa/.git/annex/objects/Xk/Mx/SHA256--abc"),
+    false,
+  )
+  assertEquals(
+    shouldIncludeFile("derivatives/fmriprep/sub-01/.datalad/config"),
+    false,
+  )
+  // .datalad directories
+  assertEquals(shouldIncludeFile("derivatives/.datalad/config"), false)
+  assertEquals(shouldIncludeFile("sourcedata/.datalad/metadata/aggregate_v1.json"), false)
+  // Other hidden files in submodules
+  assertEquals(shouldIncludeFile("derivatives/.gitattributes"), false)
+  assertEquals(shouldIncludeFile("sourcedata/.gitignore"), false)
+})
+
+Deno.test("shouldIncludeFile() includes symlinks in submodules", () => {
+  // Symlinks themselves don't have .git in their path
+  // They point to .git/annex/objects but the path we check is the symlink location
+  assertEquals(shouldIncludeFile("derivatives/sub-01/anat/sub-01_T1w.nii.gz"), true)
+})

cli/src/commands/upload.ts

@@ -13,6 +13,26 @@ import { createDatasetAffirmed } from "./create-dataset.ts"
 import { CreateDatasetAffirmedError } from "../error.ts"
 import validatorConfig from "../validator-config.json" with { type: "json" }
 
+/**
+ * Check if a path contains a hidden directory or file (starting with .)
+ * Matches paths like "derivatives/.git/..." or "derivatives/.datalad/..." in submodules
+ */
+export const hiddenPathPattern = /[\/\\]\./
+
+/**
+ * Determine if a file should be included in the upload based on its relative path
+ * @param relativePath Path relative to the dataset root
+ * @returns true if the file should be uploaded
+ */
+export function shouldIncludeFile(relativePath: string): boolean {
+  // Skip hidden files/dirs in submodules (e.g., derivatives/.git/, derivatives/.datalad/)
+  const hasHiddenComponent = hiddenPathPattern.test(relativePath)
+  return (
+    relativePath === ".bidsignore" ||
+    (!relativePath.startsWith(".") && !hasHiddenComponent)
+  )
+}
+
 async function getRepoDir(url: URL): Promise<string> {
   const LOCAL_STORAGE_KEY = `openneuro_cli_${url.hostname}_`
   const repoDir = localStorage.getItem(LOCAL_STORAGE_KEY)
@@ -42,10 +62,7 @@ export async function addGitFiles(
     })
   ) {
     const relativePath = relative(dataset_directory_abs, walkEntry.path)
-    if (
-      relativePath === ".bidsignore" ||
-      !relativePath.startsWith(".")
-    ) {
+    if (shouldIncludeFile(relativePath)) {
       worker.postMessage({
         "command": "add",
         "path": walkEntry.path,