Merge pull request #3767 from github/henrymercer/overlay-reduce-minimum-git-version

Reduce the minimum Git version required for overlay

CHANGELOG.md

@@ -4,7 +4,7 @@ See the [releases page](https://github.com/github/codeql-action/releases) for th
 
 ## [UNRELEASED]
 
-No user facing changes.
+- Reduced the minimum Git version required for [improved incremental analysis](https://github.com/github/roadmap/issues/1158) from 2.38.0 to 2.11.0. [#3767](https://github.com/github/codeql-action/pull/3767)
 
 ## 4.34.1 - 20 Mar 2026
 

lib/analyze-action-post.js

@@ -162158,11 +162158,11 @@ var decodeGitFilePath = function(filePath) {
 var getFileOidsUnderPath = async function(basePath) {
   const stdout = await runGitCommand(
     basePath,
-    ["ls-files", "--recurse-submodules", "--format=%(objectname)_%(path)"],
+    ["ls-files", "--recurse-submodules", "--stage"],
     "Cannot list Git OIDs of tracked files."
   );
   const fileOidMap = {};
-  const regex = /^([0-9a-f]{40})_(.+)$/;
+  const regex = /^[0-9]+ ([0-9a-f]{40}) [0-9]+\t(.+)$/;
   for (const line of stdout.split("\n")) {
     if (line) {
       const match = line.match(regex);

lib/analyze-action.js

@@ -107778,11 +107778,11 @@ var decodeGitFilePath = function(filePath) {
 var getFileOidsUnderPath = async function(basePath) {
   const stdout = await runGitCommand(
     basePath,
-    ["ls-files", "--recurse-submodules", "--format=%(objectname)_%(path)"],
+    ["ls-files", "--recurse-submodules", "--stage"],
     "Cannot list Git OIDs of tracked files."
   );
   const fileOidMap = {};
-  const regex = /^([0-9a-f]{40})_(.+)$/;
+  const regex = /^[0-9]+ ([0-9a-f]{40}) [0-9]+\t(.+)$/;
   for (const line of stdout.split("\n")) {
     if (line) {
       const match = line.match(regex);

lib/autobuild-action.js

@@ -104212,11 +104212,11 @@ var decodeGitFilePath = function(filePath) {
 var getFileOidsUnderPath = async function(basePath) {
   const stdout = await runGitCommand(
     basePath,
-    ["ls-files", "--recurse-submodules", "--format=%(objectname)_%(path)"],
+    ["ls-files", "--recurse-submodules", "--stage"],
     "Cannot list Git OIDs of tracked files."
   );
   const fileOidMap = {};
-  const regex = /^([0-9a-f]{40})_(.+)$/;
+  const regex = /^[0-9]+ ([0-9a-f]{40}) [0-9]+\t(.+)$/;
   for (const line of stdout.split("\n")) {
     if (line) {
       const match = line.match(regex);

lib/init-action-post.js

@@ -165672,11 +165672,11 @@ var decodeGitFilePath = function(filePath) {
 var getFileOidsUnderPath = async function(basePath) {
   const stdout = await runGitCommand(
     basePath,
-    ["ls-files", "--recurse-submodules", "--format=%(objectname)_%(path)"],
+    ["ls-files", "--recurse-submodules", "--stage"],
     "Cannot list Git OIDs of tracked files."
   );
   const fileOidMap = {};
-  const regex = /^([0-9a-f]{40})_(.+)$/;
+  const regex = /^[0-9]+ ([0-9a-f]{40}) [0-9]+\t(.+)$/;
   for (const line of stdout.split("\n")) {
     if (line) {
       const match = line.match(regex);

lib/init-action.js

@@ -105196,7 +105196,7 @@ var core8 = __toESM(require_core());
 var toolrunner2 = __toESM(require_toolrunner());
 var io3 = __toESM(require_io());
 var semver3 = __toESM(require_semver2());
-var GIT_MINIMUM_VERSION_FOR_OVERLAY = "2.38.0";
+var GIT_MINIMUM_VERSION_FOR_OVERLAY = "2.11.0";
 var GitVersionInfo = class {
   constructor(truncatedVersion, fullVersion) {
     this.truncatedVersion = truncatedVersion;
@@ -105306,11 +105306,11 @@ var getGitRoot = async function(sourceRoot) {
 var getFileOidsUnderPath = async function(basePath) {
   const stdout = await runGitCommand(
     basePath,
-    ["ls-files", "--recurse-submodules", "--format=%(objectname)_%(path)"],
+    ["ls-files", "--recurse-submodules", "--stage"],
     "Cannot list Git OIDs of tracked files."
   );
   const fileOidMap = {};
-  const regex = /^([0-9a-f]{40})_(.+)$/;
+  const regex = /^[0-9]+ ([0-9a-f]{40}) [0-9]+\t(.+)$/;
   for (const line of stdout.split("\n")) {
     if (line) {
       const match = line.match(regex);

lib/resolve-environment-action.js

@@ -104205,11 +104205,11 @@ var decodeGitFilePath = function(filePath) {
 var getFileOidsUnderPath = async function(basePath) {
   const stdout = await runGitCommand(
     basePath,
-    ["ls-files", "--recurse-submodules", "--format=%(objectname)_%(path)"],
+    ["ls-files", "--recurse-submodules", "--stage"],
     "Cannot list Git OIDs of tracked files."
   );
   const fileOidMap = {};
-  const regex = /^([0-9a-f]{40})_(.+)$/;
+  const regex = /^[0-9]+ ([0-9a-f]{40}) [0-9]+\t(.+)$/;
   for (const line of stdout.split("\n")) {
     if (line) {
       const match = line.match(regex);

lib/setup-codeql-action.js

@@ -104072,11 +104072,11 @@ var decodeGitFilePath = function(filePath) {
 var getFileOidsUnderPath = async function(basePath) {
   const stdout = await runGitCommand(
     basePath,
-    ["ls-files", "--recurse-submodules", "--format=%(objectname)_%(path)"],
+    ["ls-files", "--recurse-submodules", "--stage"],
     "Cannot list Git OIDs of tracked files."
   );
   const fileOidMap = {};
-  const regex = /^([0-9a-f]{40})_(.+)$/;
+  const regex = /^[0-9]+ ([0-9a-f]{40}) [0-9]+\t(.+)$/;
   for (const line of stdout.split("\n")) {
     if (line) {
       const match = line.match(regex);

lib/upload-lib.js

@@ -107363,11 +107363,11 @@ var decodeGitFilePath = function(filePath) {
 var getFileOidsUnderPath = async function(basePath) {
   const stdout = await runGitCommand(
     basePath,
-    ["ls-files", "--recurse-submodules", "--format=%(objectname)_%(path)"],
+    ["ls-files", "--recurse-submodules", "--stage"],
     "Cannot list Git OIDs of tracked files."
   );
   const fileOidMap = {};
-  const regex = /^([0-9a-f]{40})_(.+)$/;
+  const regex = /^[0-9]+ ([0-9a-f]{40}) [0-9]+\t(.+)$/;
   for (const line of stdout.split("\n")) {
     if (line) {
       const match = line.match(regex);

lib/upload-sarif-action.js

@@ -107047,11 +107047,11 @@ var decodeGitFilePath = function(filePath) {
 var getFileOidsUnderPath = async function(basePath) {
   const stdout = await runGitCommand(
     basePath,
-    ["ls-files", "--recurse-submodules", "--format=%(objectname)_%(path)"],
+    ["ls-files", "--recurse-submodules", "--stage"],
     "Cannot list Git OIDs of tracked files."
   );
   const fileOidMap = {};
-  const regex = /^([0-9a-f]{40})_(.+)$/;
+  const regex = /^[0-9]+ ([0-9a-f]{40}) [0-9]+\t(.+)$/;
   for (const line of stdout.split("\n")) {
     if (line) {
       const match = line.match(regex);

src/config-utils.test.ts

@@ -1936,7 +1936,7 @@ test.serial(
   "Fallback due to old git version",
   {
     overlayDatabaseEnvVar: "overlay",
-    gitVersion: new GitVersionInfo("2.30.0", "2.30.0"), // Version below required 2.38.0
+    gitVersion: new GitVersionInfo("2.10.0", "2.10.0"), // Version below required 2.11.0
   },
   {
     disabledReason: OverlayDisabledReason.IncompatibleGit,

src/git-utils.test.ts

@@ -347,9 +347,9 @@ test.serial("getFileOidsUnderPath returns correct file mapping", async (t) => {
   const runGitCommandStub = sinon
     .stub(gitUtils as any, "runGitCommand")
     .resolves(
-      "30d998ded095371488be3a729eb61d86ed721a18_lib/git-utils.js\n" +
-        "d89514599a9a99f22b4085766d40af7b99974827_lib/git-utils.js.map\n" +
-        "a47c11f5bfdca7661942d2c8f1b7209fb0dfdf96_src/git-utils.ts",
+      "100644 30d998ded095371488be3a729eb61d86ed721a18 0\tlib/git-utils.js\n" +
+        "100644 d89514599a9a99f22b4085766d40af7b99974827 0\tlib/git-utils.js.map\n" +
+        "100644 a47c11f5bfdca7661942d2c8f1b7209fb0dfdf96 0\tsrc/git-utils.ts",
     );
 
   const result = await gitUtils.getFileOidsUnderPath("/fake/path");
@@ -362,7 +362,7 @@ test.serial("getFileOidsUnderPath returns correct file mapping", async (t) => {
 
   t.deepEqual(runGitCommandStub.firstCall.args, [
     "/fake/path",
-    ["ls-files", "--recurse-submodules", "--format=%(objectname)_%(path)"],
+    ["ls-files", "--recurse-submodules", "--stage"],
     "Cannot list Git OIDs of tracked files.",
   ]);
 });
@@ -371,9 +371,9 @@ test.serial("getFileOidsUnderPath handles quoted paths", async (t) => {
   sinon
     .stub(gitUtils as any, "runGitCommand")
     .resolves(
-      "30d998ded095371488be3a729eb61d86ed721a18_lib/normal-file.js\n" +
-        'd89514599a9a99f22b4085766d40af7b99974827_"lib/file with spaces.js"\n' +
-        'a47c11f5bfdca7661942d2c8f1b7209fb0dfdf96_"lib/file\\twith\\ttabs.js"',
+      "100644 30d998ded095371488be3a729eb61d86ed721a18 0\tlib/normal-file.js\n" +
+        '100644 d89514599a9a99f22b4085766d40af7b99974827 0\t"lib/file with spaces.js"\n' +
+        '100644 a47c11f5bfdca7661942d2c8f1b7209fb0dfdf96 0\t"lib/file\\twith\\ttabs.js"',
     );
 
   const result = await gitUtils.getFileOidsUnderPath("/fake/path");
@@ -398,9 +398,9 @@ test.serial(
     sinon
       .stub(gitUtils as any, "runGitCommand")
       .resolves(
-        "30d998ded095371488be3a729eb61d86ed721a18_lib/git-utils.js\n" +
+        "100644 30d998ded095371488be3a729eb61d86ed721a18 0\tlib/git-utils.js\n" +
           "invalid-line-format\n" +
-          "a47c11f5bfdca7661942d2c8f1b7209fb0dfdf96_src/git-utils.ts",
+          "100644 a47c11f5bfdca7661942d2c8f1b7209fb0dfdf96 0\tsrc/git-utils.ts",
       );
 
     await t.throwsAsync(

src/git-utils.ts

@@ -14,10 +14,11 @@ import {
 import { ConfigurationError, getRequiredEnvParam } from "./util";
 
 /**
- * Minimum Git version required for overlay analysis. The `git ls-files --format`
- * option, which is used by `getFileOidsUnderPath`, was introduced in Git 2.38.0.
+ * Minimum Git version required for overlay analysis. The
+ * `git ls-files --recurse-submodules` option, which is used by
+ * `getFileOidsUnderPath`, was introduced in Git 2.11.0.
  */
-export const GIT_MINIMUM_VERSION_FOR_OVERLAY = "2.38.0";
+export const GIT_MINIMUM_VERSION_FOR_OVERLAY = "2.11.0";
 
 /**
  * Git version information
@@ -252,24 +253,28 @@ export const getGitRoot = async function (
  *
  * @param basePath A path into the Git repository.
  * @returns a map from file paths (relative to `basePath`) to Git OIDs.
- * @throws {Error} if "git ls-tree" produces unexpected output.
+ * @throws {Error} if "git ls-files" produces unexpected output.
  */
 export const getFileOidsUnderPath = async function (
   basePath: string,
 ): Promise<{ [key: string]: string }> {
   // Without the --full-name flag, the path is relative to the current working
   // directory of the git command, which is basePath.
+  //
+  // We use --stage rather than --format here because --stage has been available since Git 2.11.0,
+  // while --format was only introduced in Git 2.38.0, which would limit overlay rollout.
   const stdout = await runGitCommand(
     basePath,
-    ["ls-files", "--recurse-submodules", "--format=%(objectname)_%(path)"],
+    ["ls-files", "--recurse-submodules", "--stage"],
     "Cannot list Git OIDs of tracked files.",
   );
 
   const fileOidMap: { [key: string]: string } = {};
-  // With --format=%(objectname)_%(path), the output is a list of lines like:
-  // 30d998ded095371488be3a729eb61d86ed721a18_lib/git-utils.js
-  // d89514599a9a99f22b4085766d40af7b99974827_lib/git-utils.js.map
-  const regex = /^([0-9a-f]{40})_(.+)$/;
+  // With --stage, the output is a list of lines like:
+  // 100644 4c51bc1d9e86cd86e01b0f340cb8ce095c33b283 0\tsrc/git-utils.test.ts
+  // 100644 6b792ea543ce75d7a8a03df591e3c85311ecb64f 0\tsrc/git-utils.ts
+  // The fields are: <mode> <oid> <stage>\t<path>
+  const regex = /^[0-9]+ ([0-9a-f]{40}) [0-9]+\t(.+)$/;
   for (const line of stdout.split("\n")) {
     if (line) {
       const match = line.match(regex);