move all hash computations to hash-inputs

hash-inputs/action.yml

@@ -1,18 +1,18 @@
-name: 'CodeQL: Hash Database'
-description: 'Hashes the built database, using a stable hash'
+name: 'CodeQL: Hash query inputs'
+description: 'Hashes the query inputs, such as the database, queries and the cli itself'
 author: 'GitHub'
 inputs:
   token:
     default: ${{ github.token }}
   matrix:
     default: ${{ toJson(matrix) }}
-  mode:
+  database-mode:
     description: The hashing strategy (currently supports ".rel")
     required: false
     default: ".rel"
 outputs:
-  hash:
-    description: the stable hash of the database
+  hashes:
+    description: A per-language JSON object with hashes for the inputs
 runs:
   using: 'node12'
   main: '../lib/hash-database-action.js'

lib/hash-inputs-action.js

@@ -0,0 +1,106 @@
+"use strict";
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (Object.hasOwnProperty.call(mod, k)) result[k] = mod[k];
+    result["default"] = mod;
+    return result;
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const core = __importStar(require("@actions/core"));
+const fs = __importStar(require("fs"));
+const path = __importStar(require("path"));
+const crypto = __importStar(require("crypto"));
+const actionsUtil = __importStar(require("./actions-util"));
+const config_utils = __importStar(require("./config-utils"));
+const logging_1 = require("./logging");
+const util = __importStar(require("./util"));
+const languages_1 = require("./languages");
+async function getCodeQLHash(_config) {
+    return "DUMMY_CODEQL_HASH";
+}
+async function getQueriesHash(_language, config, logger) {
+    // Compute hash
+    const globHash = require("glob-hash");
+    const finalHash = await globHash({
+        include: [
+            // @esbena: isn't this a bit too aggressive? Could we select qlpack directories instead?
+            `${config.tempDir}/**/.cache/data/**`,
+            `${config.toolCacheDir}/**/.cache/data/**`,
+        ],
+        files: false,
+    });
+    logger.info(`queries-hash: ${finalHash}`);
+    return finalHash;
+}
+async function getDatabaseHash(language, config, logger) {
+    const dbPath = util.getCodeQLDatabasePath(config.tempDir, language);
+    let relDir = path.join(dbPath, `db-${language}`, "default");
+    let combined_all = crypto.createHash("sha256");
+    let combined_noExtractionTime = crypto.createHash("sha256");
+    let files = {};
+    let relFiles = fs
+        .readdirSync(relDir)
+        .filter((n) => n.endsWith(".rel"))
+        .map((n) => path.join(relDir, n));
+    if (relFiles.length === 0) {
+        throw new Error(`No '.rel' files found in ${relDir}. Has the 'create-database' action been called?`);
+    }
+    for (const relFile of relFiles) {
+        let content = fs.readFileSync(relFile); // XXX this ought to be chunked for large tables!
+        let solo = crypto.createHash("sha256");
+        solo.update(content);
+        files[path.relative(dbPath, relFile)] = solo.digest("hex");
+        if (language === languages_1.Language.javascript &&
+            path.basename(relFile) !== "extraction_time.rel") {
+            combined_noExtractionTime.update(content);
+        }
+        combined_all.update(content);
+    }
+    let stableHash = combined_noExtractionTime.digest("hex");
+    logger.info("database-hash:");
+    logger.info(JSON.stringify({
+        language,
+        combined: {
+            all: combined_all.digest("hex"),
+            noExtractionTime: stableHash,
+            files,
+        },
+    }, null, 2));
+    return stableHash;
+}
+async function run() {
+    const logger = logging_1.getActionsLogger();
+    try {
+        actionsUtil.prepareLocalRunEnvironment();
+        const config = await config_utils.getConfig(actionsUtil.getRequiredEnvParam("RUNNER_TEMP"), logger);
+        if (config === undefined) {
+            throw new Error("Config file could not be found at expected location. Has the 'init' action been called?");
+        }
+        let hashesByLanguage = {};
+        for (const language of config.languages) {
+            hashesByLanguage /* XXX circumvent aggressive typescript */[language] = {
+                queries: await getQueriesHash(language, config, logger),
+                database: await getDatabaseHash(language, config, logger),
+                codeql: getCodeQLHash(config),
+            };
+            core.setOutput("hashes", JSON.stringify(hashesByLanguage));
+        }
+    }
+    catch (error) {
+        core.setFailed(`We were unable to hash the database.  ${error.message}`);
+        console.log(error);
+        return;
+    }
+}
+async function runWrapper() {
+    try {
+        await run();
+    }
+    catch (error) {
+        core.setFailed(`hash-database action failed. ${error}`);
+        console.log(error);
+    }
+}
+void runWrapper();
+//# sourceMappingURL=hash-inputs-action.js.map

lib/hash-inputs-action.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"hash-inputs-action.js","sourceRoot":"","sources":["../src/hash-inputs-action.ts"],"names":[],"mappings":";;;;;;;;;AAAA,oDAAsC;AAEtC,uCAAyB;AACzB,2CAA6B;AAC7B,+CAAiC;AAEjC,4DAA8C;AAC9C,6DAA+C;AAC/C,uCAAqD;AACrD,6CAA+B;AAC/B,2CAAuC;AAEvC,KAAK,UAAU,aAAa,CAAC,OAA4B;IACvD,OAAO,mBAAmB,CAAC;AAC7B,CAAC;AAED,KAAK,UAAU,cAAc,CAC3B,SAAmB,EACnB,MAA2B,EAC3B,MAAc;IAEd,eAAe;IACf,MAAM,QAAQ,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC;IACtC,MAAM,SAAS,GAAG,MAAM,QAAQ,CAAC;QAC/B,OAAO,EAAE;YACP,wFAAwF;YACxF,GAAG,MAAM,CAAC,OAAO,oBAAoB;YACrC,GAAG,MAAM,CAAC,YAAY,oBAAoB;SAC3C;QACD,KAAK,EAAE,KAAK;KACb,CAAC,CAAC;IACH,MAAM,CAAC,IAAI,CAAC,iBAAiB,SAAS,EAAE,CAAC,CAAC;IAC1C,OAAO,SAAS,CAAC;AACnB,CAAC;AACD,KAAK,UAAU,eAAe,CAC5B,QAAkB,EAClB,MAA2B,EAC3B,MAAc;IAEd,MAAM,MAAM,GAAG,IAAI,CAAC,qBAAqB,CAAC,MAAM,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;IACpE,IAAI,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,MAAM,QAAQ,EAAE,EAAE,SAAS,CAAC,CAAC;IAC5D,IAAI,YAAY,GAAG,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;IAC/C,IAAI,yBAAyB,GAAG,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;IAC5D,IAAI,KAAK,GAEL,EAAE,CAAC;IACP,IAAI,QAAQ,GAAG,EAAE;SACd,WAAW,CAAC,MAAM,CAAC;SACnB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;SACjC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC;IACpC,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE;QACzB,MAAM,IAAI,KAAK,CACb,4BAA4B,MAAM,iDAAiD,CACpF,CAAC;KACH;IACD,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE;QAC9B,IAAI,OAAO,GAAG,EAAE,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC,iDAAiD;QACzF,IAAI,IAAI,GAAG,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;QACvC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QACrB,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAC3D,IACE,QAAQ,KAAK,oBAAQ,CAAC,UAAU;YAChC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,KAAK,qBAAqB,EAChD;YACA,yBAAyB,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;SAC3C;QACD,YAAY,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;KAC9B;IACD,IAAI,UAAU,GAAG,yBAAyB,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IACzD,MAAM,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;IAC9B,MAAM,CAAC,IAAI,CACT,IAAI,CAAC,SAAS,CACZ;QACE,QAAQ;QACR,QAAQ,EAAE;YACR,GAAG,EAAE,YAAY,CAAC,MAAM,CAAC,KAAK,CAAC;YAC/B,gBAAgB,EAAE,UAAU;YAC5B,KAAK;SACN;KACF,EACD,IAAI,EACJ,CAAC,CACF,CACF,CAAC;IACF,OAAO,UAAU,CAAC;AACpB,CAAC;AACD,KAAK,UAAU,GAAG;IAChB,MAAM,MAAM,GAAG,0BAAgB,EAAE,CAAC;IAClC,IAAI;QACF,WAAW,CAAC,0BAA0B,EAAE,CAAC;QAEzC,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC,SAAS,CACzC,WAAW,CAAC,mBAAmB,CAAC,aAAa,CAAC,EAC9C,MAAM,CACP,CAAC;QACF,IAAI,MAAM,KAAK,SAAS,EAAE;YACxB,MAAM,IAAI,KAAK,CACb,yFAAyF,CAC1F,CAAC;SACH;QACD,IAAI,gBAAgB,GAMhB,EAAE,CAAC;QACP,KAAK,MAAM,QAAQ,IAAI,MAAM,CAAC,SAAS,EAAE;YACtC,gBAAwB,CAAC,0CAA0C,CAClE,QAAQ,CACT,GAAG;gBACF,OAAO,EAAE,MAAM,cAAc,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,CAAC;gBACvD,QAAQ,EAAE,MAAM,eAAe,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,CAAC;gBACzD,MAAM,EAAE,aAAa,CAAC,MAAM,CAAC;aAC9B,CAAC;YACF,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,gBAAgB,CAAC,CAAC,CAAC;SAC5D;KACF;IAAC,OAAO,KAAK,EAAE;QACd,IAAI,CAAC,SAAS,CAAC,yCAAyC,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;QACzE,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QACnB,OAAO;KACR;AACH,CAAC;AAED,KAAK,UAAU,UAAU;IACvB,IAAI;QACF,MAAM,GAAG,EAAE,CAAC;KACb;IAAC,OAAO,KAAK,EAAE;QACd,IAAI,CAAC,SAAS,CAAC,gCAAgC,KAAK,EAAE,CAAC,CAAC;QACxD,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;KACpB;AACH,CAAC;AAED,KAAK,UAAU,EAAE,CAAC"}

lib/init-action.js

@@ -163,17 +163,6 @@ async function compileQueries(codeql, config, logger) {
             }
         }
     }
-    // Compute hash
-    const globHash = require("glob-hash");
-    const finalHash = await globHash({
-        include: [
-            `${config.tempDir}/**/.cache/data/**`,
-            `${config.toolCacheDir}/**/.cache/data/**`,
-        ],
-        files: false,
-    });
-    logger.info(`FinalHash: ${finalHash}`);
-    return finalHash;
 }
 void runWrapper();
 //# sourceMappingURL=init-action.js.map

src/hash-database-action.ts

@@ -1,90 +0,0 @@
-import * as core from "@actions/core";
-
-import * as fs from "fs";
-import * as path from "path";
-import * as crypto from "crypto";
-
-import * as actionsUtil from "./actions-util";
-import * as config_utils from "./config-utils";
-import { getActionsLogger } from "./logging";
-import * as util from "./util";
-
-async function run() {
-  const logger = getActionsLogger();
-  try {
-    actionsUtil.prepareLocalRunEnvironment();
-
-    const config = await config_utils.getConfig(
-      actionsUtil.getRequiredEnvParam("RUNNER_TEMP"),
-      logger
-    );
-    if (config === undefined) {
-      throw new Error(
-        "Config file could not be found at expected location. Has the 'init' action been called?"
-      );
-    }
-
-    for (const language of config.languages) {
-      const dbPath = util.getCodeQLDatabasePath(config.tempDir, language);
-      let relDir = path.join(dbPath, `db-${language}`, "default");
-      let combined_all = crypto.createHash("sha256");
-      let combined_noExtractionTime = crypto.createHash("sha256");
-      let files: {
-        [name: string]: string;
-      } = {};
-      let relFiles = fs
-        .readdirSync(relDir)
-        .filter((n) => n.endsWith(".rel"))
-        .map((n) => path.join(relDir, n));
-      if (relFiles.length === 0) {
-        throw new Error(
-          `No '.rel' files found in ${relDir}. Has the 'create-database' action been called?`
-        );
-      }
-      for (const relFile of relFiles) {
-        let content = fs.readFileSync(relFile); // XXX this ought to be chunked for large tables!
-        let solo = crypto.createHash("sha256");
-        solo.update(content);
-        files[path.relative(dbPath, relFile)] = solo.digest("hex");
-        if (
-          language === "javascript" &&
-          path.basename(relFile) !== "extraction_time.rel"
-        ) {
-          combined_noExtractionTime.update(content);
-        }
-        combined_all.update(content);
-      }
-      let stableHash = combined_noExtractionTime.digest("hex");
-      logger.info(
-        JSON.stringify(
-          {
-            language,
-            combined: {
-              all: combined_all.digest("hex"),
-              noExtractionTime: stableHash,
-              files,
-            },
-          },
-          null,
-          2
-        )
-      );
-      core.setOutput("hash", stableHash);
-    }
-  } catch (error) {
-    core.setFailed(`We were unable to hash the database.  ${error.message}`);
-    console.log(error);
-    return;
-  }
-}
-
-async function runWrapper() {
-  try {
-    await run();
-  } catch (error) {
-    core.setFailed(`hash-database action failed. ${error}`);
-    console.log(error);
-  }
-}
-
-void runWrapper();

src/hash-inputs-action.ts

@@ -0,0 +1,134 @@
+import * as core from "@actions/core";
+
+import * as fs from "fs";
+import * as path from "path";
+import * as crypto from "crypto";
+
+import * as actionsUtil from "./actions-util";
+import * as config_utils from "./config-utils";
+import { getActionsLogger, Logger } from "./logging";
+import * as util from "./util";
+import { Language } from "./languages";
+
+async function getCodeQLHash(_config: config_utils.Config) {
+  return "DUMMY_CODEQL_HASH";
+}
+
+async function getQueriesHash(
+  _language: Language,
+  config: config_utils.Config,
+  logger: Logger
+): Promise<string> {
+  // Compute hash
+  const globHash = require("glob-hash");
+  const finalHash = await globHash({
+    include: [
+      // @esbena: isn't this a bit too aggressive? Could we select qlpack directories instead?
+      `${config.tempDir}/**/.cache/data/**`,
+      `${config.toolCacheDir}/**/.cache/data/**`,
+    ],
+    files: false, // MG: List matched files for debugging
+  });
+  logger.info(`queries-hash: ${finalHash}`);
+  return finalHash;
+}
+async function getDatabaseHash(
+  language: Language,
+  config: config_utils.Config,
+  logger: Logger
+): Promise<string> {
+  const dbPath = util.getCodeQLDatabasePath(config.tempDir, language);
+  let relDir = path.join(dbPath, `db-${language}`, "default");
+  let combined_all = crypto.createHash("sha256");
+  let combined_noExtractionTime = crypto.createHash("sha256");
+  let files: {
+    [name: string]: string;
+  } = {};
+  let relFiles = fs
+    .readdirSync(relDir)
+    .filter((n) => n.endsWith(".rel"))
+    .map((n) => path.join(relDir, n));
+  if (relFiles.length === 0) {
+    throw new Error(
+      `No '.rel' files found in ${relDir}. Has the 'create-database' action been called?`
+    );
+  }
+  for (const relFile of relFiles) {
+    let content = fs.readFileSync(relFile); // XXX this ought to be chunked for large tables!
+    let solo = crypto.createHash("sha256");
+    solo.update(content);
+    files[path.relative(dbPath, relFile)] = solo.digest("hex");
+    if (
+      language === Language.javascript &&
+      path.basename(relFile) !== "extraction_time.rel"
+    ) {
+      combined_noExtractionTime.update(content);
+    }
+    combined_all.update(content);
+  }
+  let stableHash = combined_noExtractionTime.digest("hex");
+  logger.info("database-hash:");
+  logger.info(
+    JSON.stringify(
+      {
+        language,
+        combined: {
+          all: combined_all.digest("hex"),
+          noExtractionTime: stableHash,
+          files,
+        },
+      },
+      null,
+      2
+    )
+  );
+  return stableHash;
+}
+async function run() {
+  const logger = getActionsLogger();
+  try {
+    actionsUtil.prepareLocalRunEnvironment();
+
+    const config = await config_utils.getConfig(
+      actionsUtil.getRequiredEnvParam("RUNNER_TEMP"),
+      logger
+    );
+    if (config === undefined) {
+      throw new Error(
+        "Config file could not be found at expected location. Has the 'init' action been called?"
+      );
+    }
+    let hashesByLanguage: {
+      [language in keyof typeof Language]?: {
+        queries: string;
+        database: string;
+        codeql: string;
+      };
+    } = {};
+    for (const language of config.languages) {
+      (hashesByLanguage as any) /* XXX circumvent aggressive typescript */[
+        language
+      ] = {
+        queries: await getQueriesHash(language, config, logger),
+        database: await getDatabaseHash(language, config, logger),
+        codeql: getCodeQLHash(config),
+      };
+      core.setOutput("hashes", JSON.stringify(hashesByLanguage));
+    }
+  } catch (error) {
+    core.setFailed(`We were unable to hash the database.  ${error.message}`);
+    console.log(error);
+    return;
+  }
+}
+
+async function runWrapper() {
+  try {
+    await run();
+  } catch (error) {
+    core.setFailed(`hash-database action failed. ${error}`);
+    console.log(error);
+  }
+}
+
+void runWrapper();

src/init-action.ts

@@ -252,7 +252,7 @@ async function compileQueries(
   codeql: CodeQL,
   config: configUtils.Config,
   logger: Logger
-): Promise<string> {
+): Promise<void> {
   // MG: This method is based of `runQueries`.
   //       Creating the query suite file could be refactored out of that method.
   for (const language of config.languages) {
@@ -286,17 +286,6 @@ async function compileQueries(
       }
     }
   }
-  // Compute hash
-  const globHash = require("glob-hash");
-  const finalHash = await globHash({
-    include: [
-      `${config.tempDir}/**/.cache/data/**`,
-      `${config.toolCacheDir}/**/.cache/data/**`,
-    ],
-    files: false, // MG: List matched files for debugging
-  });
-  logger.info(`FinalHash: ${finalHash}`);
-  return finalHash;
 }
 
 void runWrapper();