diff --git a/hash-database/action.yml b/hash-database/action.yml new file mode 100644 index 000000000..71ab73a00 --- /dev/null +++ b/hash-database/action.yml @@ -0,0 +1,18 @@ +name: 'CodeQL: Hash Database' +description: 'Hashes the built database, using a stable hash' +author: 'GitHub' +inputs: + token: + default: ${{ github.token }} + matrix: + default: ${{ toJson(matrix) }} + threads: + description: The number of threads to be used by CodeQL. + required: false + default: 1 # a surgical way of forcing all `util.getThreadsFlag(...)` to resolve to 1 for our purposes +outputs: + hash: + description: the stable hash of the database +runs: + using: 'node12' + main: '../lib/hash-database-action.js' diff --git a/lib/hash-database-action.js b/lib/hash-database-action.js new file mode 100644 index 000000000..fed29ba50 --- /dev/null +++ b/lib/hash-database-action.js @@ -0,0 +1,82 @@ +"use strict"; +var __importStar = (this && this.__importStar) || function (mod) { + if (mod && mod.__esModule) return mod; + var result = {}; + if (mod != null) for (var k in mod) if (Object.hasOwnProperty.call(mod, k)) result[k] = mod[k]; + result["default"] = mod; + return result; +}; +Object.defineProperty(exports, "__esModule", { value: true }); +const core = __importStar(require("@actions/core")); +const fs = __importStar(require("fs")); +const path = __importStar(require("path")); +const crypto = __importStar(require("crypto")); +const codeql_1 = require("./codeql"); +const actionsUtil = __importStar(require("./actions-util")); +const config_utils = __importStar(require("./config-utils")); +const logging_1 = require("./logging"); +const util = __importStar(require("./util")); +async function run() { + const logger = logging_1.getActionsLogger(); + try { + actionsUtil.prepareLocalRunEnvironment(); + const config = await config_utils.getConfig(actionsUtil.getRequiredEnvParam("RUNNER_TEMP"), logger); + if (config === undefined) { + throw new Error("Config file could not be found at expected location. Has the 'init' action been called?"); + } + let codeql = codeql_1.getCodeQL(config.codeQLCmd); + for (const language of config.languages) { + logger.startGroup(`Extracting ${language}`); + await codeql.extractScannedLanguage(util.getCodeQLDatabasePath(config.tempDir, language), language); + logger.endGroup(); + logger.startGroup(`Finalizing ${language}`); + await codeql.finalizeDatabase(util.getCodeQLDatabasePath(config.tempDir, language), util.getThreadsFlag(actionsUtil.getOptionalInput("threads"), logger)); + logger.endGroup(); + // XXX this early extraction may break the subsequent analyze action step - do we need a bailout there? + const dbPath = util.getCodeQLDatabasePath(config.tempDir, language); + let relDir = path.join(dbPath, `db-${language}`, "default"); + let combined_all = crypto.createHash("sha256"); + let combined_noExtractionTime = crypto.createHash("sha256"); + let files = {}; + for (const relFile of fs + .readdirSync(relDir) + .filter((n) => n.endsWith(".rel")) + .map((n) => path.join(relDir, n))) { + let content = fs.readFileSync(relFile); // XXX this ought to be chunked for large tables! + let solo = crypto.createHash("sha256"); + solo.update(content); + files[path.relative(dbPath, relFile)] = solo.digest("hex"); + if (path.basename(relFile) !== "extraction_time.rel") { + combined_noExtractionTime.update(content); + } + combined_all.update(content); + } + let stableHash = combined_noExtractionTime.digest("hex"); + logger.info(JSON.stringify({ + language, + combined: { + all: combined_all.digest("hex"), + noExtractionTime: stableHash, + files, + }, + }, null, 2)); + core.setOutput("hash", stableHash); + } + } + catch (error) { + core.setFailed(`We were unable to hash the database. ${error.message}`); + console.log(error); + return; + } +} +async function runWrapper() { + try { + await run(); + } + catch (error) { + core.setFailed(`hash-database action failed. ${error}`); + console.log(error); + } +} +void runWrapper(); +//# sourceMappingURL=hash-database-action.js.map \ No newline at end of file diff --git a/lib/hash-database-action.js.map b/lib/hash-database-action.js.map new file mode 100644 index 000000000..eb24d7eb9 --- /dev/null +++ b/lib/hash-database-action.js.map @@ -0,0 +1 @@ +{"version":3,"file":"hash-database-action.js","sourceRoot":"","sources":["../src/hash-database-action.ts"],"names":[],"mappings":";;;;;;;;;AAAA,oDAAsC;AAEtC,uCAAyB;AACzB,2CAA6B;AAC7B,+CAAiC;AAEjC,qCAAqC;AAErC,4DAA8C;AAC9C,6DAA+C;AAC/C,uCAA6C;AAC7C,6CAA+B;AAE/B,KAAK,UAAU,GAAG;IAChB,MAAM,MAAM,GAAG,0BAAgB,EAAE,CAAC;IAClC,IAAI;QACF,WAAW,CAAC,0BAA0B,EAAE,CAAC;QAEzC,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC,SAAS,CACzC,WAAW,CAAC,mBAAmB,CAAC,aAAa,CAAC,EAC9C,MAAM,CACP,CAAC;QACF,IAAI,MAAM,KAAK,SAAS,EAAE;YACxB,MAAM,IAAI,KAAK,CACb,yFAAyF,CAC1F,CAAC;SACH;QACD,IAAI,MAAM,GAAG,kBAAS,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC;QACzC,KAAK,MAAM,QAAQ,IAAI,MAAM,CAAC,SAAS,EAAE;YACvC,MAAM,CAAC,UAAU,CAAC,cAAc,QAAQ,EAAE,CAAC,CAAC;YAC5C,MAAM,MAAM,CAAC,sBAAsB,CACjC,IAAI,CAAC,qBAAqB,CAAC,MAAM,CAAC,OAAO,EAAE,QAAQ,CAAC,EACpD,QAAQ,CACT,CAAC;YACF,MAAM,CAAC,QAAQ,EAAE,CAAC;YAClB,MAAM,CAAC,UAAU,CAAC,cAAc,QAAQ,EAAE,CAAC,CAAC;YAC5C,MAAM,MAAM,CAAC,gBAAgB,CAC3B,IAAI,CAAC,qBAAqB,CAAC,MAAM,CAAC,OAAO,EAAE,QAAQ,CAAC,EACpD,IAAI,CAAC,cAAc,CAAC,WAAW,CAAC,gBAAgB,CAAC,SAAS,CAAC,EAAE,MAAM,CAAC,CACrE,CAAC;YACF,MAAM,CAAC,QAAQ,EAAE,CAAC;YAElB,uGAAuG;YAEvG,MAAM,MAAM,GAAG,IAAI,CAAC,qBAAqB,CAAC,MAAM,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;YACpE,IAAI,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,MAAM,QAAQ,EAAE,EAAE,SAAS,CAAC,CAAC;YAC5D,IAAI,YAAY,GAAG,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;YAC/C,IAAI,yBAAyB,GAAG,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;YAC5D,IAAI,KAAK,GAEL,EAAE,CAAC;YACP,KAAK,MAAM,OAAO,IAAI,EAAE;iBACrB,WAAW,CAAC,MAAM,CAAC;iBACnB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;iBACjC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE;gBACnC,IAAI,OAAO,GAAG,EAAE,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC,iDAAiD;gBACzF,IAAI,IAAI,GAAG,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;gBACvC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;gBACrB,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;gBAC3D,IAAI,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,KAAK,qBAAqB,EAAE;oBACpD,yBAAyB,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;iBAC3C;gBACD,YAAY,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;aAC9B;YACD,IAAI,UAAU,GAAG,yBAAyB,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YACzD,MAAM,CAAC,IAAI,CACT,IAAI,CAAC,SAAS,CACZ;gBACE,QAAQ;gBACR,QAAQ,EAAE;oBACR,GAAG,EAAE,YAAY,CAAC,MAAM,CAAC,KAAK,CAAC;oBAC/B,gBAAgB,EAAE,UAAU;oBAC5B,KAAK;iBACN;aACF,EACD,IAAI,EACJ,CAAC,CACF,CACF,CAAC;YACF,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;SACpC;KACF;IAAC,OAAO,KAAK,EAAE;QACd,IAAI,CAAC,SAAS,CAAC,yCAAyC,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;QACzE,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QACnB,OAAO;KACR;AACH,CAAC;AAED,KAAK,UAAU,UAAU;IACvB,IAAI;QACF,MAAM,GAAG,EAAE,CAAC;KACb;IAAC,OAAO,KAAK,EAAE;QACd,IAAI,CAAC,SAAS,CAAC,gCAAgC,KAAK,EAAE,CAAC,CAAC;QACxD,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;KACpB;AACH,CAAC;AAED,KAAK,UAAU,EAAE,CAAC"} \ No newline at end of file diff --git a/src/hash-database-action.ts b/src/hash-database-action.ts new file mode 100644 index 000000000..9a4396944 --- /dev/null +++ b/src/hash-database-action.ts @@ -0,0 +1,98 @@ +import * as core from "@actions/core"; + +import * as fs from "fs"; +import * as path from "path"; +import * as crypto from "crypto"; + +import { getCodeQL } from "./codeql"; + +import * as actionsUtil from "./actions-util"; +import * as config_utils from "./config-utils"; +import { getActionsLogger } from "./logging"; +import * as util from "./util"; + +async function run() { + const logger = getActionsLogger(); + try { + actionsUtil.prepareLocalRunEnvironment(); + + const config = await config_utils.getConfig( + actionsUtil.getRequiredEnvParam("RUNNER_TEMP"), + logger + ); + if (config === undefined) { + throw new Error( + "Config file could not be found at expected location. Has the 'init' action been called?" + ); + } + let codeql = getCodeQL(config.codeQLCmd); + for (const language of config.languages) { + logger.startGroup(`Extracting ${language}`); + await codeql.extractScannedLanguage( + util.getCodeQLDatabasePath(config.tempDir, language), + language + ); + logger.endGroup(); + logger.startGroup(`Finalizing ${language}`); + await codeql.finalizeDatabase( + util.getCodeQLDatabasePath(config.tempDir, language), + util.getThreadsFlag(actionsUtil.getOptionalInput("threads"), logger) + ); + logger.endGroup(); + + // XXX this early extraction may break the subsequent analyze action step - do we need a bailout there? + + const dbPath = util.getCodeQLDatabasePath(config.tempDir, language); + let relDir = path.join(dbPath, `db-${language}`, "default"); + let combined_all = crypto.createHash("sha256"); + let combined_noExtractionTime = crypto.createHash("sha256"); + let files: { + [name: string]: string; + } = {}; + for (const relFile of fs + .readdirSync(relDir) + .filter((n) => n.endsWith(".rel")) + .map((n) => path.join(relDir, n))) { + let content = fs.readFileSync(relFile); // XXX this ought to be chunked for large tables! + let solo = crypto.createHash("sha256"); + solo.update(content); + files[path.relative(dbPath, relFile)] = solo.digest("hex"); + if (path.basename(relFile) !== "extraction_time.rel") { + combined_noExtractionTime.update(content); + } + combined_all.update(content); + } + let stableHash = combined_noExtractionTime.digest("hex"); + logger.info( + JSON.stringify( + { + language, + combined: { + all: combined_all.digest("hex"), + noExtractionTime: stableHash, + files, + }, + }, + null, + 2 + ) + ); + core.setOutput("hash", stableHash); + } + } catch (error) { + core.setFailed(`We were unable to hash the database. ${error.message}`); + console.log(error); + return; + } +} + +async function runWrapper() { + try { + await run(); + } catch (error) { + core.setFailed(`hash-database action failed. ${error}`); + console.log(error); + } +} + +void runWrapper();