From 56407639dd6552b7c1cda6b37f33cd29d4376b83 Mon Sep 17 00:00:00 2001 From: Marco Gario Date: Thu, 10 Dec 2020 12:19:48 +0100 Subject: [PATCH] Add node lib/dbhash_cli.js --- lib/hash-inputs-action.js | 39 ++------------------------ lib/hash-inputs-action.js.map | 2 +- src/dbhash_cli.ts | 15 ++++++++++ src/hash-inputs-action.ts | 52 +++-------------------------------- 4 files changed, 22 insertions(+), 86 deletions(-) create mode 100644 src/dbhash_cli.ts diff --git a/lib/hash-inputs-action.js b/lib/hash-inputs-action.js index a116e6eaa..ca14c9e04 100644 --- a/lib/hash-inputs-action.js +++ b/lib/hash-inputs-action.js @@ -20,14 +20,11 @@ var __importStar = (this && this.__importStar) || function (mod) { }; Object.defineProperty(exports, "__esModule", { value: true }); const core = __importStar(require("@actions/core")); -const fs = __importStar(require("fs")); -const path = __importStar(require("path")); -const crypto = __importStar(require("crypto")); const actionsUtil = __importStar(require("./actions-util")); const config_utils = __importStar(require("./config-utils")); const logging_1 = require("./logging"); const util = __importStar(require("./util")); -const languages_1 = require("./languages"); +const hash_inputs_1 = require("./hash-inputs"); async function getCodeQLHash(_config) { return "DUMMY_CODEQL_HASH"; } @@ -47,39 +44,7 @@ async function getQueriesHash(_language, config, logger) { } async function getDatabaseHash(language, config, logger) { const dbPath = util.getCodeQLDatabasePath(config.tempDir, language); - let relDir = path.join(dbPath, `db-${language}`, "default"); - let combined_all = crypto.createHash("sha256"); - let combined_noExtractionTime = crypto.createHash("sha256"); - let files = {}; - let relFiles = fs - .readdirSync(relDir) - .filter((n) => n.endsWith(".rel")) - .map((n) => path.join(relDir, n)); - if (relFiles.length === 0) { - throw new Error(`No '.rel' files found in ${relDir}. Has the 'create-database' action been called?`); - } - for (const relFile of relFiles) { - let content = fs.readFileSync(relFile); // XXX this ought to be chunked for large tables! - let solo = crypto.createHash("sha256"); - solo.update(content); - files[path.relative(dbPath, relFile)] = solo.digest("hex"); - if (language === languages_1.Language.javascript && - path.basename(relFile) !== "extraction_time.rel") { - combined_noExtractionTime.update(content); - } - combined_all.update(content); - } - let stableHash = combined_noExtractionTime.digest("hex"); - logger.info("database-hash:"); - logger.info(JSON.stringify({ - language, - combined: { - all: combined_all.digest("hex"), - noExtractionTime: stableHash, - files, - }, - }, null, 2)); - return stableHash; + return hash_inputs_1.DatabaseHash(language, dbPath, logger); } async function run() { const logger = logging_1.getActionsLogger(); diff --git a/lib/hash-inputs-action.js.map b/lib/hash-inputs-action.js.map index d9f47f51e..c91c982b5 100644 --- a/lib/hash-inputs-action.js.map +++ b/lib/hash-inputs-action.js.map @@ -1 +1 @@ -{"version":3,"file":"hash-inputs-action.js","sourceRoot":"","sources":["../src/hash-inputs-action.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;AAAA,oDAAsC;AAEtC,uCAAyB;AACzB,2CAA6B;AAC7B,+CAAiC;AAEjC,4DAA8C;AAC9C,6DAA+C;AAC/C,uCAAqD;AACrD,6CAA+B;AAC/B,2CAAuC;AAEvC,KAAK,UAAU,aAAa,CAAC,OAA4B;IACvD,OAAO,mBAAmB,CAAC;AAC7B,CAAC;AAED,KAAK,UAAU,cAAc,CAC3B,SAAmB,EACnB,MAA2B,EAC3B,MAAc;IAEd,eAAe;IACf,MAAM,QAAQ,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC;IACtC,MAAM,SAAS,GAAG,MAAM,QAAQ,CAAC;QAC/B,OAAO,EAAE;YACP,wFAAwF;YACxF,GAAG,MAAM,CAAC,OAAO,oBAAoB;YACrC,GAAG,MAAM,CAAC,YAAY,oBAAoB;SAC3C;QACD,KAAK,EAAE,KAAK;KACb,CAAC,CAAC;IACH,MAAM,CAAC,IAAI,CAAC,iBAAiB,SAAS,EAAE,CAAC,CAAC;IAC1C,OAAO,SAAS,CAAC;AACnB,CAAC;AACD,KAAK,UAAU,eAAe,CAC5B,QAAkB,EAClB,MAA2B,EAC3B,MAAc;IAEd,MAAM,MAAM,GAAG,IAAI,CAAC,qBAAqB,CAAC,MAAM,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;IACpE,IAAI,MAAM,GAAG,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,MAAM,QAAQ,EAAE,EAAE,SAAS,CAAC,CAAC;IAC5D,IAAI,YAAY,GAAG,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;IAC/C,IAAI,yBAAyB,GAAG,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;IAC5D,IAAI,KAAK,GAEL,EAAE,CAAC;IACP,IAAI,QAAQ,GAAG,EAAE;SACd,WAAW,CAAC,MAAM,CAAC;SACnB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;SACjC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC;IACpC,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE;QACzB,MAAM,IAAI,KAAK,CACb,4BAA4B,MAAM,iDAAiD,CACpF,CAAC;KACH;IACD,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE;QAC9B,IAAI,OAAO,GAAG,EAAE,CAAC,YAAY,CAAC,OAAO,CAAC,CAAC,CAAC,iDAAiD;QACzF,IAAI,IAAI,GAAG,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;QACvC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;QACrB,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QAC3D,IACE,QAAQ,KAAK,oBAAQ,CAAC,UAAU;YAChC,IAAI,CAAC,QAAQ,CAAC,OAAO,CAAC,KAAK,qBAAqB,EAChD;YACA,yBAAyB,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;SAC3C;QACD,YAAY,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;KAC9B;IACD,IAAI,UAAU,GAAG,yBAAyB,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IACzD,MAAM,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;IAC9B,MAAM,CAAC,IAAI,CACT,IAAI,CAAC,SAAS,CACZ;QACE,QAAQ;QACR,QAAQ,EAAE;YACR,GAAG,EAAE,YAAY,CAAC,MAAM,CAAC,KAAK,CAAC;YAC/B,gBAAgB,EAAE,UAAU;YAC5B,KAAK;SACN;KACF,EACD,IAAI,EACJ,CAAC,CACF,CACF,CAAC;IACF,OAAO,UAAU,CAAC;AACpB,CAAC;AACD,KAAK,UAAU,GAAG;IAChB,MAAM,MAAM,GAAG,0BAAgB,EAAE,CAAC;IAClC,IAAI;QACF,WAAW,CAAC,0BAA0B,EAAE,CAAC;QAEzC,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC,SAAS,CACzC,WAAW,CAAC,mBAAmB,CAAC,aAAa,CAAC,EAC9C,MAAM,CACP,CAAC;QACF,IAAI,MAAM,KAAK,SAAS,EAAE;YACxB,MAAM,IAAI,KAAK,CACb,yFAAyF,CAC1F,CAAC;SACH;QACD,IAAI,gBAAgB,GAMhB,EAAE,CAAC;QACP,KAAK,MAAM,QAAQ,IAAI,MAAM,CAAC,SAAS,EAAE;YACtC,gBAAwB,CAAC,0CAA0C,CAClE,QAAQ,CACT,GAAG;gBACF,OAAO,EAAE,MAAM,cAAc,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,CAAC;gBACvD,QAAQ,EAAE,MAAM,eAAe,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,CAAC;gBACzD,MAAM,EAAE,aAAa,CAAC,MAAM,CAAC;aAC9B,CAAC;SACH;QACD,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QACvB,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,gBAAgB,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;QACvD,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,gBAAgB,CAAC,CAAC,CAAC;KAC5D;IAAC,OAAO,KAAK,EAAE;QACd,IAAI,CAAC,SAAS,CAAC,uCAAuC,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;QACvE,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QACnB,OAAO;KACR;AACH,CAAC;AAED,KAAK,UAAU,UAAU;IACvB,IAAI;QACF,MAAM,GAAG,EAAE,CAAC;KACb;IAAC,OAAO,KAAK,EAAE;QACd,IAAI,CAAC,SAAS,CAAC,8BAA8B,KAAK,EAAE,CAAC,CAAC;QACtD,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;KACpB;AACH,CAAC;AAED,KAAK,UAAU,EAAE,CAAC"} \ No newline at end of file +{"version":3,"file":"hash-inputs-action.js","sourceRoot":"","sources":["../src/hash-inputs-action.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;AAAA,oDAAsC;AAGtC,4DAA8C;AAC9C,6DAA+C;AAC/C,uCAAqD;AACrD,6CAA+B;AAE/B,+CAA0C;AAE1C,KAAK,UAAU,aAAa,CAAC,OAA4B;IACvD,OAAO,mBAAmB,CAAC;AAC7B,CAAC;AAED,KAAK,UAAU,cAAc,CAC3B,SAAmB,EACnB,MAA2B,EAC3B,MAAc;IAEd,eAAe;IACf,MAAM,QAAQ,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC;IACtC,MAAM,SAAS,GAAG,MAAM,QAAQ,CAAC;QAC/B,OAAO,EAAE;YACP,wFAAwF;YACxF,GAAG,MAAM,CAAC,OAAO,oBAAoB;YACrC,GAAG,MAAM,CAAC,YAAY,oBAAoB;SAC3C;QACD,KAAK,EAAE,KAAK;KACb,CAAC,CAAC;IACH,MAAM,CAAC,IAAI,CAAC,iBAAiB,SAAS,EAAE,CAAC,CAAC;IAC1C,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,KAAK,UAAU,eAAe,CAC5B,QAAkB,EAClB,MAA2B,EAC3B,MAAc;IAEd,MAAM,MAAM,GAAG,IAAI,CAAC,qBAAqB,CAAC,MAAM,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;IACpE,OAAO,0BAAY,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC;AAChD,CAAC;AAED,KAAK,UAAU,GAAG;IAChB,MAAM,MAAM,GAAG,0BAAgB,EAAE,CAAC;IAClC,IAAI;QACF,WAAW,CAAC,0BAA0B,EAAE,CAAC;QAEzC,MAAM,MAAM,GAAG,MAAM,YAAY,CAAC,SAAS,CACzC,WAAW,CAAC,mBAAmB,CAAC,aAAa,CAAC,EAC9C,MAAM,CACP,CAAC;QACF,IAAI,MAAM,KAAK,SAAS,EAAE;YACxB,MAAM,IAAI,KAAK,CACb,yFAAyF,CAC1F,CAAC;SACH;QACD,IAAI,gBAAgB,GAMhB,EAAE,CAAC;QACP,KAAK,MAAM,QAAQ,IAAI,MAAM,CAAC,SAAS,EAAE;YACtC,gBAAwB,CAAC,0CAA0C,CAClE,QAAQ,CACT,GAAG;gBACF,OAAO,EAAE,MAAM,cAAc,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,CAAC;gBACvD,QAAQ,EAAE,MAAM,eAAe,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,CAAC;gBACzD,MAAM,EAAE,aAAa,CAAC,MAAM,CAAC;aAC9B,CAAC;SACH;QACD,MAAM,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QACvB,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,gBAAgB,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;QACvD,IAAI,CAAC,SAAS,CAAC,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,gBAAgB,CAAC,CAAC,CAAC;KAC5D;IAAC,OAAO,KAAK,EAAE;QACd,IAAI,CAAC,SAAS,CAAC,uCAAuC,KAAK,CAAC,OAAO,EAAE,CAAC,CAAC;QACvE,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QACnB,OAAO;KACR;AACH,CAAC;AAED,KAAK,UAAU,UAAU;IACvB,IAAI;QACF,MAAM,GAAG,EAAE,CAAC;KACb;IAAC,OAAO,KAAK,EAAE;QACd,IAAI,CAAC,SAAS,CAAC,8BAA8B,KAAK,EAAE,CAAC,CAAC;QACtD,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;KACpB;AACH,CAAC;AAED,KAAK,UAAU,EAAE,CAAC"} \ No newline at end of file diff --git a/src/dbhash_cli.ts b/src/dbhash_cli.ts new file mode 100644 index 000000000..37aba16dd --- /dev/null +++ b/src/dbhash_cli.ts @@ -0,0 +1,15 @@ +import {DatabaseHash} from "./hash-inputs"; +import {getRunnerLogger} from "./logging"; +import {Language} from "./languages"; + +const dbPath = process.argv[2]; +const logger = getRunnerLogger(true); + +/* TODO: Do we need to unboundle here or before? */ +const stableHash = DatabaseHash( + Language.javascript, + dbPath, + logger +); + +logger.info(`stableHash: ${stableHash}`); diff --git a/src/hash-inputs-action.ts b/src/hash-inputs-action.ts index d274c9a90..7667d9a49 100644 --- a/src/hash-inputs-action.ts +++ b/src/hash-inputs-action.ts @@ -1,14 +1,12 @@ import * as core from "@actions/core"; -import * as fs from "fs"; -import * as path from "path"; -import * as crypto from "crypto"; import * as actionsUtil from "./actions-util"; import * as config_utils from "./config-utils"; import { getActionsLogger, Logger } from "./logging"; import * as util from "./util"; import { Language } from "./languages"; +import {DatabaseHash} from "./hash-inputs" async function getCodeQLHash(_config: config_utils.Config) { return "DUMMY_CODEQL_HASH"; @@ -32,58 +30,16 @@ async function getQueriesHash( logger.info(`queries-hash: ${finalHash}`); return finalHash; } + async function getDatabaseHash( language: Language, config: config_utils.Config, logger: Logger ): Promise { const dbPath = util.getCodeQLDatabasePath(config.tempDir, language); - let relDir = path.join(dbPath, `db-${language}`, "default"); - let combined_all = crypto.createHash("sha256"); - let combined_noExtractionTime = crypto.createHash("sha256"); - let files: { - [name: string]: string; - } = {}; - let relFiles = fs - .readdirSync(relDir) - .filter((n) => n.endsWith(".rel")) - .map((n) => path.join(relDir, n)); - if (relFiles.length === 0) { - throw new Error( - `No '.rel' files found in ${relDir}. Has the 'create-database' action been called?` - ); - } - for (const relFile of relFiles) { - let content = fs.readFileSync(relFile); // XXX this ought to be chunked for large tables! - let solo = crypto.createHash("sha256"); - solo.update(content); - files[path.relative(dbPath, relFile)] = solo.digest("hex"); - if ( - language === Language.javascript && - path.basename(relFile) !== "extraction_time.rel" - ) { - combined_noExtractionTime.update(content); - } - combined_all.update(content); - } - let stableHash = combined_noExtractionTime.digest("hex"); - logger.info("database-hash:"); - logger.info( - JSON.stringify( - { - language, - combined: { - all: combined_all.digest("hex"), - noExtractionTime: stableHash, - files, - }, - }, - null, - 2 - ) - ); - return stableHash; + return DatabaseHash(language, dbPath, logger); } + async function run() { const logger = getActionsLogger(); try {