Merge remote-tracking branch 'origin/main' into platform_lang_pkg

src/config-utils.test.ts

@@ -14,6 +14,13 @@ import * as util from './util';
 
 setupTests(test);
 
+// Returns the filepath of the newly-created file
+function createConfigFile(inputFileContents: string, tmpDir: string): string {
+  const configFilePath = path.join(tmpDir, 'input');
+  fs.writeFileSync(configFilePath, inputFileContents, 'utf8');
+  return configFilePath;
+}
+
 type GetContentsResponse = { content?: string; } | {}[];
 
 function mockGetContents(content: GetContentsResponse): sinon.SinonStub<any, any> {
@@ -241,13 +248,12 @@ test("load non-empty input", async t => {
     };
 
     const languages = [Language.javascript];
-    const configFile = 'input';
-    fs.writeFileSync(path.join(tmpDir, configFile), inputFileContents, 'utf8');
+    const configFilePath = createConfigFile(inputFileContents, tmpDir);
 
     const actualConfig = await configUtils.initConfig(
       languages,
       undefined,
-      configFile,
+      configFilePath,
       tmpDir,
       tmpDir,
       codeQL,
@@ -261,7 +267,7 @@ test("load non-empty input", async t => {
   });
 });
 
-test("default queries are used", async t => {
+test("Default queries are used", async t => {
   return await util.withTmpDir(async tmpDir => {
     // Check that the default behaviour is to add the default queries.
     // In this case if a config file is specified but does not include
@@ -295,13 +301,12 @@ test("default queries are used", async t => {
     fs.mkdirSync(path.join(tmpDir, 'foo'));
 
     const languages = [Language.javascript];
-    const configFile = 'input';
-    fs.writeFileSync(path.join(tmpDir, configFile), inputFileContents, 'utf8');
+    const configFilePath = createConfigFile(inputFileContents, tmpDir);
 
     await configUtils.initConfig(
       languages,
       undefined,
-      configFile,
+      configFilePath,
       tmpDir,
       tmpDir,
       codeQL,
@@ -317,6 +322,23 @@ test("default queries are used", async t => {
   });
 });
 
+/**
+ * Returns the provided queries, just in the right format for a resolved query
+ * This way we can test by seeing which returned items are in the final
+ * configuration.
+ */
+function queriesToResolvedQueryForm(queries: string[]) {
+  const dummyResolvedQueries = {};
+  queries.forEach(q => { dummyResolvedQueries[q] = {}; });
+  return {
+    byLanguage: {
+      'javascript': dummyResolvedQueries,
+    },
+    noDeclaredLanguage: {},
+    multipleDeclaredLanguages: {},
+  };
+}
+
 test("Queries can be specified in config file", async t => {
   return await util.withTmpDir(async tmpDir => {
     const inputFileContents = `
@@ -324,8 +346,7 @@ test("Queries can be specified in config file", async t => {
       queries:
         - uses: ./foo`;
 
-    const configFile = path.join(tmpDir, 'input');
-    fs.writeFileSync(configFile, inputFileContents, 'utf8');
+    const configFilePath = createConfigFile(inputFileContents, tmpDir);
 
     fs.mkdirSync(path.join(tmpDir, 'foo'));
 
@@ -333,18 +354,7 @@ test("Queries can be specified in config file", async t => {
     const codeQL = setCodeQL({
       resolveQueries: async function(queries: string[], extraSearchPath: string | undefined) {
         resolveQueriesArgs.push({queries, extraSearchPath});
-        // Return what we're given, just in the right format for a resolved query
-        // This way we can test by seeing which returned items are in the final
-        // configuration.
-        const dummyResolvedQueries = {};
-        queries.forEach(q => { dummyResolvedQueries[q] = {}; });
-        return {
-          byLanguage: {
-            'javascript': dummyResolvedQueries,
-          },
-          noDeclaredLanguage: {},
-          multipleDeclaredLanguages: {},
-        };
+        return queriesToResolvedQueryForm(queries);
       },
     });
 
@@ -353,7 +363,7 @@ test("Queries can be specified in config file", async t => {
     const config = await configUtils.initConfig(
       languages,
       undefined,
-      configFile,
+      configFilePath,
       tmpDir,
       tmpDir,
       codeQL,
@@ -383,8 +393,7 @@ test("Queries from config file can be overridden in workflow file", async t => {
       queries:
         - uses: ./foo`;
 
-    const configFile = path.join(tmpDir, 'input');
-    fs.writeFileSync(configFile, inputFileContents, 'utf8');
+    const configFilePath = createConfigFile(inputFileContents, tmpDir);
 
     // This config item should take precedence over the config file but shouldn't affect the default queries.
     const queries = './override';
@@ -396,18 +405,7 @@ test("Queries from config file can be overridden in workflow file", async t => {
     const codeQL = setCodeQL({
       resolveQueries: async function(queries: string[], extraSearchPath: string | undefined) {
         resolveQueriesArgs.push({queries, extraSearchPath});
-        // Return what we're given, just in the right format for a resolved query
-        // This way we can test overriding by seeing which returned items are in
-        // the final configuration.
-        const dummyResolvedQueries = {};
-        queries.forEach(q => { dummyResolvedQueries[q] = {}; });
-        return {
-          byLanguage: {
-            'javascript': dummyResolvedQueries,
-          },
-          noDeclaredLanguage: {},
-          multipleDeclaredLanguages: {},
-        };
+        return queriesToResolvedQueryForm(queries);
       },
     });
 
@@ -416,7 +414,7 @@ test("Queries from config file can be overridden in workflow file", async t => {
     const config = await configUtils.initConfig(
       languages,
       queries,
-      configFile,
+      configFilePath,
       tmpDir,
       tmpDir,
       codeQL,
@@ -439,6 +437,54 @@ test("Queries from config file can be overridden in workflow file", async t => {
   });
 });
 
+test("Queries in workflow file can be used in tandem with the 'disable default queries' option", async t => {
+  return await util.withTmpDir(async tmpDir => {
+    process.env['RUNNER_TEMP'] = tmpDir;
+    process.env['GITHUB_WORKSPACE'] = tmpDir;
+
+    const inputFileContents = `
+      name: my config
+      disable-default-queries: true`;
+    const configFilePath = createConfigFile(inputFileContents, tmpDir);
+
+    const queries = './workflow-query';
+    fs.mkdirSync(path.join(tmpDir, 'workflow-query'));
+
+    const resolveQueriesArgs: {queries: string[], extraSearchPath: string | undefined}[] = [];
+    const codeQL = setCodeQL({
+      resolveQueries: async function(queries: string[], extraSearchPath: string | undefined) {
+        resolveQueriesArgs.push({queries, extraSearchPath});
+        return queriesToResolvedQueryForm(queries);
+      },
+    });
+
+    const languages = [Language.javascript];
+
+    const config = await configUtils.initConfig(
+      languages,
+      queries,
+      configFilePath,
+      tmpDir,
+      tmpDir,
+      codeQL,
+      tmpDir,
+      'token',
+      'https://github.example.com',
+      getRunnerLogger(true));
+
+    // Check resolveQueries was called correctly
+    // It'll be called once for `./workflow-query`,
+    // but won't be called for the default one since that was disabled
+    t.deepEqual(resolveQueriesArgs.length, 1);
+    t.deepEqual(resolveQueriesArgs[0].queries.length, 1);
+    t.regex(resolveQueriesArgs[0].queries[0], /.*\/workflow-query$/);
+
+    // Now check that the end result contains only the workflow query, and not the default one
+    t.deepEqual(config.queries['javascript'].length, 1);
+    t.regex(config.queries['javascript'][0], /.*\/workflow-query$/);
+  });
+});
+
 test("Multiple queries can be specified in workflow file, no config file required", async t => {
   return await util.withTmpDir(async tmpDir => {
     fs.mkdirSync(path.join(tmpDir, 'override1'));
@@ -450,18 +496,7 @@ test("Multiple queries can be specified in workflow file, no config file require
     const codeQL = setCodeQL({
       resolveQueries: async function(queries: string[], extraSearchPath: string | undefined) {
         resolveQueriesArgs.push({queries, extraSearchPath});
-        // Return what we're given, just in the right format for a resolved query
-        // This way we can test overriding by seeing which returned items are in
-        // the final configuration.
-        const dummyResolvedQueries = {};
-        queries.forEach(q => { dummyResolvedQueries[q] = {}; });
-        return {
-          byLanguage: {
-            'javascript': dummyResolvedQueries,
-          },
-          noDeclaredLanguage: {},
-          multipleDeclaredLanguages: {},
-        };
+        return queriesToResolvedQueryForm(queries);
       },
     });
 
@@ -496,6 +531,67 @@ test("Multiple queries can be specified in workflow file, no config file require
   });
 });
 
+test("Queries in workflow file can be added to the set of queries without overriding config file", async t => {
+  return await util.withTmpDir(async tmpDir => {
+    process.env['RUNNER_TEMP'] = tmpDir;
+    process.env['GITHUB_WORKSPACE'] = tmpDir;
+
+    const inputFileContents = `
+      name: my config
+      queries:
+        - uses: ./foo`;
+    const configFilePath = createConfigFile(inputFileContents, tmpDir);
+
+    // These queries shouldn't override anything, because the value is prefixed with "+"
+    const queries = '+./additional1,./additional2';
+
+    fs.mkdirSync(path.join(tmpDir, 'foo'));
+    fs.mkdirSync(path.join(tmpDir, 'additional1'));
+    fs.mkdirSync(path.join(tmpDir, 'additional2'));
+
+    const resolveQueriesArgs: {queries: string[], extraSearchPath: string | undefined}[] = [];
+    const codeQL = setCodeQL({
+      resolveQueries: async function(queries: string[], extraSearchPath: string | undefined) {
+        resolveQueriesArgs.push({queries, extraSearchPath});
+        return queriesToResolvedQueryForm(queries);
+      },
+    });
+
+    const languages = [Language.javascript];
+
+    const config = await configUtils.initConfig(
+      languages,
+      queries,
+      configFilePath,
+      tmpDir,
+      tmpDir,
+      codeQL,
+      tmpDir,
+      'token',
+      'https://github.example.com',
+      getRunnerLogger(true));
+
+    // Check resolveQueries was called correctly
+    // It'll be called once for the default queries,
+    // once for each of additional1 and additional2,
+    // and once for './foo' from the config file
+    t.deepEqual(resolveQueriesArgs.length, 4);
+    t.deepEqual(resolveQueriesArgs[1].queries.length, 1);
+    t.regex(resolveQueriesArgs[1].queries[0], /.*\/additional1$/);
+    t.deepEqual(resolveQueriesArgs[2].queries.length, 1);
+    t.regex(resolveQueriesArgs[2].queries[0], /.*\/additional2$/);
+    t.deepEqual(resolveQueriesArgs[3].queries.length, 1);
+    t.regex(resolveQueriesArgs[3].queries[0], /.*\/foo$/);
+
+    // Now check that the end result contains all the queries
+    t.deepEqual(config.queries['javascript'].length, 4);
+    t.regex(config.queries['javascript'][0], /javascript-code-scanning.qls$/);
+    t.regex(config.queries['javascript'][1], /.*\/additional1$/);
+    t.regex(config.queries['javascript'][2], /.*\/additional2$/);
+    t.regex(config.queries['javascript'][3], /.*\/foo$/);
+  });
+});
+
 test("Invalid queries in workflow file handled correctly", async t => {
   return await util.withTmpDir(async tmpDir => {
     const queries = 'foo/bar@v1@v3';

src/config-utils.ts

@@ -540,10 +540,6 @@ export async function getLanguages(
   return parsedLanguages;
 }
 
-/**
- * Returns true if queries were provided in the workflow file
- * (and thus added), otherwise false
- */
 async function addQueriesFromWorkflow(
   codeQL: CodeQL,
   queriesInput: string,
@@ -554,6 +550,10 @@ async function addQueriesFromWorkflow(
   githubUrl: string,
   logger: Logger) {
 
+  queriesInput = queriesInput.trim();
+  // "+" means "don't override config file" - see shouldAddConfigFileQueries
+  queriesInput = queriesInput.replace(/^\+/, '');
+
   for (const query of queriesInput.split(',')) {
     await parseQueryUses(
       languages,
@@ -567,6 +567,18 @@ async function addQueriesFromWorkflow(
   }
 }
 
+// Returns true if either no queries were provided in the workflow.
+// or if the queries in the workflow were provided in "additive" mode,
+// indicating that they shouldn't override the config queries but
+// should instead be added in addition
+function shouldAddConfigFileQueries(queriesInput: string | undefined): boolean {
+  if (queriesInput) {
+    return queriesInput.trimStart().substr(0, 1) === '+';
+  }
+
+  return true;
+}
+
 /**
  * Get the default config for when the user has not supplied one.
  */
@@ -662,6 +674,8 @@ async function loadConfig(
 
   // If queries were provided using `with` in the action configuration,
   // they should take precedence over the queries in the config file
+  // unless they're prefixed with "+", in which case they supplement those
+  // in the config file.
   if (queriesInput) {
     await addQueriesFromWorkflow(
       codeQL,
@@ -672,7 +686,8 @@ async function loadConfig(
       checkoutPath,
       githubUrl,
       logger);
-  } else if (QUERIES_PROPERTY in parsedYAML) {
+  }
+  if (shouldAddConfigFileQueries(queriesInput) && QUERIES_PROPERTY in parsedYAML) {
     if (!(parsedYAML[QUERIES_PROPERTY] instanceof Array)) {
       throw new Error(getQueriesInvalid(configFile));
     }

src/external-queries.test.ts

@@ -1,3 +1,4 @@
+import * as toolrunnner from '@actions/exec/lib/toolrunner';
 import test from 'ava';
 import * as fs from "fs";
 import * as path from "path";
@@ -11,15 +12,86 @@ setupTests(test);
 
 test("checkoutExternalQueries", async t => {
   await util.withTmpDir(async tmpDir => {
-    const ref = "df4c6869212341b601005567381944ed90906b6b";
+    // Create a test repo in a subdir of the temp dir.
+    // It should have a default branch with two commits after the initial commit, where
+    // - the first commit contains files 'a' and 'b'
+    // - the second commit contains only 'a'
+    // Place the repo in a subdir because we're going to checkout a copy in tmpDir
+    const testRepoBaseDir = path.join(tmpDir, 'test-repo-dir');
+    const repoName = 'some/repo';
+    const repoPath = path.join(testRepoBaseDir, repoName);
+    const repoGitDir = path.join(repoPath, '.git');
+
+    // Run the given git command, and return the output.
+    // Passes --git-dir and --work-tree.
+    // Any stderr output is suppressed until the command fails.
+    const runGit = async function(command: string[]): Promise<string> {
+      let stdout = '';
+      let stderr = '';
+      command = [`--git-dir=${repoGitDir}`, `--work-tree=${repoPath}`, ...command];
+      console.log('Running: git ' + command.join(' '));
+      try {
+        await new toolrunnner.ToolRunner('git', command, {
+          silent: true,
+          listeners: {
+            stdout: (data) => { stdout += data.toString(); },
+            stderr: (data) => { stderr += data.toString(); },
+          }
+        }).exec();
+      } catch (e) {
+        console.log('Command failed: git ' + command.join(' '));
+        process.stderr.write(stderr);
+        throw e;
+      }
+      return stdout.trim();
+    };
+
+    fs.mkdirSync(repoPath, { recursive: true });
+    await runGit(['init', repoPath]);
+    await runGit(['config', 'user.email', 'test@github.com']);
+    await runGit(['config', 'user.name', 'Test Test']);
+
+    fs.writeFileSync(path.join(repoPath, 'a'), 'a content');
+    await runGit(['add', 'a']);
+    await runGit(['commit', '-m', 'commit1']);
+
+    fs.writeFileSync(path.join(repoPath, 'b'), 'b content');
+    await runGit(['add', 'b']);
+    await runGit(['commit', '-m', 'commit1']);
+    const commit1Sha = await runGit(['rev-parse', 'HEAD']);
+
+    fs.unlinkSync(path.join(repoPath, 'b'));
+    await runGit(['add', 'b']);
+    await runGit(['commit', '-m', 'commit2']);
+    const commit2Sha = await runGit(['rev-parse', 'HEAD']);
+
+
+
+    // Checkout the first commit, which should contain 'a' and 'b'
+    t.false(fs.existsSync(path.join(tmpDir, repoName)));
     await externalQueries.checkoutExternalRepository(
-      "github/codeql-go",
-      ref,
-      'https://github.com',
+      repoName,
+      commit1Sha,
+      `file://${testRepoBaseDir}`,
       tmpDir,
       getRunnerLogger(true));
+    t.true(fs.existsSync(path.join(tmpDir, repoName)));
+    t.true(fs.existsSync(path.join(tmpDir, repoName, commit1Sha)));
+    t.true(fs.existsSync(path.join(tmpDir, repoName, commit1Sha, 'a')));
+    t.true(fs.existsSync(path.join(tmpDir, repoName, commit1Sha, 'b')));
 
-    // COPYRIGHT file existed in df4c6869212341b601005567381944ed90906b6b but not in the default branch
-    t.true(fs.existsSync(path.join(tmpDir, "github", "codeql-go", ref, "COPYRIGHT")));
+
+
+    // Checkout the second commit as well, which should only contain 'a'
+    t.false(fs.existsSync(path.join(tmpDir, repoName, commit2Sha)));
+    await externalQueries.checkoutExternalRepository(
+      repoName,
+      commit2Sha,
+      `file://${testRepoBaseDir}`,
+      tmpDir,
+      getRunnerLogger(true));
+    t.true(fs.existsSync(path.join(tmpDir, repoName, commit2Sha)));
+    t.true(fs.existsSync(path.join(tmpDir, repoName, commit2Sha, 'a')));
+    t.false(fs.existsSync(path.join(tmpDir, repoName, commit2Sha, 'b')));
   });
 });

src/external-queries.ts

@@ -24,7 +24,7 @@ export async function checkoutExternalRepository(
   }
 
   if (!fs.existsSync(checkoutLocation)) {
-    const repoURL = githubUrl + '/' + repository + '.git';
+    const repoURL = githubUrl + '/' + repository;
     await new toolrunnner.ToolRunner('git', ['clone', repoURL, checkoutLocation]).exec();
     await new toolrunnner.ToolRunner('git', [
       '--work-tree=' + checkoutLocation,

src/fingerprints.test.ts

@@ -111,6 +111,20 @@ test('hash', (t: ava.Assertions) => {
       "cc97dc7b1d7d8f7b:1",
       "c129715d7a2bc9a3:1"
     ]);
+
+  testHash(
+    t,
+    "x = 2\nx = 1\nprint(x)\nx = 3\nprint(x)\nx = 4\nprint(x)\n",
+    [
+      "e54938cc54b302f1:1",
+      "bb609acbe9138d60:1",
+      "1131fd5871777f34:1",
+      "5c482a0f8b35ea28:1",
+      "54517377da7028d2:1",
+      "2c644846cb18d53e:1",
+      "f1b89f20de0d133:1",
+      "c129715d7a2bc9a3:1"
+    ]);
 });
 
 function testResolveUriToFile(uri: any, index: any, artifactsURIs: any[]) {