# This is a combination of 4 commits.

# This is the 1st commit message:

Add logic to download codeql platform-language pkg

* Add `bundleName` argument to `getCodeQLBundleDownloadURL`
* Add `languages` argument to `setupCodeQL`.

The logic now tries to find the platform-language pkg before defaulting
to the full bundle. We keep the toolcache clean by adding the pl version
to the tool version.

# The commit message #2 will be skipped:

# Add simple fallback logic for download

# The commit message #3 will be skipped:

# wip linter

# The commit message #4 will be skipped:

# linter
This commit is contained in:
Marco Gario
2020-09-09 13:51:54 +02:00
parent 7951f91d00
commit 3f5bb98d7e
21 changed files with 261 additions and 226 deletions
+25 -12
View File
@@ -4,23 +4,32 @@ import nock from 'nock';
import * as path from 'path';
import * as codeql from './codeql';
import { Language } from './languages';
import { getRunnerLogger } from './logging';
import {setupTests} from './testing-utils';
import * as util from './util';
setupTests(test);
test('download codeql bundle cache', async t => {
await util.withTmpDir(async tmpDir => {
const versions = ['20200601', '20200610'];
const plVersions = ['linux64-cpp', undefined];
const languages: Language[][] = [
[Language.cpp],
[Language.cpp, Language.python] // Multi-language requires the full bundle
];
for (let i=0; i < versions.length; i++) {
for (let j=0; j < plVersions.length; j++) {
const platform = process.platform === 'win32' ? 'win64' :
process.platform === 'linux' ? 'linux64' :
process.platform === 'darwin' ? 'osx64' : undefined;
for (let i = 0; i < versions.length; i++) {
for (let j = 0; j < languages.length; j++) {
const version = versions[i];
const plVersion = plVersions[j];
const pkg = plVersion ? `codeql-${plVersion}.tar.gz` : `codeql-bundle.tar.gz`
const plVersion = (languages[j].length === 1) ? `${platform}-${languages[j][0]}` : undefined;
const pkg = plVersion ? `codeql-${plVersion}.tar.gz` : `codeql-bundle.tar.gz`;
nock('https://example.com')
.get(`/download/codeql-bundle-${version}/${pkg}`)
@@ -28,7 +37,7 @@ test('download codeql bundle cache', async t => {
await codeql.setupCodeQL(
`https://example.com/download/codeql-bundle-${version}/codeql-bundle.tar.gz`,
plVersion,
languages[j],
'token',
'https://github.example.com',
tmpDir,
@@ -37,7 +46,6 @@ test('download codeql bundle cache', async t => {
getRunnerLogger(true));
const toolcacheVersion = plVersion ? `0.0.0-${version}-${plVersion}` : `0.0.0-${version}`;
console.debug(toolcacheVersion)
t.assert(toolcache.find('CodeQL', toolcacheVersion), `Looking for ${toolcacheVersion} - ${plVersion}`);
}
}
@@ -53,7 +61,6 @@ test('use codeql bundle cache if pl version is not available', async t => {
// If we look for a pl version but find in cache the bundle, we use the bundle
await util.withTmpDir(async tmpDir => {
const version = '20200601';
const plVersion = 'linux64-cpp';
nock('https://example.com')
.get(`/download/codeql-bundle-${version}/codeql-bundle.tar.gz`)
@@ -61,7 +68,7 @@ test('use codeql bundle cache if pl version is not available', async t => {
await codeql.setupCodeQL(
`https://example.com/download/codeql-bundle-${version}/codeql-bundle.tar.gz`,
undefined,
[],
'token',
'https://github.example.com',
tmpDir,
@@ -72,10 +79,10 @@ test('use codeql bundle cache if pl version is not available', async t => {
t.assert(toolcache.find('CodeQL', `0.0.0-${version}`));
t.is(toolcache.findAllVersions('CodeQL').length, 1);
// Now try to request the plVersion, and see that we do not change the cache
// Now try to request the cpp version, and see that we do not change the cache
await codeql.setupCodeQL(
`https://example.com/download/codeql-bundle-${version}/codeql-bundle.tar.gz`,
plVersion,
[Language.cpp],
'token',
'https://github.example.com',
tmpDir,
@@ -89,6 +96,12 @@ test('use codeql bundle cache if pl version is not available', async t => {
});
});
// test('use larger bundles if smaller ones are unavailble', async t => {
// // TODO: This should check the fallback behavior of getCodeQLBundleDownloadURL
// t.fail()
// });
test('parse codeql bundle url version', t => {
+33 -15
View File
@@ -127,7 +127,7 @@ function getCodeQLActionRepository(mode: util.Mode): string {
}
async function getCodeQLBundleDownloadURL(
bundleName: string,
bundleNames: string[],
githubAuth: string,
githubUrl: string,
mode: util.Mode,
@@ -158,10 +158,13 @@ async function getCodeQLBundleDownloadURL(
repo: repositoryName,
tag: CODEQL_BUNDLE_VERSION
});
for (let asset of release.data.assets) {
if (asset.name === bundleName) {
logger.info(`Found CodeQL bundle in ${downloadSource[1]} on ${downloadSource[0]} with URL ${asset.url}.`);
return asset.url;
// See if any of the bundles appears in the assets list
for (let bundleName of bundleNames) {
for (let asset of release.data.assets) {
if (asset.name === bundleName) {
logger.info(`Found CodeQL bundle in ${downloadSource[1]} on ${downloadSource[0]} with URL ${asset.url}.`);
return asset.url;
}
}
}
} catch (e) {
@@ -194,7 +197,7 @@ async function toolcacheDownloadTool(
export async function setupCodeQL(
codeqlURL: string | undefined,
plVersion: string | undefined,
languages: Language[],
githubAuth: string,
githubUrl: string,
tempDir: string,
@@ -208,6 +211,22 @@ export async function setupCodeQL(
process.env['RUNNER_TEMP'] = tempDir;
process.env['RUNNER_TOOL_CACHE'] = toolsDir;
// Compute package version
let plVersion: string | undefined = undefined;
let platform: string;
if (process.platform === 'win32') {
platform = "win64";
} else if (process.platform === 'linux') {
platform = "linux64";
} else if (process.platform === 'darwin') {
platform = "osx64";
} else {
throw new Error("Unsupported platform: " + process.platform);
}
if (languages.length === 1) {
plVersion = `${platform}-${languages[0]}`;
}
try {
// The URL identifies the release version. E.g., codeql-20200901 .
// The plVersion identifies the platform-language combination of the package
@@ -221,7 +240,6 @@ export async function setupCodeQL(
logger.debug(`PL Version ${plVersion}`);
if (plVersion) {
codeqlFolder = toolcache.find('CodeQL', `${codeqlURLVersion}-${plVersion}`);
if (codeqlFolder) {
logger.debug(`CodeQL found in cache ${codeqlFolder}`);
@@ -236,15 +254,15 @@ export async function setupCodeQL(
}
if (!codeqlFolder) {
const codeqlToolcacheVersion = plVersion ? `${codeqlURLVersion}-${plVersion}`: codeqlURLVersion;
const codeqlToolcacheVersion = plVersion ? `${codeqlURLVersion}-${plVersion}` : codeqlURLVersion;
logger.debug(`CodeQL not found in cache`);
if (!codeqlURL) {
let pkgName = plVersion ? CODEQL_BUNDLE_NAME.replace("-bundle", `-${plVersion}`) : CODEQL_BUNDLE_NAME; // TODO : Maybe move template a constant?
codeqlURL = await getCodeQLBundleDownloadURL(pkgName, githubAuth, githubUrl, mode, logger);
}
else if (plVersion) {
let pkgName = CODEQL_BUNDLE_NAME.replace("-bundle", `-${plVersion}`)
codeqlURL = codeqlURL.replace(CODEQL_BUNDLE_NAME, pkgName)
const bundles = [
CODEQL_BUNDLE_NAME.replace("-bundle", `-bundle-${plVersion}`),
CODEQL_BUNDLE_NAME.replace("-bundle", `-bundle-${platform}`),
CODEQL_BUNDLE_NAME,
];
codeqlURL = await getCodeQLBundleDownloadURL(bundles, githubAuth, githubUrl, mode, logger);
}
logger.debug(`Using CodeQL URL: ${codeqlURL}`);
@@ -263,7 +281,7 @@ export async function setupCodeQL(
logger.debug(`CodeQL bundle download to ${codeqlPath} complete.`);
const codeqlExtracted = await toolcache.extractTar(codeqlPath);
logger.debug(`Caching ${codeqlToolcacheVersion}`)
logger.debug(`Caching ${codeqlToolcacheVersion}`);
codeqlFolder = await toolcache.cacheDir(codeqlExtracted, 'CodeQL', codeqlToolcacheVersion);
}
+40 -73
View File
@@ -43,7 +43,7 @@ function mockListLanguages(languages: string[]) {
test("load empty config", async t => {
return await util.withTmpDir(async tmpDir => {
const logger = getRunnerLogger(true);
const languages = 'javascript,python';
const languages = [Language.javascript, Language.python];
const codeQL = setCodeQL({
resolveQueries: async function() {
@@ -59,7 +59,6 @@ test("load empty config", async t => {
languages,
undefined,
undefined,
{ owner: 'github', repo: 'example '},
tmpDir,
tmpDir,
codeQL,
@@ -71,12 +70,10 @@ test("load empty config", async t => {
t.deepEqual(config, await configUtils.getDefaultConfig(
languages,
undefined,
{ owner: 'github', repo: 'example '},
tmpDir,
tmpDir,
codeQL,
tmpDir,
'token',
'https://github.example.com',
logger));
});
@@ -104,10 +101,9 @@ test("loading config saves config", async t => {
t.deepEqual(await configUtils.getConfig(tmpDir, logger), undefined);
const config1 = await configUtils.initConfig(
'javascript,python',
[Language.javascript, Language.python],
undefined,
undefined,
{ owner: 'github', repo: 'example '},
tmpDir,
tmpDir,
codeQL,
@@ -129,10 +125,9 @@ test("load input outside of workspace", async t => {
return await util.withTmpDir(async tmpDir => {
try {
await configUtils.initConfig(
undefined,
[],
undefined,
'../input',
{ owner: 'github', repo: 'example '},
tmpDir,
tmpDir,
getCachedCodeQL(),
@@ -154,10 +149,9 @@ test("load non-local input with invalid repo syntax", async t => {
try {
await configUtils.initConfig(
undefined,
[],
undefined,
configFile,
{ owner: 'github', repo: 'example '},
tmpDir,
tmpDir,
getCachedCodeQL(),
@@ -174,7 +168,7 @@ test("load non-local input with invalid repo syntax", async t => {
test("load non-existent input", async t => {
return await util.withTmpDir(async tmpDir => {
const languages = 'javascript';
const languages = [Language.javascript];
const configFile = 'input';
t.false(fs.existsSync(path.join(tmpDir, configFile)));
@@ -183,7 +177,6 @@ test("load non-existent input", async t => {
languages,
undefined,
configFile,
{ owner: 'github', repo: 'example '},
tmpDir,
tmpDir,
getCachedCodeQL(),
@@ -247,7 +240,7 @@ test("load non-empty input", async t => {
codeQLCmd: codeQL.getPath(),
};
const languages = 'javascript';
const languages = [Language.javascript];
const configFile = 'input';
fs.writeFileSync(path.join(tmpDir, configFile), inputFileContents, 'utf8');
@@ -255,7 +248,6 @@ test("load non-empty input", async t => {
languages,
undefined,
configFile,
{ owner: 'github', repo: 'example '},
tmpDir,
tmpDir,
codeQL,
@@ -302,7 +294,7 @@ test("default queries are used", async t => {
fs.mkdirSync(path.join(tmpDir, 'foo'));
const languages = 'javascript';
const languages = [Language.javascript];
const configFile = 'input';
fs.writeFileSync(path.join(tmpDir, configFile), inputFileContents, 'utf8');
@@ -310,7 +302,6 @@ test("default queries are used", async t => {
languages,
undefined,
configFile,
{ owner: 'github', repo: 'example '},
tmpDir,
tmpDir,
codeQL,
@@ -357,13 +348,12 @@ test("Queries can be specified in config file", async t => {
},
});
const languages = 'javascript';
const languages = [Language.javascript];
const config = await configUtils.initConfig(
languages,
undefined,
configFile,
{ owner: 'github', repo: 'example '},
tmpDir,
tmpDir,
codeQL,
@@ -421,13 +411,12 @@ test("Queries from config file can be overridden in workflow file", async t => {
},
});
const languages = 'javascript';
const languages = [Language.javascript];
const config = await configUtils.initConfig(
languages,
queries,
configFile,
{ owner: 'github', repo: 'example '},
tmpDir,
tmpDir,
codeQL,
@@ -476,13 +465,12 @@ test("Multiple queries can be specified in workflow file, no config file require
},
});
const languages = 'javascript';
const languages = [Language.javascript];
const config = await configUtils.initConfig(
languages,
queries,
undefined,
{ owner: 'github', repo: 'example '},
tmpDir,
tmpDir,
codeQL,
@@ -511,7 +499,7 @@ test("Multiple queries can be specified in workflow file, no config file require
test("Invalid queries in workflow file handled correctly", async t => {
return await util.withTmpDir(async tmpDir => {
const queries = 'foo/bar@v1@v3';
const languages = 'javascript';
const languages = [Language.javascript];
// This function just needs to be type-correct; it doesn't need to do anything,
// since we're deliberately passing in invalid data
@@ -532,7 +520,6 @@ test("Invalid queries in workflow file handled correctly", async t => {
languages,
queries,
undefined,
{ owner: 'github', repo: 'example '},
tmpDir,
tmpDir,
codeQL,
@@ -584,13 +571,12 @@ test("API client used when reading remote config", async t => {
fs.mkdirSync(path.join(tmpDir, 'foo/bar/dev'), { recursive: true });
const configFile = 'octo-org/codeql-config/config.yaml@main';
const languages = 'javascript';
const languages = [Language.javascript];
await configUtils.initConfig(
languages,
undefined,
configFile,
{ owner: 'github', repo: 'example '},
tmpDir,
tmpDir,
codeQL,
@@ -610,10 +596,9 @@ test("Remote config handles the case where a directory is provided", async t =>
const repoReference = 'octo-org/codeql-config/config.yaml@main';
try {
await configUtils.initConfig(
undefined,
[],
undefined,
repoReference,
{ owner: 'github', repo: 'example '},
tmpDir,
tmpDir,
getCachedCodeQL(),
@@ -638,10 +623,9 @@ test("Invalid format of remote config handled correctly", async t => {
const repoReference = 'octo-org/codeql-config/config.yaml@main';
try {
await configUtils.initConfig(
undefined,
[],
undefined,
repoReference,
{ owner: 'github', repo: 'example '},
tmpDir,
tmpDir,
getCachedCodeQL(),
@@ -657,51 +641,35 @@ test("Invalid format of remote config handled correctly", async t => {
});
test("No detected languages", async t => {
return await util.withTmpDir(async tmpDir => {
mockListLanguages([]);
mockListLanguages([]);
try {
await configUtils.getLanguages(
undefined,
{ owner: 'github', repo: 'example ' },
'token',
'https://github.example.com',
getRunnerLogger(true));
throw new Error('initConfig did not throw error');
} catch (err) {
t.deepEqual(err, new Error(configUtils.getNoLanguagesError()));
}
try {
await configUtils.initConfig(
undefined,
undefined,
undefined,
{ owner: 'github', repo: 'example '},
tmpDir,
tmpDir,
getCachedCodeQL(),
tmpDir,
'token',
'https://github.example.com',
getRunnerLogger(true));
throw new Error('initConfig did not throw error');
} catch (err) {
t.deepEqual(err, new Error(configUtils.getNoLanguagesError()));
}
});
});
test("Unknown languages", async t => {
return await util.withTmpDir(async tmpDir => {
const languages = 'ruby,english';
try {
await configUtils.initConfig(
languages,
undefined,
undefined,
{ owner: 'github', repo: 'example '},
tmpDir,
tmpDir,
getCachedCodeQL(),
tmpDir,
'token',
'https://github.example.com',
getRunnerLogger(true));
throw new Error('initConfig did not throw error');
} catch (err) {
t.deepEqual(err, new Error(configUtils.getUnknownLanguagesError(['ruby', 'english'])));
}
});
const languages = 'ruby,english';
try {
await configUtils.getLanguages(
languages,
{ owner: 'github', repo: 'example ' },
'token',
'https://github.example.com',
getRunnerLogger(true));
throw new Error('initConfig did not throw error');
} catch (err) {
t.deepEqual(err, new Error(configUtils.getUnknownLanguagesError(['ruby', 'english'])));
}
});
function doInvalidInputTest(
@@ -721,7 +689,7 @@ function doInvalidInputTest(
},
});
const languages = 'javascript';
const languages = [Language.javascript];
const configFile = 'input';
const inputFile = path.join(tmpDir, configFile);
fs.writeFileSync(inputFile, inputFileContents, 'utf8');
@@ -731,7 +699,6 @@ function doInvalidInputTest(
languages,
undefined,
configFile,
{ owner: 'github', repo: 'example '},
tmpDir,
tmpDir,
codeQL,
+9 -27
View File
@@ -5,7 +5,7 @@ import * as path from 'path';
import * as api from './api-client';
import { CodeQL, ResolveQueriesOutput } from './codeql';
import * as externalQueries from "./external-queries";
import { Language, parseLanguage } from "./languages";
import { Language, parseLanguage } from './languages';
import { Logger } from './logging';
import { RepositoryNwo } from './repository';
@@ -491,7 +491,8 @@ async function getLanguagesInRepo(
* If no languages could be detected from either the workflow or the repository
* then throw an error.
*/
async function getLanguages(
export async function getLanguages(
// Maybe move to init.ts
languagesInput: string | undefined,
repository: RepositoryNwo,
githubAuth: string,
@@ -570,23 +571,15 @@ async function addQueriesFromWorkflow(
* Get the default config for when the user has not supplied one.
*/
export async function getDefaultConfig(
languagesInput: string | undefined,
languages: Language[],
queriesInput: string | undefined,
repository: RepositoryNwo,
tempDir: string,
toolCacheDir: string,
codeQL: CodeQL,
checkoutPath: string,
githubAuth: string,
githubUrl: string,
logger: Logger): Promise<Config> {
const languages = await getLanguages(
languagesInput,
repository,
githubAuth,
githubUrl,
logger);
const queries = {};
await addDefaultQueries(codeQL, languages, queries);
if (queriesInput) {
@@ -617,10 +610,9 @@ export async function getDefaultConfig(
* Load the config from the given file.
*/
async function loadConfig(
languagesInput: string | undefined,
languages: Language[],
queriesInput: string | undefined,
configFile: string,
repository: RepositoryNwo,
tempDir: string,
toolCacheDir: string,
codeQL: CodeQL,
@@ -653,13 +645,6 @@ async function loadConfig(
}
}
const languages = await getLanguages(
languagesInput,
repository,
githubAuth,
githubUrl,
logger);
const queries = {};
const pathsIgnore: string[] = [];
const paths: string[] = [];
@@ -760,10 +745,9 @@ async function loadConfig(
* a default config. The parsed config is then stored to a known location.
*/
export async function initConfig(
languagesInput: string | undefined,
languages: Language[],
queriesInput: string | undefined,
configFile: string | undefined,
repository: RepositoryNwo,
tempDir: string,
toolCacheDir: string,
codeQL: CodeQL,
@@ -778,22 +762,19 @@ export async function initConfig(
if (!configFile) {
logger.debug('No configuration file was provided');
config = await getDefaultConfig(
languagesInput,
languages,
queriesInput,
repository,
tempDir,
toolCacheDir,
codeQL,
checkoutPath,
githubAuth,
githubUrl,
logger);
} else {
config = await loadConfig(
languagesInput,
languages,
queriesInput,
configFile,
repository,
tempDir,
toolCacheDir,
codeQL,
@@ -896,3 +877,4 @@ export async function getConfig(tempDir: string, logger: Logger): Promise<Config
logger.debug(configString);
return JSON.parse(configString);
}
+11 -2
View File
@@ -57,20 +57,29 @@ async function run() {
if (!await util.sendStatusReport(await util.createStatusReportBase('init', 'starting', startedAt), true)) {
return;
}
const repositoryNWO = parseRepositoryNwo(util.getRequiredEnvParam('GITHUB_REPOSITORY'));
const languages = await configUtils.getLanguages(
core.getInput('languages'),
repositoryNWO,
core.getInput('token'),
util.getRequiredEnvParam('GITHUB_SERVER_URL'),
logger);
codeql = await initCodeQL(
core.getInput('tools'),
languages,
core.getInput('token'),
util.getRequiredEnvParam('GITHUB_SERVER_URL'),
util.getRequiredEnvParam('RUNNER_TEMP'),
util.getRequiredEnvParam('RUNNER_TOOL_CACHE'),
'actions',
logger);
config = await initConfig(
core.getInput('languages'),
languages,
core.getInput('queries'),
core.getInput('config-file'),
parseRepositoryNwo(util.getRequiredEnvParam('GITHUB_REPOSITORY')),
util.getRequiredEnvParam('RUNNER_TEMP'),
util.getRequiredEnvParam('RUNNER_TOOL_CACHE'),
codeql,
+7 -6
View File
@@ -5,13 +5,15 @@ import * as path from 'path';
import * as analysisPaths from './analysis-paths';
import { CodeQL, setupCodeQL } from './codeql';
import * as configUtils from './config-utils';
import {Language} from './languages';
import { Logger } from './logging';
import { RepositoryNwo } from './repository';
import { getCombinedTracerConfig, TracerConfig } from './tracer-config';
import * as util from './util';
export async function initCodeQL(
codeqlURL: string | undefined,
languages: Language[],
githubAuth: string,
githubUrl: string,
tempDir: string,
@@ -20,9 +22,10 @@ export async function initCodeQL(
logger: Logger): Promise<CodeQL> {
logger.startGroup('Setup CodeQL tools');
const codeql = await setupCodeQL(
codeqlURL,
undefined, // MG: FIXME
languages,
githubAuth,
githubUrl,
tempDir,
@@ -35,10 +38,9 @@ export async function initCodeQL(
}
export async function initConfig(
languagesInput: string | undefined,
languages: Language[],
queriesInput: string | undefined,
configFile: string | undefined,
repository: RepositoryNwo,
tempDir: string,
toolCacheDir: string,
codeQL: CodeQL,
@@ -49,10 +51,9 @@ export async function initConfig(
logger.startGroup('Load language configuration');
const config = await configUtils.initConfig(
languagesInput,
languages,
queriesInput,
configFile,
repository,
tempDir,
toolCacheDir,
codeQL,
+15 -5
View File
@@ -6,7 +6,7 @@ import * as path from 'path';
import { runAnalyze } from './analyze';
import { determineAutobuildLanguage, runAutobuild } from './autobuild';
import { CodeQL, getCodeQL } from './codeql';
import { Config, getConfig } from './config-utils';
import { Config, getConfig, getLanguages } from './config-utils';
import { initCodeQL, initConfig, injectWindowsTracer, runInit } from './init';
import { Language, parseLanguage } from './languages';
import { getRunnerLogger } from './logging';
@@ -139,14 +139,25 @@ program
fs.rmdirSync(tempDir, { recursive: true });
fs.mkdirSync(tempDir, { recursive: true });
const githubUrl = parseGithubUrl(cmd.githubUrl);
const repositoryNWO = parseRepositoryNwo(cmd.repository);
const languages = await getLanguages(
cmd.languages,
repositoryNWO,
cmd.githubAuth,
githubUrl,
logger);
let codeql: CodeQL;
if (cmd.codeqlPath !== undefined) {
codeql = getCodeQL(cmd.codeqlPath);
} else {
codeql = await initCodeQL(
undefined,
languages,
cmd.githubAuth,
parseGithubUrl(cmd.githubUrl),
githubUrl,
tempDir,
toolsDir,
'runner',
@@ -154,16 +165,15 @@ program
}
const config = await initConfig(
cmd.languages,
languages,
cmd.queries,
cmd.configFile,
parseRepositoryNwo(cmd.repository),
tempDir,
toolsDir,
codeql,
cmd.checkoutPath || process.cwd(),
cmd.githubAuth,
parseGithubUrl(cmd.githubUrl),
githubUrl,
logger);
const tracerConfig = await runInit(codeql, config);