Skip to content

Commit

Permalink
Update tool to upload multiple files (#20)
Browse files Browse the repository at this point in the history 
Rename `file` property to `files` and update endpoint to upload multiple files according to pixee platform changes

It will retrieve sonar issues/hotspots by pagination if the results exceed the max page limit and upload multiple sonar files
  • Loading branch information
@carlosu7
carlosu7 authored Jul 1, 2024
1 parent 5c7fcc8 commit 238f1ae
Show file tree
Hide file tree
Showing 6 changed files with 136 additions and 52 deletions.
18 changes: 6 additions & 12 deletions __tests__/action.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ import * as github from "../src/github";
let getInputMock: jest.SpiedFunction<typeof core.getInput>;
let getGitHubContextMock: jest.SpiedFunction<typeof github.getGitHubContext>;
let getTempDir: jest.SpiedFunction<typeof github.getTempDir>;
let uploadInputFileMock: jest.SpiedFunction<typeof pixee.uploadInputFile>;
let uploadInputFileMock: jest.SpiedFunction<typeof pixee.uploadInputFiles>;
let retrieveSonarIssuesMock: jest.SpiedFunction<
typeof sonar.retrieveSonarIssues
>;
Expand All @@ -30,7 +30,7 @@ describe("action", () => {
.mockImplementation()
.mockReturnValue(os.tmpdir());
uploadInputFileMock = jest
.spyOn(pixee, "uploadInputFile")
.spyOn(pixee, "uploadInputFiles")
.mockImplementation();
triggerPrAnalysisMock = jest
.spyOn(pixee, "triggerPrAnalysis")
Expand Down Expand Up @@ -100,10 +100,7 @@ describe("action", () => {

await run();

expect(uploadInputFileMock).toHaveBeenCalledWith(
"sonar_issues",
"file.json",
);
expect(uploadInputFileMock).toHaveBeenCalledWith("sonar_issues", new Array("file.json"));
});

it("should upload the given semgrep file", async () => {
Expand All @@ -129,7 +126,7 @@ describe("action", () => {

await run();

expect(uploadInputFileMock).toHaveBeenCalledWith("semgrep", "file.json");
expect(uploadInputFileMock).toHaveBeenCalledWith("semgrep", new Array("file.json"));
});

it("should upload the given snyk sarif file", async () => {
Expand All @@ -155,7 +152,7 @@ describe("action", () => {

await run();

expect(uploadInputFileMock).toHaveBeenCalledWith("snyk", "file.json");
expect(uploadInputFileMock).toHaveBeenCalledWith("snyk", ["file.json"]);
});
});

Expand Down Expand Up @@ -206,10 +203,7 @@ describe("action", () => {

expect(retrieveSonarIssuesMock).toHaveBeenCalled();
expect(retrieveSonarHotspotsMock).toHaveBeenCalled();
expect(uploadInputFileMock).toHaveBeenCalledWith(
"sonar_issues",
expect.stringMatching(/sonar-issues.json$/),
);
expect(uploadInputFileMock).toHaveBeenCalled();
});
});
});
34 changes: 32 additions & 2 deletions __tests__/pixee-platform.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@ import fs from "fs";
import * as core from "@actions/core";
import * as tmp from "tmp";
import * as github from "../src/github";
import { uploadInputFile } from "../src/pixee-platform";
import { uploadInputFiles } from "../src/pixee-platform";
import axios from "axios";

let getInputMock: jest.SpiedFunction<typeof core.getInput>;
Expand Down Expand Up @@ -38,7 +38,37 @@ describe("pixee-platform", () => {
}
});

await uploadInputFile("sonar_issues", file.name);
await uploadInputFiles("sonar_issues", new Array(file.name));

expect(axios.put).toHaveBeenCalledWith(
"https://api.pixee.ai/analysis-input/owner/repo/sha/sonar_issues",
expect.anything(), // cannot assert the form content
{
headers: {
Authorization: "Bearer token", // Assert the authorization header
"content-type": expect.stringContaining("multipart/form-data"), // Assert the content type header
},
},
);
});

it("uploads input files", async () => {
const file1 = tmp.fileSync();
fs.writeFileSync(file1.name, "{}");
const file2 = tmp.fileSync();
fs.writeFileSync(file2.name, "{}");
// mock axios.put to avoid making a real HTTP request
jest.spyOn(axios, "put").mockResolvedValue(undefined);
getInputMock.mockImplementation((name: string) => {
switch (name) {
case "pixee-api-url":
return "https://api.pixee.ai";
default:
return "";
}
});

await uploadInputFiles("sonar_issues", new Array(file1.name, file2.name));

expect(axios.put).toHaveBeenCalledWith(
"https://api.pixee.ai/analysis-input/owner/repo/sha/sonar_issues",
Expand Down
24 changes: 18 additions & 6 deletions __tests__/sonar.test.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,10 +36,12 @@ describe("sonar", () => {
sonarInputs,
path,
queryParamKey: "componentKeys",
pageSize: 500,
page: 1
});

expect(result).toBe(
"https://sonar.io/api/issues/search?componentKeys=myComponent&resolved=false&ps=500&pullRequest=123",
"https://sonar.io/api/issues/search?componentKeys=myComponent&resolved=false&ps=500&p=1&pullRequest=123",
);
});

Expand All @@ -55,10 +57,12 @@ describe("sonar", () => {
sonarInputs,
path,
queryParamKey: "componentKeys",
pageSize: 500,
page: 1
});

expect(result).toBe(
"https://sonar.io/api/issues/search?componentKeys=myComponent&resolved=false&ps=500",
"https://sonar.io/api/issues/search?componentKeys=myComponent&resolved=false&ps=500&p=1",
);
});

Expand All @@ -69,10 +73,12 @@ describe("sonar", () => {
sonarInputs,
path,
queryParamKey: "projectKey",
pageSize: 500,
page: 1
});

expect(result).toBe(
"https://sonar.io/api/issues/search?projectKey=myComponent&resolved=false&ps=500&pullRequest=123",
"https://sonar.io/api/issues/search?projectKey=myComponent&resolved=false&ps=500&p=1&pullRequest=123",
);
});

Expand All @@ -90,10 +96,12 @@ describe("sonar", () => {
sonarInputs,
path,
queryParamKey: "componentKeys",
pageSize: 500,
page: 1
});

expect(result).toBe(
"https://sonar.io/api/issues/search?componentKeys=myComponent+with+spaces&resolved=false&ps=500&pullRequest=123",
"https://sonar.io/api/issues/search?componentKeys=myComponent+with+spaces&resolved=false&ps=500&p=1&pullRequest=123",
);
});

Expand All @@ -111,10 +119,12 @@ describe("sonar", () => {
sonarInputs,
path,
queryParamKey: "componentKeys",
pageSize: 500,
page: 1
});

expect(result).toBe(
"https://sonar.io/api/issues/search?componentKeys=myComponent&resolved=false&ps=500&pullRequest=123",
"https://sonar.io/api/issues/search?componentKeys=myComponent&resolved=false&ps=500&p=1&pullRequest=123",
);
});

Expand All @@ -133,10 +143,12 @@ describe("sonar", () => {
sonarInputs,
path,
queryParamKey: "projectKey",
pageSize: 500,
page: 1
});

expect(result).toBe(
"https://sonar.io/context/api/issues/search?projectKey=myComponent&resolved=false&ps=500&pullRequest=123",
"https://sonar.io/context/api/issues/search?projectKey=myComponent&resolved=false&ps=500&p=1&pullRequest=123",
);
});
});
78 changes: 55 additions & 23 deletions src/action.ts
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import * as core from "@actions/core";
import fs from "fs";
import { Tool, getTool } from "./inputs";
import { triggerPrAnalysis, uploadInputFile } from "./pixee-platform";
import { triggerPrAnalysis, uploadInputFiles } from "./pixee-platform";
import {
SONAR_RESULT,
getSonarInputs,
Expand All @@ -12,6 +12,13 @@ import { getDefectDojoInputs, retrieveDefectDojoResults } from "./defect-dojo";
import { getGitHubContext, getTempDir } from "./github";
import { getContrastInputs, retrieveContrastResults } from "./contrast";

interface SonarResults {
totalResults: number;
results: any
}

const MAX_PAGE_SIZE = 500;

/**
* Runs the action.
*
Expand All @@ -25,30 +32,30 @@ export async function run() {
switch (tool) {
case "contrast":
const contrastFile = await fetchOrLocateContrastResultsFile();
await uploadInputFile(tool, contrastFile);
await uploadInputFiles(tool, new Array(contrastFile));
core.info(`Uploaded ${contrastFile} to Pixeebot for analysis`);
break;
case "defectdojo":
const file = await fetchOrLocateDefectDojoResultsFile();
await uploadInputFile(tool, file);
await uploadInputFiles(tool, new Array(file));
core.info(`Uploaded ${file} to Pixeebot for analysis`);
break;
case "sonar":
const issuesfile = await fetchOrLocateSonarResultsFile("issues");
await uploadInputFile("sonar_issues", issuesfile);
core.info(`Uploaded ${issuesfile} to Pixeebot for analysis`);
const issuesfiles = await fetchOrLocateSonarResultsFile("issues");
await uploadInputFiles("sonar_issues", issuesfiles);
core.info(`Uploaded ${issuesfiles} to Pixeebot for analysis`);

const hotspotFile = await fetchOrLocateSonarResultsFile("hotspots");
await uploadInputFile("sonar_hotspots", hotspotFile);
core.info(`Uploaded ${hotspotFile} to Pixeebot for analysis`);
const hotspotFiles = await fetchOrLocateSonarResultsFile("hotspots");
await uploadInputFiles("sonar_hotspots", hotspotFiles);
core.info(`Uploaded ${hotspotFiles} to Pixeebot for analysis`);
break;
default:
const inputFile = core.getInput("file");
if (!inputFile) {
throw new Error(`Tool "${tool}" requires a file input`);
}

await uploadInputFile(tool, inputFile);
await uploadInputFiles(tool, new Array(inputFile));
core.info(`Uploaded ${inputFile} for ${tool} to Pixeebot for analysis`);
}

Expand All @@ -73,14 +80,32 @@ async function fetchOrLocateContrastResultsFile() {
return fetchOrLocateResultsFile("contrast", results, fileName, false);
}

async function fetchOrLocateSonarResultsFile(resultType: SONAR_RESULT) {
let results =
resultType == "issues"
? await fetchSonarIssues()
: await fetchSonarHotspots();
let fileName = `sonar-${resultType}.json`;
async function fetchOrLocateSonarResultsFile(
resultType: SONAR_RESULT
): Promise<Array<string>> {

let page = 1;
const files = new Array();
let isAllResults = false;

while (!isAllResults) {
let sonarResults =
resultType == "issues"
? await fetchSonarIssues(MAX_PAGE_SIZE, page)
: await fetchSonarHotspots(MAX_PAGE_SIZE, page);
let fileName = `sonar-${resultType}-${page}.json`;

let file = await fetchOrLocateResultsFile("sonar", sonarResults.results, fileName);

let total = sonarResults.totalResults;

return fetchOrLocateResultsFile("sonar", results, fileName);
files.push(file);

isAllResults = page * MAX_PAGE_SIZE >= total;
page++;
}

return files;
}

async function fetchOrLocateResultsFile(
Expand All @@ -106,9 +131,13 @@ async function fetchOrLocateResultsFile(
return file;
}

async function fetchSonarIssues() {
async function fetchSonarIssues(
pageSize: number,
page: number
) : Promise<SonarResults>{
const sonarInputs = getSonarInputs();
const results = await retrieveSonarIssues(sonarInputs);
const results = await retrieveSonarIssues(sonarInputs, pageSize, page);

core.info(
`Found ${results.total} Sonar issues for component ${sonarInputs.componentKey}`,
);
Expand All @@ -118,17 +147,20 @@ async function fetchSonarIssues() {
);
}

return results;
return {results, totalResults: results.total};
}

async function fetchSonarHotspots() {
async function fetchSonarHotspots(
pageSize: number,
page: number
) : Promise<SonarResults>{
const sonarInputs = getSonarInputs();
const results = await retrieveSonarHotspots(sonarInputs);
const results = await retrieveSonarHotspots(sonarInputs, pageSize, page);
core.info(
`Found ${results.paging.total} Sonar hotspots for component ${sonarInputs.componentKey}`,
);

return results;
return {results, totalResults: results.paging.total};
}

async function fetchDefectDojoFindings() {
Expand Down
13 changes: 9 additions & 4 deletions src/pixee-platform.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,12 +5,17 @@ import FormData from "form-data";
import { TOOL_PATH } from "./inputs";
import { getGitHubContext, getRepositoryInfo } from "./github";

export async function uploadInputFile(tool: TOOL_PATH, file: string) {
const fileContent = fs.readFileSync(file, "utf-8");
export async function uploadInputFiles(tool: TOOL_PATH, files: Array<string>) {
const form = new FormData();
form.append("file", fileContent);
const pixeeUrl = core.getInput("pixee-api-url");

const path = require('path');

// Append each file to the form data
files.forEach(file => {
form.append("files", fs.readFileSync(file), path.basename(file));
});

const pixeeUrl = core.getInput("pixee-api-url");
const token = await core.getIDToken(pixeeUrl);
const url = buildUploadApiUrl(tool);

Expand Down
Loading

0 comments on commit 238f1ae

Please sign in to comment.