8000 persistence-elastic: handle es8 bulk response by conleth · Pull Request #3391 · secureCodeBox/secureCodeBox · GitHub
[go: up one dir, main page]
Skip to content

persistence-elastic: handle es8 bulk response #3391

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
J12934 merged 5 commits into from
Dec 10, 2025
Merged

persistence-elastic: handle es8 bulk response #3391

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
42281d4
persistence-elastic: handle es8 bulk response
conleth Nov 25, 2025
6594e76
Adding my details
conleth Nov 25, 2025
7618b71
Extra n in my name
conleth Nov 25, 2025
94b6871
Refactor elastic hook tests to reduce duplication
conleth Dec 9, 2025
9d39ea9
Merge branch 'secureCodeBox:main' into bugfix/persistence-elastic-es8…
conleth Dec 9, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CONTRIBUTORS.md
View file
Open in desktop
8000
Original file line number Diff line number Diff line change
Expand Up @@ -61,3 +61,4 @@ Committing with `git commit -s` will add the sign-off at the end of the commit m
- Kai Schäfer <kai.schaefer@claranet.com>
- Joel Saß <joel.sass@iteratec.com>
- Patrick Weiss <patrick.weiss@iteratec.com>
- Conleth Kennedy <conlethkennedy@gmail.com>
5 changes: 3 additions & 2 deletions hooks/persistence-elastic/hook/hook.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -105,9 +105,10 @@ export async function handle({
},
]);

const { body: bulkResponse } = await client.bulk({ refresh: true, body });
const bulkResponseRaw = await client.bulk({ refresh: true, body });
const bulkResponse = bulkResponseRaw?.body ?? bulkResponseRaw;

if (bulkResponse.errors) {
if (bulkResponse?.errors) {
console.error("Bulk Request had errors:");
console.log(bulkResponse);
}
Expand Down
159 changes: 72 additions & 87 deletions hooks/persistence-elastic/hook/hook.test.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
import { handle } from "./hook";

let elasticClient;
const buildGetFindings = (findings) => async () => findings;

beforeEach(() => {
elasticClient = {
Expand Down Expand Up @@ -32,10 +33,36 @@ const scan = {

const testDate = new Date("2020-11-11");

const scanDocumentBody = {
"@timestamp": testDate,
id: scan.metadata.uid,
labels: scan.metadata.labels,
name: scan.metadata.name,
parameters: scan.spec.parameters,
scan_type: scan.spec.scanType,
type: "scan",
};

const expectScanIndexCalledWith = (index, client = elasticClient) => {
expect(client.index).toHaveBeenCalledTimes(1);
expect(client.index).toHaveBeenCalledWith({
body: scanDocumentBody,
index,
});
};

const findingsWithOpenPort = [
{
id: "4560b3e6-1219-4f5f-9b44-6579f5a32407",
name: "Port 5601 is open",
category: "Open Port",
},
];

test(&quo 10000 t;should only send scan summary document if no findings are passing in", async () => {
const findings = [];

const getFindings = async () => findings;
const getFindings = buildGetFindings(findings);

await handle({
getFindings,
Expand All @@ -46,34 +73,12 @@ test("should only send scan summary document if no findings are passing in", asy
client: elasticClient,
});

expect(elasticClient.index).toHaveBeenCalledTimes(1);
expect(elasticClient.index).toHaveBeenCalledWith({
body: {
"@timestamp": testDate,
id: "09988cdf-1fc7-4f85-95ee-1b1d65dbc7cc",
labels: {
company: "iteratec",
},
name: "demo-scan",
parameters: ["-Pn", "localhost"],
scan_type: "Nmap",
type: "scan",
},
index: `scb_default_2020-11-11`,
});
expectScanIndexCalledWith(`scb_default_2020-11-11`);
expect(elasticClient.bulk).not.toHaveBeenCalled();
});

test("should send findings to elasticsearch with given prefix", async () => {
const findings = [
{
id: "4560b3e6-1219-4f5f-9b44-6579f5a32407",
name: "Port 5601 is open",
category: "Open Port",
},
];

const getFindings = async () => findings;
const getFindings = buildGetFindings(findingsWithOpenPort);

await handle({
getFindings,
Expand All @@ -85,21 +90,7 @@ test("should send findings to elasticsearch with given prefix", async () => {
client: elasticClient,
});

expect(elasticClient.index).toHaveBeenCalledTimes(1);
expect(elasticClient.index).toHaveBeenCalledWith({
body: {
"@timestamp": testDate,
id: "09988cdf-1fc7-4f85-95ee-1b1d65dbc7cc",
labels: {
company: "iteratec",
},
name: "demo-scan",
parameters: ["-Pn", "localhost"],
scan_type: "Nmap",
type: "scan",
},
index: `myPrefix_default_2020-11-11`,
});
expectScanIndexCalledWith(`myPrefix_default_2020-11-11`);

expect(elasticClient.bulk).toHaveBeenCalledTimes(1);
expect(elasticClient.bulk).toHaveBeenCalledWith({
Expand Down Expand Up @@ -130,7 +121,7 @@ test("should send findings to elasticsearch with given prefix", async () => {
test("should not append namespace if 'appendNamespace' is null", async () => {
const findings = [];

const getFindings = async () => findings;
const getFindings = buildGetFindings(findings);

await handle({
getFindings,
Expand All @@ -140,27 +131,13 @@ test("should not append namespace if 'appendNamespace' is null", async () => {
client: elasticClient,
});

expect(elasticClient.index).toBeCalledTimes(1);
expect(elasticClient.index).toBeCalledWith({
body: {
"@timestamp": testDate,
id: "09988cdf-1fc7-4f85-95ee-1b1d65dbc7cc",
labels: {
company: "iteratec",
},
name: "demo-scan",
parameters: ["-Pn", "localhost"],
scan_type: "Nmap",
type: "scan",
},
index: `scb_2020-11-11`,
});
expectScanIndexCalledWith(`scb_2020-11-11`);
});

test("should append date format yyyy", async () => {
const findings = [];

const getFindings = async () => findings;
const getFindings = buildGetFindings(findings);

await handle({
getFindings,
Expand All @@ -171,27 +148,13 @@ test("should append date format yyyy", async () => {
client: elasticClient,
});

expect(elasticClient.index).toBeCalledTimes(1);
expect(elasticClient.index).toBeCalledWith({
body: {
"@timestamp": testDate,
id: "09988cdf-1fc7-4f85-95ee-1b1d65dbc7cc",
labels: {
company: "iteratec",
},
name: "demo-scan",
parameters: ["-Pn", "localhost"],
scan_type: "Nmap",
type: "scan",
},
index: `scb_2020`,
});
expectScanIndexCalledWith(`scb_2020`);
});

test("should append week format like yyyy/'W'W -> 2020/W46", async () => {
const findings = [];

const getFindings = async () => findings;
const getFindings = buildGetFindings(findings);

await handle({
getFindings,
Expand All @@ -202,19 +165,41 @@ test("should append week format like yyyy/'W'W -> 2020/W46", async () => {
client: elasticClient,
});

expect(elasticClient.index).toBeCalledTimes(1);
expect(elasticClient.index).toBeCalledWith({
body: {
"@timestamp": testDate,
id: "09988cdf-1fc7-4f85-95ee-1b1d65dbc7cc",
5285 labels: {
company: "iteratec",
},
name: "demo-scan",
parameters: ["-Pn", "localhost"],
scan_type: "Nmap",
type: "scan",
expectScanIndexCalledWith(`scb_2020/W46`);
});

test("should handle elasticsearch v8 bulk response shape", async () => {
const findings = findingsWithOpenPort;
const getFindings = buildGetFindings(findings);
const v8BulkResponse = { errors: true, items: [] };

const v8Client = {
indices: {
create: jest.fn(),
},
index: `scb_2020/W46`,
});
index: jest.fn(),
bulk: jest.fn(() => v8BulkResponse),
};

const consoleErrorSpy = jest
.spyOn(console, "error")
.mockImplementation(() => {});
const consoleLogSpy = jest.spyOn(console, "log").mockImplementation(() => {});

try {
await handle({
getFindings,
scan,
now: testDate,
tenant: "default",
appendNamespace: true,
client: v8Client,
});
expect(v8Client.bulk).toHaveBeenCalledTimes(1);
expect(consoleErrorSpy).toHaveBeenCalledWith("Bulk Request had errors:");
expect(consoleLogSpy).toHaveBeenCalledWith(v8BulkResponse);
} finally {
consoleErrorSpy.mockRestore();
consoleLogSpy.mockRestore();
}
});
Loading
0