diff --git a/CHANGELOG.md b/CHANGELOG.md index 6b81e457..4f7fc3ba 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,12 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## [0.8.3](https://github.com/traceloop/openllmetry-js/compare/v0.8.2...v0.8.3) (2024-05-16) + +### Bug Fixes + +- **sdk:** api for manual logging of LLM calls ([#264](https://github.com/traceloop/openllmetry-js/issues/264)) ([500097c](https://github.com/traceloop/openllmetry-js/commit/500097cb3534ccafef9c50d93d94f6606cd75016)) + ## [0.8.2](https://github.com/traceloop/openllmetry-js/compare/v0.8.1...v0.8.2) (2024-05-07) ### Bug Fixes diff --git a/lerna.json b/lerna.json index d14330be..664bb0fa 100644 --- a/lerna.json +++ b/lerna.json @@ -1,6 +1,6 @@ { "$schema": "node_modules/lerna/schemas/lerna-schema.json", - "version": "0.8.2", + "version": "0.8.3", "packages": ["packages/*"], "useNx": true } diff --git a/package-lock.json b/package-lock.json index d26103b0..480f5440 100644 --- a/package-lock.json +++ b/package-lock.json @@ -23454,7 +23454,7 @@ }, "packages/instrumentation-pinecone": { "name": "@traceloop/instrumentation-pinecone", - "version": "0.8.0", + "version": "0.8.3", "license": "Apache-2.0", "dependencies": { "@opentelemetry/core": "^1.22.0", @@ -24139,7 +24139,7 @@ }, "packages/traceloop-sdk": { "name": "@traceloop/node-server-sdk", - "version": "0.8.2", + "version": "0.8.3", "license": "Apache-2.0", "dependencies": { "@opentelemetry/exporter-trace-otlp-proto": "^0.49.1", @@ -24154,7 +24154,7 @@ "@traceloop/instrumentation-langchain": "^0.8.0", "@traceloop/instrumentation-llamaindex": "^0.8.0", "@traceloop/instrumentation-openai": "^0.8.2", - "@traceloop/instrumentation-pinecone": "^0.8.0", + "@traceloop/instrumentation-pinecone": "^0.8.3", "@traceloop/instrumentation-vertexai": "^0.8.0", "@types/nunjucks": "^3.2.5", "cross-fetch": "^4.0.0", diff --git a/packages/instrumentation-pinecone/CHANGELOG.md b/packages/instrumentation-pinecone/CHANGELOG.md index f8317437..3f7c51f4 100644 --- a/packages/instrumentation-pinecone/CHANGELOG.md +++ b/packages/instrumentation-pinecone/CHANGELOG.md @@ -3,6 +3,10 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## [0.8.3](https://github.com/traceloop/openllmetry-js/compare/v0.8.2...v0.8.3) (2024-05-16) + +**Note:** Version bump only for package @traceloop/instrumentation-pinecone + # [0.8.0](https://github.com/traceloop/openllmetry-js/compare/v0.7.0...v0.8.0) (2024-04-29) **Note:** Version bump only for package @traceloop/instrumentation-pinecone diff --git a/packages/instrumentation-pinecone/package.json b/packages/instrumentation-pinecone/package.json index 797fe31e..eaac41e4 100644 --- a/packages/instrumentation-pinecone/package.json +++ b/packages/instrumentation-pinecone/package.json @@ -1,6 +1,6 @@ { "name": "@traceloop/instrumentation-pinecone", - "version": "0.8.0", + "version": "0.8.3", "description": "OpenTelemetry instrumentation for pinecone vector DB", "main": "dist/index.js", "module": "dist/index.mjs", diff --git a/packages/instrumentation-pinecone/recordings/Test-Pinecone-instrumentation_3324481504/should-set-attributes-in-span-for-DB-deletes_1246828897/recording.har b/packages/instrumentation-pinecone/recordings/Test-Pinecone-instrumentation_3324481504/should-set-attributes-in-span-for-DB-deletes_1246828897/recording.har new file mode 100644 index 00000000..371d72a5 --- /dev/null +++ b/packages/instrumentation-pinecone/recordings/Test-Pinecone-instrumentation_3324481504/should-set-attributes-in-span-for-DB-deletes_1246828897/recording.har @@ -0,0 +1,480 @@ +{ + "log": { + "_recordingName": "Test Pinecone instrumentation/should set attributes in span for DB deletes", + "creator": { + "comment": "persister:fs", + "name": "Polly.JS", + "version": "6.0.6" + }, + "entries": [ + { + "_id": "973ce6e359a848bafd751dfc6a1cf4eb", + "_order": 0, + "cache": {}, + "request": { + "bodySize": 0, + "cookies": [], + "headers": [ + { + "name": "user-agent", + "value": "@pinecone-database/pinecone v2.0.0; lang=typescript; node v18.20.2" + } + ], + "headersSize": 175, + "httpVersion": "HTTP/1.1", + "method": "GET", + "queryString": [], + "url": "https://api.pinecone.io/indexes" + }, + "response": { + "bodySize": 476, + "content": { + "mimeType": "application/json", + "size": 476, + "text": "{\"indexes\":[{\"name\":\"pincone-instrumentation-test\",\"metric\":\"cosine\",\"dimension\":8,\"status\":{\"ready\":true,\"state\":\"Ready\"},\"host\":\"pincone-instrumentation-test-i5qrol6.svc.aped-4627-b74a.pinecone.io\",\"spec\":{\"serverless\":{\"region\":\"us-east-1\",\"cloud\":\"aws\"}}},{\"name\":\"quickstart\",\"metric\":\"euclidean\",\"dimension\":8,\"status\":{\"ready\":true,\"state\":\"Ready\"},\"host\":\"quickstart-i5qrol6.svc.aped-4627-b74a.pinecone.io\",\"spec\":{\"serverless\":{\"region\":\"us-east-1\",\"cloud\":\"aws\"}}}]}" + }, + "cookies": [], + "headers": [ + { + "name": "access-control-allow-origin", + "value": "*" + }, + { + "name": "access-control-expose-headers", + "value": "*" + }, + { + "name": "alt-svc", + "value": "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000" + }, + { + "name": "content-length", + "value": "476" + }, + { + "name": "content-type", + "value": "application/json" + }, + { + "name": "date", + "value": "Wed, 08 May 2024 18:38:24 GMT" + }, + { + "name": "server", + "value": "Google Frontend" + }, + { + "name": "vary", + "value": "origin,access-control-request-method,access-control-request-headers" + }, + { + "name": "via", + "value": "1.1 google" + }, + { + "name": "x-cloud-trace-context", + "value": "907fdaf1963b2a32321e2b51aa6fc055" + }, + { + "name": "x-pinecone-api-version", + "value": "2024-04" + } + ], + "headersSize": 422, + "httpVersion": "HTTP/1.1", + "redirectURL": "", + "status": 200, + "statusText": "OK" + }, + "startedDateTime": "2024-05-08T18:38:24.219Z", + "time": 391, + "timings": { + "blocked": -1, + "connect": -1, + "dns": -1, + "receive": 0, + "send": 0, + "ssl": -1, + "wait": 391 + } + }, + { + "_id": "6ad32f1905ae86de05946ba5b4176041", + "_order": 0, + "cache": {}, + "request": { + "bodySize": 0, + "cookies": [], + "headers": [ + { + "name": "user-agent", + "value": "@pinecone-database/pinecone v2.0.0; lang=typescript; node v18.20.2" + } + ], + "headersSize": 204, + "httpVersion": "HTTP/1.1", + "method": "GET", + "queryString": [], + "url": "https://api.pinecone.io/indexes/pincone-instrumentation-test" + }, + "response": { + "bodySize": 247, + "content": { + "mimeType": "application/json", + "size": 247, + "text": "{\"name\":\"pincone-instrumentation-test\",\"metric\":\"cosine\",\"dimension\":8,\"status\":{\"ready\":true,\"state\":\"Ready\"},\"host\":\"pincone-instrumentation-test-i5qrol6.svc.aped-4627-b74a.pinecone.io\",\"spec\":{\"serverless\":{\"region\":\"us-east-1\",\"cloud\":\"aws\"}}}" + }, + "cookies": [], + "headers": [ + { + "name": "access-control-allow-origin", + "value": "*" + }, + { + "name": "access-control-expose-headers", + "value": "*" + }, + { + "name": "alt-svc", + "value": "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000" + }, + { + "name": "content-length", + "value": "247" + }, + { + "name": "content-type", + "value": "application/json" + }, + { + "name": "date", + "value": "Wed, 08 May 2024 18:38:25 GMT" + }, + { + "name": "server", + "value": "Google Frontend" + }, + { + "name": "vary", + "value": "origin,access-control-request-method,access-control-request-headers" + }, + { + "name": "via", + "value": "1.1 google" + }, + { + "name": "x-cloud-trace-context", + "value": "9281d35e86341d72547116ef4158a56c" + }, + { + "name": "x-pinecone-api-version", + "value": "2024-04" + } + ], + "headersSize": 422, + "httpVersion": "HTTP/1.1", + "redirectURL": "", + "status": 200, + "statusText": "OK" + }, + "startedDateTime": "2024-05-08T18:38:24.613Z", + "time": 419, + "timings": { + "blocked": -1, + "connect": -1, + "dns": -1, + "receive": 0, + "send": 0, + "ssl": -1, + "wait": 419 + } + }, + { + "_id": "94781606ce6147e3c0a20c719f2b6f79", + "_order": 0, + "cache": {}, + "request": { + "bodySize": 34, + "cookies": [], + "headers": [ + { + "name": "content-type", + "value": "application/json" + }, + { + "name": "user-agent", + "value": "@pinecone-database/pinecone v2.0.0; lang=typescript; node v18.20.2" + } + ], + "headersSize": 267, + "httpVersion": "HTTP/1.1", + "method": "POST", + "postData": { + "mimeType": "application/json", + "params": [], + "text": "{\"ids\":[\"vec1\"],\"namespace\":\"ns1\"}" + }, + "queryString": [], + "url": "https://pincone-instrumentation-test-i5qrol6.svc.aped-4627-b74a.pinecone.io/vectors/delete" + }, + "response": { + "bodySize": 2, + "content": { + "mimeType": "application/json", + "size": 2, + "text": "{}" + }, + "cookies": [], + "headers": [ + { + "name": "connection", + "value": "keep-alive" + }, + { + "name": "content-length", + "value": "2" + }, + { + "name": "content-type", + "value": "application/json" + }, + { + "name": "date", + "value": "Wed, 08 May 2024 18:38:25 GMT" + }, + { + "name": "grpc-status", + "value": "0" + }, + { + "name": "server", + "value": "envoy" + }, + { + "name": "x-envoy-upstream-service-time", + "value": "91" + }, + { + "name": "x-pinecone-request-id", + "value": "6920948734063169772" + }, + { + "name": "x-pinecone-request-latency-ms", + "value": "90" + }, + { + "name": "x-pinecone-request-lsn", + "value": "2" + } + ], + "headersSize": 286, + "httpVersion": "HTTP/1.1", + "redirectURL": "", + "status": 200, + "statusText": "OK" + }, + "startedDateTime": "2024-05-08T18:38:25.282Z", + "time": 196, + "timings": { + "blocked": -1, + "connect": -1, + "dns": -1, + "receive": 0, + "send": 0, + "ssl": -1, + "wait": 196 + } + }, + { + "_id": "11ac4731b0451b51cf20206ccefcd831", + "_order": 0, + "cache": {}, + "request": { + "bodySize": 41, + "cookies": [], + "headers": [ + { + "name": "content-type", + "value": "application/json" + }, + { + "name": "user-agent", + "value": "@pinecone-database/pinecone v2.0.0; lang=typescript; node v18.20.2" + } + ], + "headersSize": 267, + "httpVersion": "HTTP/1.1", + "method": "POST", + "postData": { + "mimeType": "application/json", + "params": [], + "text": "{\"ids\":[\"vec2\",\"vec3\"],\"namespace\":\"ns1\"}" + }, + "queryString": [], + "url": "https://pincone-instrumentation-test-i5qrol6.svc.aped-4627-b74a.pinecone.io/vectors/delete" + }, + "response": { + "bodySize": 2, + "content": { + "mimeType": "application/json", + "size": 2, + "text": "{}" + }, + "cookies": [], + "headers": [ + { + "name": "connection", + "value": "keep-alive" + }, + { + "name": "content-length", + "value": "2" + }, + { + "name": "content-type", + "value": "application/json" + }, + { + "name": "date", + "value": "Wed, 08 May 2024 18:38:25 GMT" + }, + { + "name": "grpc-status", + "value": "0" + }, + { + "name": "server", + "value": "envoy" + }, + { + "name": "x-envoy-upstream-service-time", + "value": "56" + }, + { + "name": "x-pinecone-request-id", + "value": "5665856712736780298" + }, + { + "name": "x-pinecone-request-latency-ms", + "value": "56" + }, + { + "name": "x-pinecone-request-lsn", + "value": "3" + } + ], + "headersSize": 286, + "httpVersion": "HTTP/1.1", + "redirectURL": "", + "status": 200, + "statusText": "OK" + }, + "startedDateTime": "2024-05-08T18:38:25.697Z", + "time": 160, + "timings": { + "blocked": -1, + "connect": -1, + "dns": -1, + "receive": 0, + "send": 0, + "ssl": -1, + "wait": 160 + } + }, + { + "_id": "2b0270661d35668ac5d6bb44538c6071", + "_order": 0, + "cache": {}, + "request": { + "bodySize": 36, + "cookies": [], + "headers": [ + { + "name": "content-type", + "value": "application/json" + }, + { + "name": "user-agent", + "value": "@pinecone-database/pinecone v2.0.0; lang=typescript; node v18.20.2" + } + ], + "headersSize": 267, + "httpVersion": "HTTP/1.1", + "method": "POST", + "postData": { + "mimeType": "application/json", + "params": [], + "text": "{\"deleteAll\":true,\"namespace\":\"ns1\"}" + }, + "queryString": [], + "url": "https://pincone-instrumentation-test-i5qrol6.svc.aped-4627-b74a.pinecone.io/vectors/delete" + }, + "response": { + "bodySize": 2, + "content": { + "mimeType": "application/json", + "size": 2, + "text": "{}" + }, + "cookies": [], + "headers": [ + { + "name": "connection", + "value": "keep-alive" + }, + { + "name": "content-length", + "value": "2" + }, + { + "name": "content-type", + "value": "application/json" + }, + { + "name": "date", + "value": "Wed, 08 May 2024 18:38:26 GMT" + }, + { + "name": "grpc-status", + "value": "0" + }, + { + "name": "server", + "value": "envoy" + }, + { + "name": "x-envoy-upstream-service-time", + "value": "13" + }, + { + "name": "x-pinecone-request-id", + "value": "1106291243399043652" + }, + { + "name": "x-pinecone-request-latency-ms", + "value": "13" + } + ], + "headersSize": 259, + "httpVersion": "HTTP/1.1", + "redirectURL": "", + "status": 200, + "statusText": "OK" + }, + "startedDateTime": "2024-05-08T18:38:26.055Z", + "time": 117, + "timings": { + "blocked": -1, + "connect": -1, + "dns": -1, + "receive": 0, + "send": 0, + "ssl": -1, + "wait": 117 + } + } + ], + "pages": [], + "version": "1.2" + } +} diff --git a/packages/instrumentation-pinecone/recordings/Test-Pinecone-instrumentation_3324481504/should-set-attributes-in-span-for-DB-query_413889577/recording.har b/packages/instrumentation-pinecone/recordings/Test-Pinecone-instrumentation_3324481504/should-set-attributes-in-span-for-DB-query_413889577/recording.har index e8dff995..8ced1adc 100644 --- a/packages/instrumentation-pinecone/recordings/Test-Pinecone-instrumentation_3324481504/should-set-attributes-in-span-for-DB-query_413889577/recording.har +++ b/packages/instrumentation-pinecone/recordings/Test-Pinecone-instrumentation_3324481504/should-set-attributes-in-span-for-DB-query_413889577/recording.har @@ -8,49 +8,48 @@ }, "entries": [ { - "_id": "f66a36afd832b48fbd5bc0354289943f", + "_id": "973ce6e359a848bafd751dfc6a1cf4eb", "_order": 0, "cache": {}, "request": { - "bodySize": 36, + "bodySize": 0, "cookies": [], "headers": [ - { - "name": "content-type", - "value": "application/json" - }, { "name": "user-agent", - "value": "@pinecone-database/pinecone v2.0.0; lang=typescript; node v18.17.1" + "value": "@pinecone-database/pinecone v2.0.0; lang=typescript; node v18.20.2" } ], - "headersSize": 244, + "headersSize": 175, "httpVersion": "HTTP/1.1", - "method": "POST", - "postData": { - "mimeType": "application/json", - "params": [], - "text": "{\"deleteAll\":true,\"namespace\":\"ns1\"}" - }, + "method": "GET", "queryString": [], - "url": "https://tests-wujirot.svc.apw5-4e34-81fa.pinecone.io/vectors/delete" + "url": "https://api.pinecone.io/indexes" }, "response": { - "bodySize": 2, + "bodySize": 476, "content": { "mimeType": "application/json", - "size": 2, - "text": "{}" + "size": 476, + "text": "{\"indexes\":[{\"name\":\"pincone-instrumentation-test\",\"metric\":\"cosine\",\"dimension\":8,\"status\":{\"ready\":true,\"state\":\"Ready\"},\"host\":\"pincone-instrumentation-test-i5qrol6.svc.aped-4627-b74a.pinecone.io\",\"spec\":{\"serverless\":{\"region\":\"us-east-1\",\"cloud\":\"aws\"}}},{\"name\":\"quickstart\",\"metric\":\"euclidean\",\"dimension\":8,\"status\":{\"ready\":true,\"state\":\"Ready\"},\"host\":\"quickstart-i5qrol6.svc.aped-4627-b74a.pinecone.io\",\"spec\":{\"serverless\":{\"region\":\"us-east-1\",\"cloud\":\"aws\"}}}]}" }, "cookies": [], "headers": [ { - "name": "connection", - "value": "keep-alive" + "name": "access-control-allow-origin", + "value": "*" + }, + { + "name": "access-control-expose-headers", + "value": "*" + }, + { + "name": "alt-svc", + "value": "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000" }, { "name": "content-length", - "value": "2" + "value": "476" }, { "name": "content-type", @@ -58,33 +57,37 @@ }, { "name": "date", - "value": "Mon, 12 Feb 2024 22:04:51 GMT" + "value": "Wed, 08 May 2024 18:38:22 GMT" }, { - "name": "grpc-status", - "value": "0" + "name": "server", + "value": "Google Frontend" }, { - "name": "server", - "value": "envoy" + "name": "vary", + "value": "origin,access-control-request-method,access-control-request-headers" }, { - "name": "x-envoy-upstream-service-time", - "value": "51" + "name": "via", + "value": "1.1 google" }, { - "name": "x-pinecone-request-latency-ms", - "value": "51" + "name": "x-cloud-trace-context", + "value": "9a0c0b26470aa2ff33f16005469a8af2" + }, + { + "name": "x-pinecone-api-version", + "value": "2024-04" } ], - "headersSize": 215, + "headersSize": 422, "httpVersion": "HTTP/1.1", "redirectURL": "", "status": 200, "statusText": "OK" }, - "startedDateTime": "2024-02-12T22:04:51.474Z", - "time": 386, + "startedDateTime": "2024-05-08T18:38:21.941Z", + "time": 520, "timings": { "blocked": -1, "connect": -1, @@ -92,53 +95,52 @@ "receive": 0, "send": 0, "ssl": -1, - "wait": 386 + "wait": 520 } }, { - "_id": "06c4c271f12bc4fc8462a05bfee87380", + "_id": "6ad32f1905ae86de05946ba5b4176041", "_order": 0, "cache": {}, "request": { - "bodySize": 287, + "bodySize": 0, "cookies": [], "headers": [ - { - "name": "content-type", - "value": "application/json" - }, { "name": "user-agent", - "value": "@pinecone-database/pinecone v2.0.0; lang=typescript; node v18.17.1" + "value": "@pinecone-database/pinecone v2.0.0; lang=typescript; node v18.20.2" } ], - "headersSize": 244, + "headersSize": 204, "httpVersion": "HTTP/1.1", - "method": "POST", - "postData": { - "mimeType": "application/json", - "params": [], - "text": "{\"vectors\":[{\"id\":\"vec1\",\"values\":[0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1]},{\"id\":\"vec2\",\"values\":[0.2,0.2,0.2,0.2,0.2,0.2,0.2,0.2]},{\"id\":\"vec3\",\"values\":[0.3,0.3,0.3,0.3,0.3,0.3,0.3,0.3],\"metadata\":{\"test_meta\":42}},{\"id\":\"vec4\",\"values\":[0.4,0.4,0.4,0.4,0.4,0.4,0.4,0.4]}],\"namespace\":\"ns1\"}" - }, + "method": "GET", "queryString": [], - "url": "https://tests-wujirot.svc.apw5-4e34-81fa.pinecone.io/vectors/upsert" + "url": "https://api.pinecone.io/indexes/pincone-instrumentation-test" }, "response": { - "bodySize": 19, + "bodySize": 247, "content": { "mimeType": "application/json", - "size": 19, - "text": "{\"upsertedCount\":4}" + "size": 247, + "text": "{\"name\":\"pincone-instrumentation-test\",\"metric\":\"cosine\",\"dimension\":8,\"status\":{\"ready\":true,\"state\":\"Ready\"},\"host\":\"pincone-instrumentation-test-i5qrol6.svc.aped-4627-b74a.pinecone.io\",\"spec\":{\"serverless\":{\"region\":\"us-east-1\",\"cloud\":\"aws\"}}}" }, "cookies": [], "headers": [ { - "name": "connection", - "value": "keep-alive" + "name": "access-control-allow-origin", + "value": "*" + }, + { + "name": "access-control-expose-headers", + "value": "*" + }, + { + "name": "alt-svc", + "value": "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000" }, { "name": "content-length", - "value": "19" + "value": "247" }, { "name": "content-type", @@ -146,37 +148,37 @@ }, { "name": "date", - "value": "Mon, 12 Feb 2024 22:04:52 GMT" + "value": "Wed, 08 May 2024 18:38:22 GMT" }, { - "name": "grpc-status", - "value": "0" + "name": "server", + "value": "Google Frontend" }, { - "name": "server", - "value": "envoy" + "name": "vary", + "value": "origin,access-control-request-method,access-control-request-headers" }, { - "name": "x-envoy-upstream-service-time", - "value": "86" + "name": "via", + "value": "1.1 google" }, { - "name": "x-pinecone-request-latency-ms", - "value": "86" + "name": "x-cloud-trace-context", + "value": "ff6b3b019ee69aba595b7376367d3b8f" }, { - "name": "x-pinecone-request-lsn", - "value": "1" + "name": "x-pinecone-api-version", + "value": "2024-04" } ], - "headersSize": 243, + "headersSize": 422, "httpVersion": "HTTP/1.1", "redirectURL": "", "status": 200, "statusText": "OK" }, - "startedDateTime": "2024-02-12T22:04:51.907Z", - "time": 421, + "startedDateTime": "2024-05-08T18:38:22.471Z", + "time": 401, "timings": { "blocked": -1, "connect": -1, @@ -184,11 +186,11 @@ "receive": 0, "send": 0, "ssl": -1, - "wait": 421 + "wait": 401 } }, { - "_id": "c652df799e470321f2aeae2f6fd92b3a", + "_id": "16a3f8ced8ec392d85d1f7f6121ac5ea", "_order": 0, "cache": {}, "request": { @@ -201,10 +203,10 @@ }, { "name": "user-agent", - "value": "@pinecone-database/pinecone v2.0.0; lang=typescript; node v18.17.1" + "value": "@pinecone-database/pinecone v2.0.0; lang=typescript; node v18.20.2" } ], - "headersSize": 235, + "headersSize": 258, "httpVersion": "HTTP/1.1", "method": "POST", "postData": { @@ -213,14 +215,14 @@ "text": "{\"namespace\":\"ns1\",\"topK\":3,\"includeValues\":true,\"includeMetadata\":true,\"vector\":[0.3,0.3,0.3,0.3,0.3,0.3,0.3,0.3]}" }, "queryString": [], - "url": "https://tests-wujirot.svc.apw5-4e34-81fa.pinecone.io/query" + "url": "https://pincone-instrumentation-test-i5qrol6.svc.aped-4627-b74a.pinecone.io/query" }, "response": { - "bodySize": 319, + "bodySize": 297, "content": { "mimeType": "application/json", - "size": 319, - "text": "{\"results\":[],\"matches\":[{\"id\":\"vec3\",\"score\":0,\"values\":[0.3,0.3,0.3,0.3,0.3,0.3,0.3,0.3],\"metadata\":{\"test_meta\":42}},{\"id\":\"vec4\",\"score\":0.0799999237,\"values\":[0.4,0.4,0.4,0.4,0.4,0.4,0.4,0.4]},{\"id\":\"vec2\",\"score\":0.0800000429,\"values\":[0.2,0.2,0.2,0.2,0.2,0.2,0.2,0.2]}],\"namespace\":\"ns1\",\"usage\":{\"readUnits\":6}}" + "size": 297, + "text": "{\"results\":[],\"matches\":[{\"id\":\"vec2\",\"score\":1,\"values\":[0.2,0.2,0.2,0.2,0.2,0.2,0.2,0.2]},{\"id\":\"vec3\",\"score\":1,\"values\":[0.3,0.3,0.3,0.3,0.3,0.3,0.3,0.3],\"metadata\":{\"test_meta\":42}},{\"id\":\"vec1\",\"score\":1,\"values\":[0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1]}],\"namespace\":\"ns1\",\"usage\":{\"readUnits\":6}}" }, "cookies": [], "headers": [ @@ -230,7 +232,7 @@ }, { "name": "content-length", - "value": "319" + "value": "297" }, { "name": "content-type", @@ -238,7 +240,7 @@ }, { "name": "date", - "value": "Mon, 12 Feb 2024 22:05:24 GMT" + "value": "Wed, 08 May 2024 18:38:23 GMT" }, { "name": "grpc-status", @@ -250,25 +252,29 @@ }, { "name": "x-envoy-upstream-service-time", - "value": "4" + "value": "82" }, { "name": "x-pinecone-max-indexed-lsn", "value": "1" }, + { + "name": "x-pinecone-request-id", + "value": "2671526720955016309" + }, { "name": "x-pinecone-request-latency-ms", - "value": "3" + "value": "81" } ], - "headersSize": 246, + "headersSize": 292, "httpVersion": "HTTP/1.1", "redirectURL": "", "status": 200, "statusText": "OK" }, - "startedDateTime": "2024-02-12T22:05:22.392Z", - "time": 1550, + "startedDateTime": "2024-05-08T18:38:23.098Z", + "time": 224, "timings": { "blocked": -1, "connect": -1, @@ -276,7 +282,7 @@ "receive": 0, "send": 0, "ssl": -1, - "wait": 1550 + "wait": 224 } } ], diff --git a/packages/instrumentation-pinecone/recordings/Test-Pinecone-instrumentation_3324481504/should-set-attributes-in-span-for-DB-upsert_1383174072/recording.har b/packages/instrumentation-pinecone/recordings/Test-Pinecone-instrumentation_3324481504/should-set-attributes-in-span-for-DB-upsert_1383174072/recording.har index 9dd743e2..3fad4860 100644 --- a/packages/instrumentation-pinecone/recordings/Test-Pinecone-instrumentation_3324481504/should-set-attributes-in-span-for-DB-upsert_1383174072/recording.har +++ b/packages/instrumentation-pinecone/recordings/Test-Pinecone-instrumentation_3324481504/should-set-attributes-in-span-for-DB-upsert_1383174072/recording.har @@ -8,7 +8,7 @@ }, "entries": [ { - "_id": "211ce6a3821dc82cf567743073ec50a6", + "_id": "973ce6e359a848bafd751dfc6a1cf4eb", "_order": 0, "cache": {}, "request": { @@ -17,21 +17,21 @@ "headers": [ { "name": "user-agent", - "value": "@pinecone-database/pinecone v2.0.0; lang=typescript; node v18.17.1" + "value": "@pinecone-database/pinecone v2.0.0; lang=typescript; node v18.20.2" } ], - "headersSize": 181, + "headersSize": 175, "httpVersion": "HTTP/1.1", "method": "GET", "queryString": [], - "url": "https://api.pinecone.io/indexes/tests" + "url": "https://api.pinecone.io/indexes" }, "response": { - "bodySize": 204, + "bodySize": 476, "content": { "mimeType": "application/json", - "size": 204, - "text": "{\"name\":\"tests\",\"metric\":\"euclidean\",\"dimension\":8,\"status\":{\"ready\":true,\"state\":\"Ready\"},\"host\":\"tests-wujirot.svc.apw5-4e34-81fa.pinecone.io\",\"spec\":{\"serverless\":{\"region\":\"us-west-2\",\"cloud\":\"aws\"}}}" + "size": 476, + "text": "{\"indexes\":[{\"name\":\"pincone-instrumentation-test\",\"metric\":\"cosine\",\"dimension\":8,\"status\":{\"ready\":true,\"state\":\"Ready\"},\"host\":\"pincone-instrumentation-test-i5qrol6.svc.aped-4627-b74a.pinecone.io\",\"spec\":{\"serverless\":{\"region\":\"us-east-1\",\"cloud\":\"aws\"}}},{\"name\":\"quickstart\",\"metric\":\"euclidean\",\"dimension\":8,\"status\":{\"ready\":true,\"state\":\"Ready\"},\"host\":\"quickstart-i5qrol6.svc.aped-4627-b74a.pinecone.io\",\"spec\":{\"serverless\":{\"region\":\"us-east-1\",\"cloud\":\"aws\"}}}]}" }, "cookies": [], "headers": [ @@ -49,7 +49,7 @@ }, { "name": "content-length", - "value": "204" + "value": "476" }, { "name": "content-type", @@ -57,7 +57,7 @@ }, { "name": "date", - "value": "Mon, 12 Feb 2024 22:04:41 GMT" + "value": "Wed, 08 May 2024 18:38:20 GMT" }, { "name": "server", @@ -73,17 +73,21 @@ }, { "name": "x-cloud-trace-context", - "value": "a5c4636b1a9d2634df43ef54b77624d3" + "value": "cb35cf514a258d1e5fc49ccce135b75c" + }, + { + "name": "x-pinecone-api-version", + "value": "2024-04" } ], - "headersSize": 389, + "headersSize": 422, "httpVersion": "HTTP/1.1", "redirectURL": "", "status": 200, "statusText": "OK" }, - "startedDateTime": "2024-02-12T22:04:40.492Z", - "time": 4952, + "startedDateTime": "2024-05-08T18:38:19.860Z", + "time": 451, "timings": { "blocked": -1, "connect": -1, @@ -91,141 +95,52 @@ "receive": 0, "send": 0, "ssl": -1, - "wait": 4952 + "wait": 451 } }, { - "_id": "f66a36afd832b48fbd5bc0354289943f", + "_id": "6ad32f1905ae86de05946ba5b4176041", "_order": 0, "cache": {}, "request": { - "bodySize": 36, + "bodySize": 0, "cookies": [], "headers": [ - { - "name": "content-type", - "value": "application/json" - }, { "name": "user-agent", - "value": "@pinecone-database/pinecone v2.0.0; lang=typescript; node v18.17.1" + "value": "@pinecone-database/pinecone v2.0.0; lang=typescript; node v18.20.2" } ], - "headersSize": 244, + "headersSize": 204, "httpVersion": "HTTP/1.1", - "method": "POST", - "postData": { - "mimeType": "application/json", - "params": [], - "text": "{\"deleteAll\":true,\"namespace\":\"ns1\"}" - }, + "method": "GET", "queryString": [], - "url": "https://tests-wujirot.svc.apw5-4e34-81fa.pinecone.io/vectors/delete" + "url": "https://api.pinecone.io/indexes/pincone-instrumentation-test" }, "response": { - "bodySize": 2, + "bodySize": 247, "content": { "mimeType": "application/json", - "size": 2, - "text": "{}" + "size": 247, + "text": "{\"name\":\"pincone-instrumentation-test\",\"metric\":\"cosine\",\"dimension\":8,\"status\":{\"ready\":true,\"state\":\"Ready\"},\"host\":\"pincone-instrumentation-test-i5qrol6.svc.aped-4627-b74a.pinecone.io\",\"spec\":{\"serverless\":{\"region\":\"us-east-1\",\"cloud\":\"aws\"}}}" }, "cookies": [], "headers": [ { - "name": "connection", - "value": "keep-alive" - }, - { - "name": "content-length", - "value": "2" - }, - { - "name": "content-type", - "value": "application/json" - }, - { - "name": "date", - "value": "Mon, 12 Feb 2024 22:04:49 GMT" - }, - { - "name": "grpc-status", - "value": "0" - }, - { - "name": "server", - "value": "envoy" - }, - { - "name": "x-envoy-upstream-service-time", - "value": "22" + "name": "access-control-allow-origin", + "value": "*" }, { - "name": "x-pinecone-request-latency-ms", - "value": "22" - } - ], - "headersSize": 215, - "httpVersion": "HTTP/1.1", - "redirectURL": "", - "status": 200, - "statusText": "OK" - }, - "startedDateTime": "2024-02-12T22:04:45.450Z", - "time": 3574, - "timings": { - "blocked": -1, - "connect": -1, - "dns": -1, - "receive": 0, - "send": 0, - "ssl": -1, - "wait": 3574 - } - }, - { - "_id": "06c4c271f12bc4fc8462a05bfee87380", - "_order": 0, - "cache": {}, - "request": { - "bodySize": 287, - "cookies": [], - "headers": [ - { - "name": "content-type", - "value": "application/json" + "name": "access-control-expose-headers", + "value": "*" }, { - "name": "user-agent", - "value": "@pinecone-database/pinecone v2.0.0; lang=typescript; node v18.17.1" - } - ], - "headersSize": 244, - "httpVersion": "HTTP/1.1", - "method": "POST", - "postData": { - "mimeType": "application/json", - "params": [], - "text": "{\"vectors\":[{\"id\":\"vec1\",\"values\":[0.1,0.1,0.1,0.1,0.1,0.1,0.1,0.1]},{\"id\":\"vec2\",\"values\":[0.2,0.2,0.2,0.2,0.2,0.2,0.2,0.2]},{\"id\":\"vec3\",\"values\":[0.3,0.3,0.3,0.3,0.3,0.3,0.3,0.3],\"metadata\":{\"test_meta\":42}},{\"id\":\"vec4\",\"values\":[0.4,0.4,0.4,0.4,0.4,0.4,0.4,0.4]}],\"namespace\":\"ns1\"}" - }, - "queryString": [], - "url": "https://tests-wujirot.svc.apw5-4e34-81fa.pinecone.io/vectors/upsert" - }, - "response": { - "bodySize": 19, - "content": { - "mimeType": "application/json", - "size": 19, - "text": "{\"upsertedCount\":4}" - }, - "cookies": [], - "headers": [ - { - "name": "connection", - "value": "keep-alive" + "name": "alt-svc", + "value": "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000" }, { "name": "content-length", - "value": "19" + "value": "247" }, { "name": "content-type", @@ -233,37 +148,37 @@ }, { "name": "date", - "value": "Mon, 12 Feb 2024 22:04:49 GMT" + "value": "Wed, 08 May 2024 18:38:20 GMT" }, { - "name": "grpc-status", - "value": "0" + "name": "server", + "value": "Google Frontend" }, { - "name": "server", - "value": "envoy" + "name": "vary", + "value": "origin,access-control-request-method,access-control-request-headers" }, { - "name": "x-envoy-upstream-service-time", - "value": "57" + "name": "via", + "value": "1.1 google" }, { - "name": "x-pinecone-request-latency-ms", - "value": "57" + "name": "x-cloud-trace-context", + "value": "18fa12473e62442e301501290e94c3fd" }, { - "name": "x-pinecone-request-lsn", - "value": "1" + "name": "x-pinecone-api-version", + "value": "2024-04" } ], - "headersSize": 243, + "headersSize": 422, "httpVersion": "HTTP/1.1", "redirectURL": "", "status": 200, "statusText": "OK" }, - "startedDateTime": "2024-02-12T22:04:49.084Z", - "time": 1220, + "startedDateTime": "2024-05-08T18:38:20.321Z", + "time": 402, "timings": { "blocked": -1, "connect": -1, @@ -271,11 +186,11 @@ "receive": 0, "send": 0, "ssl": -1, - "wait": 1220 + "wait": 402 } }, { - "_id": "4d7aeae345bd41d4f7312bcd35569193", + "_id": "b23331d14736d2d22b1a85f656157656", "_order": 0, "cache": {}, "request": { @@ -288,10 +203,10 @@ }, { "name": "user-agent", - "value": "@pinecone-database/pinecone v2.0.0; lang=typescript; node v18.17.1" + "value": "@pinecone-database/pinecone v2.0.0; lang=typescript; node v18.20.2" } ], - "headersSize": 244, + "headersSize": 267, "httpVersion": "HTTP/1.1", "method": "POST", "postData": { @@ -300,7 +215,7 @@ "text": "{\"vectors\":[{\"id\":\"vec5\",\"values\":[0.5,0.5,0.5,0.5,0.5,0.5,0.5,0.5]}],\"namespace\":\"\"}" }, "queryString": [], - "url": "https://tests-wujirot.svc.apw5-4e34-81fa.pinecone.io/vectors/upsert" + "url": "https://pincone-instrumentation-test-i5qrol6.svc.aped-4627-b74a.pinecone.io/vectors/upsert" }, "response": { "bodySize": 19, @@ -325,7 +240,7 @@ }, { "name": "date", - "value": "Mon, 12 Feb 2024 22:04:50 GMT" + "value": "Wed, 08 May 2024 18:38:21 GMT" }, { "name": "grpc-status", @@ -337,25 +252,29 @@ }, { "name": "x-envoy-upstream-service-time", - "value": "42" + "value": "25" + }, + { + "name": "x-pinecone-request-id", + "value": "2666709944482362316" }, { "name": "x-pinecone-request-latency-ms", - "value": "41" + "value": "25" }, { "name": "x-pinecone-request-lsn", "value": "1" } ], - "headersSize": 243, + "headersSize": 287, "httpVersion": "HTTP/1.1", "redirectURL": "", "status": 200, "statusText": "OK" }, - "startedDateTime": "2024-02-12T22:04:50.312Z", - "time": 1100, + "startedDateTime": "2024-05-08T18:38:20.736Z", + "time": 421, "timings": { "blocked": -1, "connect": -1, @@ -363,7 +282,7 @@ "receive": 0, "send": 0, "ssl": -1, - "wait": 1100 + "wait": 421 } } ], diff --git a/packages/instrumentation-pinecone/test/instrumentation.test.ts b/packages/instrumentation-pinecone/test/instrumentation.test.ts index db898ebf..e89357e6 100644 --- a/packages/instrumentation-pinecone/test/instrumentation.test.ts +++ b/packages/instrumentation-pinecone/test/instrumentation.test.ts @@ -14,6 +14,7 @@ * limitations under the License. */ +import type * as pineconeModuleType from "@pinecone-database/pinecone"; import { context } from "@opentelemetry/api"; import { AsyncHooksContextManager } from "@opentelemetry/context-async-hooks"; import { PineconeInstrumentation } from "../src/instrumentation"; @@ -23,97 +24,121 @@ import { InMemorySpanExporter, SimpleSpanProcessor, } from "@opentelemetry/sdk-trace-base"; -import { Pinecone, Index } from "@pinecone-database/pinecone"; -import * as pc_module from "@pinecone-database/pinecone"; - import { Polly, setupMocha as setupPolly } from "@pollyjs/core"; import FetchAdapter from "@pollyjs/adapter-fetch"; import FSPersister from "@pollyjs/persister-fs"; +import { SpanAttributes } from "@traceloop/ai-semantic-conventions"; const memoryExporter = new InMemorySpanExporter(); +const PINECONE_TEST_INDEX = "pincone-instrumentation-test"; + Polly.register(FetchAdapter); Polly.register(FSPersister); -describe.skip("Test Pinecone instrumentation", function () { +describe("Test Pinecone instrumentation", function () { const provider = new BasicTracerProvider(); + let pineconeModule: typeof pineconeModuleType; let instrumentation: PineconeInstrumentation; let contextManager: AsyncHooksContextManager; - let pc_index: Index; + let pc: pineconeModuleType.Pinecone; + let pc_index: pineconeModuleType.Index; setupPolly({ adapters: ["fetch"], persister: "fs", recordIfMissing: process.env.RECORD_MODE === "NEW", + matchRequestsBy: { + headers: false, + }, }); before(async () => { if (process.env.RECORD_MODE !== "NEW") { process.env.PINECONE_API_KEY = "test"; } - provider.addSpanProcessor(new SimpleSpanProcessor(memoryExporter)); instrumentation = new PineconeInstrumentation(); instrumentation.setTracerProvider(provider); - instrumentation.manuallyInstrument(pc_module); - // TODO: create index manually in your Pinecone Instance - // await pc.createIndex({ - // name: "tests", - // dimension: 8, - // metric: "euclidean", - // spec: { - // serverless: { - // cloud: "aws", - // region: "us-west-2", - // }, - // }, - // }); - const pc = new Pinecone(); - pc_index = pc.index("tests"); + + pineconeModule = await import("@pinecone-database/pinecone"); + + pc = new pineconeModule.Pinecone({ + apiKey: process.env.PINECONE_API_KEY || "", + }); + + pc_index = pc.index(PINECONE_TEST_INDEX); + if (process.env.RECORD_MODE == "NEW") { + await pc.createIndex({ + name: PINECONE_TEST_INDEX, + dimension: 8, + metric: "cosine", + spec: { + serverless: { + cloud: "aws", + region: "us-east-1", + }, + }, + }); + + const pc_index = pc.index(PINECONE_TEST_INDEX); + + await pc_index.namespace("ns1").upsert([ + { + id: "vec1", + values: [0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1], + }, + { + id: "vec2", + values: [0.2, 0.2, 0.2, 0.2, 0.2, 0.2, 0.2, 0.2], + }, + { + id: "vec3", + values: [0.3, 0.3, 0.3, 0.3, 0.3, 0.3, 0.3, 0.3], + metadata: { + test_meta: 42, + }, + }, + { + id: "vec4", + values: [0.4, 0.4, 0.4, 0.4, 0.4, 0.4, 0.4, 0.4], + }, + ]); + + // delay before your upserted vectors are available to query ,delete + await new Promise((resolve) => setTimeout(resolve, 5000)); + } }); beforeEach(async function () { + contextManager = new AsyncHooksContextManager().enable(); + context.setGlobalContextManager(contextManager); + const { server } = this.polly as Polly; + server.any().on("beforePersist", (_req, recording) => { recording.request.headers = recording.request.headers.filter( ({ name }: { name: string }) => name !== "api-key", ); }); - contextManager = new AsyncHooksContextManager().enable(); - context.setGlobalContextManager(contextManager); - // await pc_index.namespace("ns1").deleteAll(); - // await pc_index.namespace("ns1").upsert([ - // { - // id: "vec1", - // values: [0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1, 0.1], - // }, - // { - // id: "vec2", - // values: [0.2, 0.2, 0.2, 0.2, 0.2, 0.2, 0.2, 0.2], - // }, - // { - // id: "vec3", - // values: [0.3, 0.3, 0.3, 0.3, 0.3, 0.3, 0.3, 0.3], - // metadata: { - // test_meta: 42, - // }, - // }, - // { - // id: "vec4", - // values: [0.4, 0.4, 0.4, 0.4, 0.4, 0.4, 0.4, 0.4], - // }, - // ]); + pc = new pineconeModule.Pinecone({ + apiKey: process.env.PINECONE_API_KEY || "", + }); + pc_index = pc.index(PINECONE_TEST_INDEX); + await pc.listIndexes(); + await pc.describeIndex(PINECONE_TEST_INDEX); + memoryExporter.reset(); }); - afterEach(() => { + afterEach(async () => { memoryExporter.reset(); context.disable(); }); - after(() => { - //pc.deleteIndex("tests"); + after(async () => { + if (process.env.RECORD_MODE == "NEW") pc.deleteIndex(PINECONE_TEST_INDEX); }); it("should set attributes in span for DB upsert", async () => { @@ -123,6 +148,7 @@ describe.skip("Test Pinecone instrumentation", function () { values: [0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5], }, ]; + await pc_index.upsert(input); const spans = memoryExporter.getFinishedSpans(); @@ -131,14 +157,10 @@ describe.skip("Test Pinecone instrumentation", function () { assert.strictEqual(spans[0].name, "pinecone.upsert"); const attributes = spans[0].attributes; - assert.strictEqual(attributes["vector_db.vendor"], "Pinecone"); - }).timeout(60000); + assert.strictEqual(attributes[SpanAttributes.VECTOR_DB_VENDOR], "Pinecone"); + }); it("should set attributes in span for DB query", async () => { - // wait 30 seconds for pinecone to update to go through otherwise result can have 0 values. - // await new Promise((resolve) => setTimeout(resolve, 30000)); - - // now query await pc_index.namespace("ns1").query({ topK: 3, vector: [0.3, 0.3, 0.3, 0.3, 0.3, 0.3, 0.3, 0.3], @@ -150,9 +172,10 @@ describe.skip("Test Pinecone instrumentation", function () { assert.strictEqual(spans.length, 1); const attributes = spans[0].attributes; - assert.strictEqual(attributes["vector_db.vendor"], "Pinecone"); + assert.strictEqual(attributes[SpanAttributes.VECTOR_DB_VENDOR], "Pinecone"); const span = spans[0]; + assert.strictEqual(span.events.length, 8); assert.strictEqual(span.events[0].name, "pinecone.query.request"); assert.strictEqual(span.events[1].name, "pinecone.query.result"); @@ -162,12 +185,12 @@ describe.skip("Test Pinecone instrumentation", function () { assert.strictEqual(span.events[5].name, "pinecone.query.result.1.metadata"); assert.strictEqual(span.events[6].name, "pinecone.query.result.2"); assert.strictEqual(span.events[7].name, "pinecone.query.result.2.metadata"); - }).timeout(60000); + }); - it.skip("should set attributes in span for DB deletes", async () => { - await pc_index.deleteOne("vec1"); - await pc_index.deleteMany(["vec2", "vec3"]); - await pc_index.deleteAll(); + it("should set attributes in span for DB deletes", async () => { + await pc_index.namespace("ns1").deleteOne("vec1"); + await pc_index.namespace("ns1").deleteMany(["vec2", "vec3"]); + await pc_index.namespace("ns1").deleteAll(); const spans = memoryExporter.getFinishedSpans(); @@ -177,7 +200,10 @@ describe.skip("Test Pinecone instrumentation", function () { const span = spans[index]; assert.strictEqual(span.name, "pinecone.delete"); const attributes = span.attributes; - assert.strictEqual(attributes["vector_db.vendor"], "Pinecone"); + assert.strictEqual( + attributes[SpanAttributes.VECTOR_DB_VENDOR], + "Pinecone", + ); } }); }); diff --git a/packages/traceloop-sdk/CHANGELOG.md b/packages/traceloop-sdk/CHANGELOG.md index 41b9405e..4bd902af 100644 --- a/packages/traceloop-sdk/CHANGELOG.md +++ b/packages/traceloop-sdk/CHANGELOG.md @@ -3,6 +3,12 @@ All notable changes to this project will be documented in this file. See [Conventional Commits](https://conventionalcommits.org) for commit guidelines. +## [0.8.3](https://github.com/traceloop/openllmetry-js/compare/v0.8.2...v0.8.3) (2024-05-16) + +### Bug Fixes + +- **sdk:** api for manual logging of LLM calls ([#264](https://github.com/traceloop/openllmetry-js/issues/264)) ([500097c](https://github.com/traceloop/openllmetry-js/commit/500097cb3534ccafef9c50d93d94f6606cd75016)) + ## [0.8.2](https://github.com/traceloop/openllmetry-js/compare/v0.8.1...v0.8.2) (2024-05-07) **Note:** Version bump only for package @traceloop/node-server-sdk diff --git a/packages/traceloop-sdk/package.json b/packages/traceloop-sdk/package.json index 073a65b6..65c506e8 100644 --- a/packages/traceloop-sdk/package.json +++ b/packages/traceloop-sdk/package.json @@ -1,6 +1,6 @@ { "name": "@traceloop/node-server-sdk", - "version": "0.8.2", + "version": "0.8.3", "description": "Traceloop Software Development Kit (SDK) for Node.js", "main": "dist/index.js", "module": "dist/index.mjs", @@ -47,7 +47,7 @@ "@traceloop/instrumentation-langchain": "^0.8.0", "@traceloop/instrumentation-llamaindex": "^0.8.0", "@traceloop/instrumentation-openai": "^0.8.2", - "@traceloop/instrumentation-pinecone": "^0.8.0", + "@traceloop/instrumentation-pinecone": "^0.8.3", "@traceloop/instrumentation-vertexai": "^0.8.0", "@types/nunjucks": "^3.2.5", "cross-fetch": "^4.0.0", diff --git a/packages/traceloop-sdk/recordings/Test-SDK-Decorators_847855269/should-create-spans-for-manual-LLM-instrumentation_981493419/recording.har b/packages/traceloop-sdk/recordings/Test-SDK-Decorators_847855269/should-create-spans-for-manual-LLM-instrumentation_981493419/recording.har new file mode 100644 index 00000000..14cabe1c --- /dev/null +++ b/packages/traceloop-sdk/recordings/Test-SDK-Decorators_847855269/should-create-spans-for-manual-LLM-instrumentation_981493419/recording.har @@ -0,0 +1,232 @@ +{ + "log": { + "_recordingName": "Test SDK Decorators/should create spans for manual LLM instrumentation", + "creator": { + "comment": "persister:fs", + "name": "Polly.JS", + "version": "6.0.6" + }, + "entries": [ + { + "_id": "80b48c70cae76a3c38b9af59d5273e33", + "_order": 0, + "cache": {}, + "request": { + "bodySize": 139, + "cookies": [], + "headers": [ + { + "_fromType": "array", + "name": "content-length", + "value": "139" + }, + { + "_fromType": "array", + "name": "accept", + "value": "application/json" + }, + { + "_fromType": "array", + "name": "content-type", + "value": "application/json" + }, + { + "_fromType": "array", + "name": "user-agent", + "value": "OpenAI/JS 4.38.3" + }, + { + "_fromType": "array", + "name": "x-stainless-lang", + "value": "js" + }, + { + "_fromType": "array", + "name": "x-stainless-package-version", + "value": "4.38.3" + }, + { + "_fromType": "array", + "name": "x-stainless-os", + "value": "MacOS" + }, + { + "_fromType": "array", + "name": "x-stainless-arch", + "value": "arm64" + }, + { + "_fromType": "array", + "name": "x-stainless-runtime", + "value": "node" + }, + { + "_fromType": "array", + "name": "x-stainless-runtime-version", + "value": "v18.17.1" + }, + { + "_fromType": "array", + "name": "accept-encoding", + "value": "gzip,deflate" + }, + { + "name": "host", + "value": "api.openai.com" + } + ], + "headersSize": 470, + "httpVersion": "HTTP/1.1", + "method": "POST", + "postData": { + "mimeType": "application/json", + "params": [], + "text": "{\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"Tell me a joke about OpenTelemetry\"\n }\n ],\n \"model\": \"gpt-3.5-turbo\"\n}" + }, + "queryString": [], + "url": "https://api.openai.com/v1/chat/completions" + }, + "response": { + "bodySize": 491, + "content": { + "encoding": "base64", + "mimeType": "application/json", + "size": 491, + "text": "[\"H4sIAAAAAAAAA1SRT2vDMAzF7/kUms/taFpC017GxmAMBh3bSgtjFCdRE6+OZWxlbSj97sPpv+3iw3t68k/SPgIQqhBTEHklOa+t7k9el1t6vt+q5GmSLtP54/tuZtPSvmwW8zfRCwnKvjHnc+o2p9pqZEXmaOcOJWPoGo/jJB0n4+GwM2oqUIdYabk/uk363LiM+oN4mJySFakcvZjCZwQAsO/ewGgK3IkpDHpnpUbvZYlieikCEI50UIT0XnmWhkXvauZkGE2HvahaKFQBXCHMLJoP1FgjuxYK/EFNFh2UBJmjDd7BA+ay8RiqW9iiQ5B6K1sPXg==\",\"1lYrU8KaHLCjJtN4I04/Hi6omkrrKAtjmUbri75WRvlq5VB6MgHLM9lj/BABfHUraf5NKayj2vKKaYMmNIyTYztxPcIfMz2ZTCz1VR+NohOf8K1nrFdrZUp01qluP4EyOkS/AAAA//8DAMTz7T0eAgAA\"]" + }, + "cookies": [ + { + "domain": ".api.openai.com", + "expires": "2024-05-16T16:38:43.000Z", + "httpOnly": true, + "name": "__cf_bm", + "path": "/", + "sameSite": "None", + "secure": true, + "value": "eAINKaQ0O3tdSzB.KOxjAHZ7AdV9qQHAOaBv2.3yn1k-1715875723-1.0.1.1-lJcLz_EuaF8Y01g9hzqDWo0bzCPfONiL0mogjqJhNdyyROSixq4FgyI2Cf2bakaQxuzIumKzXIS6oNw2p_AfHw" + }, + { + "domain": ".api.openai.com", + "httpOnly": true, + "name": "_cfuvid", + "path": "/", + "sameSite": "None", + "secure": true, + "value": "mnsAAZGFXU4qk2hW2DwDVg71nGu_MASZVL2AJqgEbv4-1715875723340-0.0.1.1-604800000" + } + ], + "headers": [ + { + "name": "date", + "value": "Thu, 16 May 2024 16:08:43 GMT" + }, + { + "name": "content-type", + "value": "application/json" + }, + { + "name": "transfer-encoding", + "value": "chunked" + }, + { + "name": "connection", + "value": "keep-alive" + }, + { + "name": "openai-organization", + "value": "traceloop" + }, + { + "name": "openai-processing-ms", + "value": "804" + }, + { + "name": "openai-version", + "value": "2020-10-01" + }, + { + "name": "strict-transport-security", + "value": "max-age=15724800; includeSubDomains" + }, + { + "name": "x-ratelimit-limit-requests", + "value": "5000" + }, + { + "name": "x-ratelimit-limit-tokens", + "value": "160000" + }, + { + "name": "x-ratelimit-remaining-requests", + "value": "4999" + }, + { + "name": "x-ratelimit-remaining-tokens", + "value": "159974" + }, + { + "name": "x-ratelimit-reset-requests", + "value": "12ms" + }, + { + "name": "x-ratelimit-reset-tokens", + "value": "9ms" + }, + { + "name": "x-request-id", + "value": "req_9d2a45908700e557bd9eaf0294dd79cc" + }, + { + "name": "cf-cache-status", + "value": "DYNAMIC" + }, + { + "_fromType": "array", + "name": "set-cookie", + "value": "__cf_bm=eAINKaQ0O3tdSzB.KOxjAHZ7AdV9qQHAOaBv2.3yn1k-1715875723-1.0.1.1-lJcLz_EuaF8Y01g9hzqDWo0bzCPfONiL0mogjqJhNdyyROSixq4FgyI2Cf2bakaQxuzIumKzXIS6oNw2p_AfHw; path=/; expires=Thu, 16-May-24 16:38:43 GMT; domain=.api.openai.com; HttpOnly; Secure; SameSite=None" + }, + { + "_fromType": "array", + "name": "set-cookie", + "value": "_cfuvid=mnsAAZGFXU4qk2hW2DwDVg71nGu_MASZVL2AJqgEbv4-1715875723340-0.0.1.1-604800000; path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None" + }, + { + "name": "server", + "value": "cloudflare" + }, + { + "name": "cf-ray", + "value": "884ca0c0989159ef-MXP" + }, + { + "name": "content-encoding", + "value": "gzip" + }, + { + "name": "alt-svc", + "value": "h3=\":443\"; ma=86400" + } + ], + "headersSize": 1100, + "httpVersion": "HTTP/1.1", + "redirectURL": "", + "status": 200, + "statusText": "OK" + }, + "startedDateTime": "2024-05-16T16:08:42.253Z", + "time": 1072, + "timings": { + "blocked": -1, + "connect": -1, + "dns": -1, + "receive": 0, + "send": 0, + "ssl": -1, + "wait": 1072 + } + } + ], + "pages": [], + "version": "1.2" + } +} diff --git a/packages/traceloop-sdk/recordings/Test-SDK-Manual-Instrumentations_53777088/should-create-spans-for-manual-LLM-instrumentation_981493419/recording.har b/packages/traceloop-sdk/recordings/Test-SDK-Manual-Instrumentations_53777088/should-create-spans-for-manual-LLM-instrumentation_981493419/recording.har new file mode 100644 index 00000000..48b161a4 --- /dev/null +++ b/packages/traceloop-sdk/recordings/Test-SDK-Manual-Instrumentations_53777088/should-create-spans-for-manual-LLM-instrumentation_981493419/recording.har @@ -0,0 +1,232 @@ +{ + "log": { + "_recordingName": "Test SDK Manual Instrumentations/should create spans for manual LLM instrumentation", + "creator": { + "comment": "persister:fs", + "name": "Polly.JS", + "version": "6.0.6" + }, + "entries": [ + { + "_id": "80b48c70cae76a3c38b9af59d5273e33", + "_order": 0, + "cache": {}, + "request": { + "bodySize": 139, + "cookies": [], + "headers": [ + { + "_fromType": "array", + "name": "content-length", + "value": "139" + }, + { + "_fromType": "array", + "name": "accept", + "value": "application/json" + }, + { + "_fromType": "array", + "name": "content-type", + "value": "application/json" + }, + { + "_fromType": "array", + "name": "user-agent", + "value": "OpenAI/JS 4.38.3" + }, + { + "_fromType": "array", + "name": "x-stainless-lang", + "value": "js" + }, + { + "_fromType": "array", + "name": "x-stainless-package-version", + "value": "4.38.3" + }, + { + "_fromType": "array", + "name": "x-stainless-os", + "value": "MacOS" + }, + { + "_fromType": "array", + "name": "x-stainless-arch", + "value": "arm64" + }, + { + "_fromType": "array", + "name": "x-stainless-runtime", + "value": "node" + }, + { + "_fromType": "array", + "name": "x-stainless-runtime-version", + "value": "v18.17.1" + }, + { + "_fromType": "array", + "name": "accept-encoding", + "value": "gzip,deflate" + }, + { + "name": "host", + "value": "api.openai.com" + } + ], + "headersSize": 470, + "httpVersion": "HTTP/1.1", + "method": "POST", + "postData": { + "mimeType": "application/json", + "params": [], + "text": "{\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"Tell me a joke about OpenTelemetry\"\n }\n ],\n \"model\": \"gpt-3.5-turbo\"\n}" + }, + "queryString": [], + "url": "https://api.openai.com/v1/chat/completions" + }, + "response": { + "bodySize": 491, + "content": { + "encoding": "base64", + "mimeType": "application/json", + "size": 491, + "text": "[\"H4sIAAAAAAAAA1SRT0sDMRDF7/spxpzb0j+stb2IRRFEVKygIFLS7HQ3ms3EZNZ2Lf3uku22xUsO7+VNfvOyTQCEzsQUhCokq9KZ7uTpbT2+8qyHD/P72/xuc71aFTff89nz8JdFJyZo+YmKD6meotIZZE12byuPkjFOHYwH6cU4PR9PGqOkDE2M5Y67o17a5covqdsfDNM2WZBWGMQU3hMAgG1zRkab4UZMod85KCWGIHMU0+MlAOHJREXIEHRgafe8ranIMtoG+7WoIdMZcIHw6NC+oMES2degKEOw+IMecgImcNKzxnAJM1SyCgiaYS0DSLOWdQ==\",\"ACaCZRVqYC+VtvmZaB/cHUkN5c7TMm5lK2OO+kpbHYqFRxnIRqrA5PbxXQLw0TRS/VtSOE+l4wXTF9o4cJDux4nTH5zMYduWYGJpTvooTVo+EerAWC5W2ubonddNPZEy2SV/AAAA//8DAPdMHFkdAgAA\"]" + }, + "cookies": [ + { + "domain": ".api.openai.com", + "expires": "2024-05-16T16:37:59.000Z", + "httpOnly": true, + "name": "__cf_bm", + "path": "/", + "sameSite": "None", + "secure": true, + "value": "WpY7cnKn0lqnIt0ZLuDnRp4Prh.lZx6Du5Ab5.qd_Y0-1715875679-1.0.1.1-W7aQFN2Eyb1TwWHjPUUGCSiOe9rHqJrCbOGp_kgTq2vh8Lkh8FI7FkNV4CVlCKxlybyZOEC7GO9bENHFe2kMtA" + }, + { + "domain": ".api.openai.com", + "httpOnly": true, + "name": "_cfuvid", + "path": "/", + "sameSite": "None", + "secure": true, + "value": "zxoiDMvDYa9s7AlXYCyUCv2ggnW0Xyi.ePJqG5DfkUw-1715875679571-0.0.1.1-604800000" + } + ], + "headers": [ + { + "name": "date", + "value": "Thu, 16 May 2024 16:07:59 GMT" + }, + { + "name": "content-type", + "value": "application/json" + }, + { + "name": "transfer-encoding", + "value": "chunked" + }, + { + "name": "connection", + "value": "keep-alive" + }, + { + "name": "openai-organization", + "value": "traceloop" + }, + { + "name": "openai-processing-ms", + "value": "524" + }, + { + "name": "openai-version", + "value": "2020-10-01" + }, + { + "name": "strict-transport-security", + "value": "max-age=15724800; includeSubDomains" + }, + { + "name": "x-ratelimit-limit-requests", + "value": "5000" + }, + { + "name": "x-ratelimit-limit-tokens", + "value": "160000" + }, + { + "name": "x-ratelimit-remaining-requests", + "value": "4999" + }, + { + "name": "x-ratelimit-remaining-tokens", + "value": "159974" + }, + { + "name": "x-ratelimit-reset-requests", + "value": "12ms" + }, + { + "name": "x-ratelimit-reset-tokens", + "value": "9ms" + }, + { + "name": "x-request-id", + "value": "req_91066547d03e725758d4ee4db0740453" + }, + { + "name": "cf-cache-status", + "value": "DYNAMIC" + }, + { + "_fromType": "array", + "name": "set-cookie", + "value": "__cf_bm=WpY7cnKn0lqnIt0ZLuDnRp4Prh.lZx6Du5Ab5.qd_Y0-1715875679-1.0.1.1-W7aQFN2Eyb1TwWHjPUUGCSiOe9rHqJrCbOGp_kgTq2vh8Lkh8FI7FkNV4CVlCKxlybyZOEC7GO9bENHFe2kMtA; path=/; expires=Thu, 16-May-24 16:37:59 GMT; domain=.api.openai.com; HttpOnly; Secure; SameSite=None" + }, + { + "_fromType": "array", + "name": "set-cookie", + "value": "_cfuvid=zxoiDMvDYa9s7AlXYCyUCv2ggnW0Xyi.ePJqG5DfkUw-1715875679571-0.0.1.1-604800000; path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None" + }, + { + "name": "server", + "value": "cloudflare" + }, + { + "name": "cf-ray", + "value": "884c9fb0ccef83a6-MXP" + }, + { + "name": "content-encoding", + "value": "gzip" + }, + { + "name": "alt-svc", + "value": "h3=\":443\"; ma=86400" + } + ], + "headersSize": 1100, + "httpVersion": "HTTP/1.1", + "redirectURL": "", + "status": 200, + "statusText": "OK" + }, + "startedDateTime": "2024-05-16T16:07:58.763Z", + "time": 796, + "timings": { + "blocked": -1, + "connect": -1, + "dns": -1, + "receive": 0, + "send": 0, + "ssl": -1, + "wait": 796 + } + } + ], + "pages": [], + "version": "1.2" + } +} diff --git a/packages/traceloop-sdk/src/lib/tracing/manual.ts b/packages/traceloop-sdk/src/lib/tracing/manual.ts index 270ba782..7c123e75 100644 --- a/packages/traceloop-sdk/src/lib/tracing/manual.ts +++ b/packages/traceloop-sdk/src/lib/tracing/manual.ts @@ -1,6 +1,10 @@ -import { Span, context } from "@opentelemetry/api"; +import { Span, context, trace } from "@opentelemetry/api"; import { getTracer } from "./tracing"; -import { Events, EventAttributes } from "@traceloop/ai-semantic-conventions"; +import { + Events, + EventAttributes, + SpanAttributes, +} from "@traceloop/ai-semantic-conventions"; import { shouldSendTraces } from "."; type VectorDBCallConfig = { @@ -8,6 +12,11 @@ type VectorDBCallConfig = { type: "query" | "upsert" | "delete"; }; +type LLMCallConfig = { + vendor: string; + type: "chat" | "completion"; +}; + export class VectorSpan { private span: Span; @@ -52,24 +61,118 @@ export class VectorSpan { } } +export class LLMSpan { + private span: Span; + + constructor(span: Span) { + this.span = span; + } + + reportRequest({ + model, + messages, + }: { + model: string; + messages: { + role: string; + content?: string | unknown; + }[]; + }) { + this.span.setAttributes({ + [SpanAttributes.LLM_REQUEST_MODEL]: model, + }); + + messages.forEach((message, index) => { + this.span.setAttributes({ + [`${SpanAttributes.LLM_PROMPTS}.${index}.role`]: message.role, + [`${SpanAttributes.LLM_PROMPTS}.${index}.content`]: + typeof message.content === "string" + ? message.content + : JSON.stringify(message.content), + }); + }); + } + + reportResponse({ + model, + usage, + completions, + }: { + model: string; + usage?: { + prompt_tokens: number; + completion_tokens: number; + total_tokens: number; + }; + completions?: { + finish_reason: string; + message: { + role: "system" | "user" | "assistant"; + content: string | null; + }; + }[]; + }) { + this.span.setAttribute(SpanAttributes.LLM_RESPONSE_MODEL, model); + + if (usage) { + this.span.setAttributes({ + [SpanAttributes.LLM_USAGE_PROMPT_TOKENS]: usage.prompt_tokens, + [SpanAttributes.LLM_USAGE_COMPLETION_TOKENS]: usage.completion_tokens, + [SpanAttributes.LLM_USAGE_TOTAL_TOKENS]: usage.total_tokens, + }); + } + + completions?.forEach((completion, index) => { + this.span.setAttributes({ + [`${SpanAttributes.LLM_COMPLETIONS}.${index}.finish_reason`]: + completion.finish_reason, + [`${SpanAttributes.LLM_COMPLETIONS}.${index}.message.role`]: + completion.message.role, + [`${SpanAttributes.LLM_COMPLETIONS}.${index}.message.content`]: + completion.message.content || "", + }); + }); + } +} + export function withVectorDBCall< F extends ({ span }: { span: VectorSpan }) => ReturnType, >({ vendor, type }: VectorDBCallConfig, fn: F, thisArg?: ThisParameterType) { const entityContext = context.active(); - getTracer().startActiveSpan( + return getTracer().startActiveSpan( `${vendor}.${type}`, - {}, + { [SpanAttributes.LLM_REQUEST_TYPE]: type }, entityContext, - async (span: Span) => { + (span: Span) => { const res = fn.apply(thisArg, [{ span: new VectorSpan(span) }]); if (res instanceof Promise) { - return res.then(() => { + return res.then((resolvedRes) => { span.end(); + return resolvedRes; }); } span.end(); + return res; }, ); } + +export function withLLMCall< + F extends ({ span }: { span: LLMSpan }) => ReturnType, +>({ vendor, type }: LLMCallConfig, fn: F, thisArg?: ThisParameterType) { + const span = getTracer().startSpan(`${vendor}.${type}`, {}, context.active()); + trace.setSpan(context.active(), span); + + const res = fn.apply(thisArg, [{ span: new LLMSpan(span) }]); + if (res instanceof Promise) { + return res.then((resolvedRes) => { + span.end(); + return resolvedRes; + }); + } + + span.end(); + return res; +} diff --git a/packages/traceloop-sdk/test/decorators.test.ts b/packages/traceloop-sdk/test/decorators.test.ts index 8245c4a5..f7f85923 100644 --- a/packages/traceloop-sdk/test/decorators.test.ts +++ b/packages/traceloop-sdk/test/decorators.test.ts @@ -26,6 +26,7 @@ import { Polly, setupMocha as setupPolly } from "@pollyjs/core"; import NodeHttpAdapter from "@pollyjs/adapter-node-http"; import FSPersister from "@pollyjs/persister-fs"; import { SpanAttributes } from "@traceloop/ai-semantic-conventions"; +import { ChatCompletionMessageParam } from "openai/resources/index.mjs"; const memoryExporter = new InMemorySpanExporter(); @@ -369,4 +370,79 @@ describe("Test SDK Decorators", () => { undefined, ); }); + + it("should create spans for manual LLM instrumentation", async () => { + const result = await traceloop.withWorkflow( + { name: "joke_generator" }, + () => + traceloop.withLLMCall( + { vendor: "openai", type: "chat" }, + async ({ span }) => { + const messages: ChatCompletionMessageParam[] = [ + { role: "user", content: "Tell me a joke about OpenTelemetry" }, + ]; + const model = "gpt-3.5-turbo"; + + span.reportRequest({ model, messages }); + + const response = await openai.chat.completions.create({ + messages, + model, + }); + + span.reportResponse(response); + + return response; + }, + ), + ); + + const spans = memoryExporter.getFinishedSpans(); + const workflowSpan = spans.find( + (span) => span.name === "joke_generator.workflow", + ); + const completionSpan = spans.find((span) => span.name === "openai.chat"); + + assert.ok(result); + assert.ok(completionSpan); + assert.ok(workflowSpan); + assert.strictEqual( + workflowSpan.attributes[`${SpanAttributes.TRACELOOP_WORKFLOW_NAME}`], + "joke_generator", + ); + assert.strictEqual( + completionSpan.parentSpanId, + workflowSpan.spanContext().spanId, + ); + assert.ok(workflowSpan.startTime <= completionSpan.startTime); + assert.ok(workflowSpan.endTime >= completionSpan.endTime); + assert.strictEqual( + completionSpan.attributes[`${SpanAttributes.LLM_REQUEST_MODEL}`], + "gpt-3.5-turbo", + ); + assert.strictEqual( + completionSpan.attributes[`${SpanAttributes.LLM_RESPONSE_MODEL}`], + "gpt-3.5-turbo-0125", + ); + assert.strictEqual( + completionSpan.attributes[`${SpanAttributes.LLM_PROMPTS}.0.role`], + "user", + ); + assert.strictEqual( + completionSpan.attributes[`${SpanAttributes.LLM_PROMPTS}.0.content`], + "Tell me a joke about OpenTelemetry", + ); + assert.ok( + completionSpan.attributes[`${SpanAttributes.LLM_USAGE_TOTAL_TOKENS}`], + ); + assert.equal( + completionSpan.attributes[`${SpanAttributes.LLM_USAGE_PROMPT_TOKENS}`], + "15", + ); + assert.ok( + +completionSpan.attributes[ + `${SpanAttributes.LLM_USAGE_COMPLETION_TOKENS}` + ]! > 0, + ); + }); });