From 5d61583610012d4fdbb50238d708d7a9b901cfca Mon Sep 17 00:00:00 2001 From: Sebastian Quek Date: Mon, 13 Feb 2023 17:24:59 +0800 Subject: [PATCH 01/61] Remove engine argument --- src/serpapi.ts | 7 +------ src/types.ts | 6 ++++-- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/src/serpapi.ts b/src/serpapi.ts index 242019d..08c1786 100644 --- a/src/serpapi.ts +++ b/src/serpapi.ts @@ -74,7 +74,6 @@ export async function getJson< E extends EngineName = EngineName, P extends AllowArbitraryParams> = EngineParameters, >( - engine: E, parameters: P, callback?: (json: BaseResponse) => void, ) { @@ -84,7 +83,6 @@ export async function getJson< SEARCH_PATH, { ...parameters, - engine, api_key: key, output: "json", }, @@ -95,14 +93,13 @@ export async function getJson< if ( // https://github.com/serpapi/public-roadmap/issues/562 // https://github.com/serpapi/public-roadmap/issues/563 - engine !== "yahoo_shopping" && + parameters.engine !== "yahoo_shopping" && nextParametersFromResponse ) { const nextParameters = { ...parameters, ...nextParametersFromResponse }; if (haveParametersChanged(parameters, nextParameters)) { json.next = (innerCallback = callback) => getJson( - engine, nextParameters, innerCallback, ); @@ -131,7 +128,6 @@ export async function getHtml< E extends EngineName = EngineName, P extends AllowArbitraryParams> = EngineParameters, >( - engine: E, parameters: P, callback?: (html: string) => void, ) { @@ -141,7 +137,6 @@ export async function getHtml< SEARCH_PATH, { ...parameters, - engine, api_key: key, output: "html", }, diff --git a/src/types.ts b/src/types.ts index 0cf7749..186d92c 100644 --- a/src/types.ts +++ b/src/types.ts @@ -54,8 +54,10 @@ export type EngineName = (keyof EngineMap) | AnyEngineName; export type EngineParameters< E extends EngineName = EngineName, > = { - [K in E]: K extends keyof EngineMap ? EngineMap[K]["parameters"] - : BaseParameters & Record; + [K in E]: + & { engine: K } + & (K extends keyof EngineMap ? EngineMap[K]["parameters"] + : BaseParameters & Record); }[E]; export type BaseResponse = { From 28f03f17401ed07babe0098ee9e7849b661f6532 Mon Sep 17 00:00:00 2001 From: Sebastian Quek Date: Mon, 13 Feb 2023 16:46:53 +0800 Subject: [PATCH 02/61] Keep engine param in next parameters --- src/utils.ts | 1 - tests/utils_test.ts | 2 ++ 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/utils.ts b/src/utils.ts index 855125f..68d0653 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -53,7 +53,6 @@ export function extractNextParameters(json: { if (nextUrlString) { const nextUrl = new URL(nextUrlString); const nextParameters = Object.fromEntries(nextUrl.searchParams.entries()); - delete nextParameters["engine"]; return nextParameters as NextParameters; } } diff --git a/tests/utils_test.ts b/tests/utils_test.ts index 8e27b11..d2d8004 100644 --- a/tests/utils_test.ts +++ b/tests/utils_test.ts @@ -39,6 +39,7 @@ describe("extractNextParameters", () => { }, }), { + engine: "google", device: "desktop", gl: "us", google_domain: "google.com", @@ -61,6 +62,7 @@ describe("extractNextParameters", () => { }, ), { + engine: "google_scholar_profiles", after_author: "rZlDAYoq__8J", hl: "en", mauthors: "Mike", From fc4dd907ffba8ca88ff31a6cc32bf876be813bd9 Mon Sep 17 00:00:00 2001 From: Sebastian Quek Date: Mon, 13 Feb 2023 16:55:40 +0800 Subject: [PATCH 03/61] Update tests --- tests/engines/apple_reviews_test.ts | 20 ++-- tests/engines/baidu_test.ts | 26 ++--- tests/engines/duckduckgo_test.ts | 26 ++--- tests/engines/ebay_test.ts | 20 ++-- tests/engines/google_maps_test.ts | 20 ++-- tests/engines/google_scholar_profiles_test.ts | 20 ++-- tests/engines/google_test.ts | 96 +++++++------------ tests/engines/home_depot_test.ts | 38 ++++---- 8 files changed, 126 insertions(+), 140 deletions(-) diff --git a/tests/engines/apple_reviews_test.ts b/tests/engines/apple_reviews_test.ts index f1ee02a..0f8660e 100644 --- a/tests/engines/apple_reviews_test.ts +++ b/tests/engines/apple_reviews_test.ts @@ -60,7 +60,7 @@ describe("apple_reviews", { await t.step("async/await", async () => { const ids: number[] = []; let page; - page = await getJson(engine, { product_id: productId }); + page = await getJson({ engine, product_id: productId }); while (page) { ids.push(...page.reviews.map((r: any) => r.id)); if (ids.length >= 50) break; @@ -72,7 +72,7 @@ describe("apple_reviews", { await t.step("callback", async () => { const ids: number[] = []; await new Promise((done) => { - getJson(engine, { product_id: productId }, (page) => { + getJson({ engine, product_id: productId }, (page) => { ids.push(...page.reviews.map((r: any) => r.id)); if (ids.length < 50 && page.next) { page.next(); @@ -92,7 +92,8 @@ describe("apple_reviews", { config.api_key = null; await t.step("async/await", async () => { - const page1 = await getJson(engine, { + const page1 = await getJson({ + engine, api_key: SERPAPI_TEST_KEY, no_cache: false, product_id: productId, @@ -109,7 +110,8 @@ describe("apple_reviews", { await t.step("callback", async () => { const page1 = await new Promise>>( (res) => - getJson(engine, { + getJson({ + engine, api_key: SERPAPI_TEST_KEY, no_cache: false, product_id: productId, @@ -130,13 +132,13 @@ describe("apple_reviews", { it("getJson pagination with page", { ignore: !HAS_API_KEY, }, async (t) => { - const firstPage = await getJson(engine, { product_id: productId }); + const firstPage = await getJson({ engine, product_id: productId }); const idsOnFirstPage = firstPage.reviews.map((r: any) => r.id); await t.step("async/await", async () => { const ids: number[] = []; let page; - page = await getJson(engine, { product_id: productId, page: "2" }); + page = await getJson({ engine, product_id: productId, page: "2" }); while (page) { ids.push(...page.reviews.map((r: any) => r.id)); if (ids.length >= 50) break; @@ -153,7 +155,7 @@ describe("apple_reviews", { await t.step("callback", async () => { const ids: number[] = []; await new Promise((done) => { - getJson(engine, { product_id: productId, page: "2" }, (page) => { + getJson({ engine, product_id: productId, page: "2" }, (page) => { ids.push(...page.reviews.map((r: any) => r.id)); if (ids.length < 50 && page.next) { page.next(); @@ -176,7 +178,7 @@ describe("apple_reviews", { await t.step("async/await", async () => { let page; let pageNum = 0; - page = await getJson(engine, { product_id: productId, page: "99" }); + page = await getJson({ engine, product_id: productId, page: "99" }); while (page && pageNum < 5) { pageNum++; page = await page.next?.(); @@ -187,7 +189,7 @@ describe("apple_reviews", { await t.step("callback", async () => { let pageNum = 0; await new Promise((done) => { - getJson(engine, { product_id: productId, page: "99" }, (page) => { + getJson({ engine, product_id: productId, page: "99" }, (page) => { pageNum++; if (pageNum < 5 && page.next) { page.next(); diff --git a/tests/engines/baidu_test.ts b/tests/engines/baidu_test.ts index 8437815..408148a 100644 --- a/tests/engines/baidu_test.ts +++ b/tests/engines/baidu_test.ts @@ -60,7 +60,7 @@ describe("baidu", { await t.step("async/await", async () => { const links: string[] = []; let page; - page = await getJson(engine, { q }); + page = await getJson({ engine, q }); while (page) { links.push(...page.organic_results.map((r: any) => r.link)); if (links.length >= 20) break; @@ -72,7 +72,7 @@ describe("baidu", { await t.step("callback", async () => { const links: string[] = []; await new Promise((done) => { - getJson(engine, { q }, (page) => { + getJson({ engine, q }, (page) => { links.push(...page.organic_results.map((r: any) => r.link)); if (links.length < 20 && page.next) { page.next(); @@ -92,7 +92,8 @@ describe("baidu", { config.api_key = null; await t.step("async/await", async () => { - const page1 = await getJson(engine, { + const page1 = await getJson({ + engine, api_key: SERPAPI_TEST_KEY, no_cache: false, q, @@ -109,11 +110,10 @@ describe("baidu", { await t.step("callback", async () => { const page1 = await new Promise>>( (res) => - getJson(engine, { - api_key: SERPAPI_TEST_KEY, - no_cache: false, - q, - }, res), + getJson( + { engine, api_key: SERPAPI_TEST_KEY, no_cache: false, q }, + res, + ), ); assertMatch(executeSpy.calls[0].args[1].api_key as string, /[a-z0-9]+/); assertEquals(executeSpy.calls[0].args[1].no_cache, false); @@ -130,13 +130,13 @@ describe("baidu", { it("getJson pagination with offset + size", { ignore: !HAS_API_KEY, }, async (t) => { - const firstPage = await getJson(engine, { q }); + const firstPage = await getJson({ engine, q }); const linksOnFirstPage = firstPage.organic_results.map((r: any) => r.link); await t.step("async/await", async () => { const links: string[] = []; let page; - page = await getJson(engine, { q, pn: "30", rn: "30" }); + page = await getJson({ engine, q, pn: "30", rn: "30" }); while (page) { links.push(...page.organic_results.map((r: any) => r.link)); if (links.length >= 60) break; @@ -152,7 +152,7 @@ describe("baidu", { await t.step("callback", async () => { const links: string[] = []; await new Promise((done) => { - getJson(engine, { q, pn: "30", rn: "30" }, (page) => { + getJson({ engine, q, pn: "30", rn: "30" }, (page) => { links.push(...page.organic_results.map((r: any) => r.link)); if (links.length < 60 && page.next) { page.next(); @@ -175,7 +175,7 @@ describe("baidu", { await t.step("async/await", async () => { let page; let pageNum = 0; - page = await getJson(engine, { q, pn: "750" }); + page = await getJson({ engine, q, pn: "750" }); while (page && pageNum < 5) { pageNum++; page = await page.next?.(); @@ -186,7 +186,7 @@ describe("baidu", { await t.step("callback", async () => { let pageNum = 0; await new Promise((done) => { - getJson(engine, { q, pn: "750" }, (page) => { + getJson({ engine, q, pn: "750" }, (page) => { pageNum++; if (pageNum < 5 && page.next) { page.next(); diff --git a/tests/engines/duckduckgo_test.ts b/tests/engines/duckduckgo_test.ts index 0791b8f..792e5e5 100644 --- a/tests/engines/duckduckgo_test.ts +++ b/tests/engines/duckduckgo_test.ts @@ -60,7 +60,7 @@ describe("duckduckgo", { await t.step("async/await", async () => { const links: string[] = []; let page; - page = await getJson(engine, { q }); + page = await getJson({ engine, q }); while (page) { links.push(...page.organic_results.map((r: any) => r.link)); if (links.length >= 50) break; @@ -72,7 +72,7 @@ describe("duckduckgo", { await t.step("callback", async () => { const links: string[] = []; await new Promise((done) => { - getJson(engine, { q }, (page) => { + getJson({ engine, q }, (page) => { links.push(...page.organic_results.map((r: any) => r.link)); if (links.length < 50 && page.next) { page.next(); @@ -92,7 +92,8 @@ describe("duckduckgo", { config.api_key = null; await t.step("async/await", async () => { - const page1 = await getJson(engine, { + const page1 = await getJson({ + engine, api_key: SERPAPI_TEST_KEY, no_cache: false, q, @@ -109,11 +110,10 @@ describe("duckduckgo", { await t.step("callback", async () => { const page1 = await new Promise>>( (res) => - getJson(engine, { - api_key: SERPAPI_TEST_KEY, - no_cache: false, - q, - }, res), + getJson( + { engine, api_key: SERPAPI_TEST_KEY, no_cache: false, q }, + res, + ), ); assertMatch(executeSpy.calls[0].args[1].api_key as string, /[a-z0-9]+/); assertEquals(executeSpy.calls[0].args[1].no_cache, false); @@ -130,7 +130,7 @@ describe("duckduckgo", { it("getJson pagination with offset", { ignore: !HAS_API_KEY, }, async (t) => { - const firstPage = await getJson(engine, { q }); + const firstPage = await getJson({ engine, q }); const linksOnFirstPage: string[] = firstPage.organic_results.map(( r: any, ) => r.link); @@ -138,7 +138,7 @@ describe("duckduckgo", { await t.step("async/await", async () => { const links: string[] = []; let page; - page = await getJson(engine, { q, start: 10 }); + page = await getJson({ engine, q, start: 10 }); while (page) { links.push(...page.organic_results.map((r: any) => r.link)); if (links.length >= 50) break; @@ -155,7 +155,7 @@ describe("duckduckgo", { await t.step("callback", async () => { const links: string[] = []; await new Promise((done) => { - getJson(engine, { q, start: 10 }, (page) => { + getJson({ engine, q, start: 10 }, (page) => { links.push(...page.organic_results.map((r: any) => r.link)); if (links.length < 50 && page.next) { page.next(); @@ -178,7 +178,7 @@ describe("duckduckgo", { await t.step("async/await", async () => { let page; let pageNum = 0; - page = await getJson(engine, { q, kl: "uk-en", no_cache: true }); + page = await getJson({ engine, q, kl: "uk-en", no_cache: true }); while (page && pageNum < 8) { pageNum++; page = await page.next?.(); @@ -189,7 +189,7 @@ describe("duckduckgo", { await t.step("callback", async () => { let pageNum = 0; await new Promise((done) => { - getJson(engine, { q, kl: "uk-en" }, (page) => { + getJson({ engine, q, kl: "uk-en" }, (page) => { pageNum++; if (pageNum < 8 && page.next) { page.next(); diff --git a/tests/engines/ebay_test.ts b/tests/engines/ebay_test.ts index 5b3c41f..6ebe126 100644 --- a/tests/engines/ebay_test.ts +++ b/tests/engines/ebay_test.ts @@ -60,7 +60,7 @@ describe("ebay", { await t.step("async/await", async () => { const links: string[] = []; let page; - page = await getJson(engine, { _nkw: nkw }); + page = await getJson({ engine, _nkw: nkw }); while (page) { links.push(...page.organic_results.map((r: any) => r.link)); if (links.length >= 120) break; @@ -72,7 +72,7 @@ describe("ebay", { await t.step("callback", async () => { const links: string[] = []; await new Promise((done) => { - getJson(engine, { _nkw: nkw }, (page) => { + getJson({ engine, _nkw: nkw }, (page) => { links.push(...page.organic_results.map((r: any) => r.link)); if (links.length < 120 && page.next) { page.next(); @@ -92,7 +92,8 @@ describe("ebay", { config.api_key = null; await t.step("async/await", async () => { - const page1 = await getJson(engine, { + const page1 = await getJson({ + engine, api_key: SERPAPI_TEST_KEY, no_cache: false, _nkw: nkw, @@ -109,7 +110,8 @@ describe("ebay", { await t.step("callback", async () => { const page1 = await new Promise>>( (res) => - getJson(engine, { + getJson({ + engine, api_key: SERPAPI_TEST_KEY, no_cache: false, _nkw: nkw, @@ -130,13 +132,13 @@ describe("ebay", { it("getJson pagination with page + size", { ignore: !HAS_API_KEY, }, async (t) => { - const firstPage = await getJson(engine, { _nkw: nkw }); + const firstPage = await getJson({ engine, _nkw: nkw }); const linksOnFirstPage = firstPage.organic_results.map((r: any) => r.link); await t.step("async/await", async () => { const links: string[] = []; let page; - page = await getJson(engine, { _nkw: nkw, _pgn: "2", _ipg: "100" }); + page = await getJson({ engine, _nkw: nkw, _pgn: "2", _ipg: "100" }); while (page) { links.push(...page.organic_results.map((r: any) => r.link)); if (links.length >= 200) break; @@ -152,7 +154,7 @@ describe("ebay", { await t.step("callback", async () => { const links: string[] = []; await new Promise((done) => { - getJson(engine, { _nkw: nkw, _pgn: "2", _ipg: "100" }, (page) => { + getJson({ engine, _nkw: nkw, _pgn: "2", _ipg: "100" }, (page) => { links.push(...page.organic_results.map((r: any) => r.link)); if (links.length < 200 && page.next) { page.next(); @@ -175,7 +177,7 @@ describe("ebay", { await t.step("async/await", async () => { let page; let pageNum = 0; - page = await getJson(engine, { _nkw: nkw, _pgn: "7", _ipg: "200" }); + page = await getJson({ engine, _nkw: nkw, _pgn: "7", _ipg: "200" }); while (page && pageNum < 5) { pageNum++; page = await page.next?.(); @@ -186,7 +188,7 @@ describe("ebay", { await t.step("callback", async () => { let pageNum = 0; await new Promise((done) => { - getJson(engine, { _nkw: nkw, _pgn: "7", _ipg: "200" }, (page) => { + getJson({ engine, _nkw: nkw, _pgn: "7", _ipg: "200" }, (page) => { pageNum++; if (pageNum < 5 && page.next) { page.next(); diff --git a/tests/engines/google_maps_test.ts b/tests/engines/google_maps_test.ts index f1d4c52..ad3f6a1 100644 --- a/tests/engines/google_maps_test.ts +++ b/tests/engines/google_maps_test.ts @@ -62,7 +62,7 @@ describe("google_maps", { await t.step("async/await", async () => { const placeIds: string[] = []; let page; - page = await getJson(engine, { q, type, ll }); + page = await getJson({ engine, q, type, ll }); while (page) { placeIds.push(...page.local_results.map((r: any) => r.place_id)); if (placeIds.length >= 40) break; @@ -74,7 +74,7 @@ describe("google_maps", { await t.step("callback", async () => { const placeIds: string[] = []; await new Promise((done) => { - getJson(engine, { q, type, ll }, (page) => { + getJson({ engine, q, type, ll }, (page) => { placeIds.push(...page.local_results.map((r: any) => r.place_id)); if (placeIds.length < 40 && page.next) { page.next(); @@ -94,7 +94,8 @@ describe("google_maps", { config.api_key = null; await t.step("async/await", async () => { - const page1 = await getJson(engine, { + const page1 = await getJson({ + engine, api_key: SERPAPI_TEST_KEY, no_cache: false, q, @@ -113,7 +114,8 @@ describe("google_maps", { await t.step("callback", async () => { const page1 = await new Promise>>( (res) => - getJson(engine, { + getJson({ + engine, api_key: SERPAPI_TEST_KEY, no_cache: false, q, @@ -136,7 +138,7 @@ describe("google_maps", { it("getJson pagination with offset", { ignore: !HAS_API_KEY, }, async (t) => { - const firstPage = await getJson(engine, { q, type, ll }); + const firstPage = await getJson({ engine, q, type, ll }); const placeIdsOnFirstPage: string[] = firstPage.local_results.map(( r: any, ) => r.place_id); @@ -144,7 +146,7 @@ describe("google_maps", { await t.step("async/await", async () => { const placeIds: string[] = []; let page; - page = await getJson(engine, { q, type, ll, start: 40 }); + page = await getJson({ engine, q, type, ll, start: 40 }); while (page) { placeIds.push(...page.local_results.map((r: any) => r.place_id)); if (placeIds.length >= 40) break; @@ -161,7 +163,7 @@ describe("google_maps", { await t.step("callback", async () => { const placeIds: string[] = []; await new Promise((done) => { - getJson(engine, { q, type, ll, start: 20 }, (page) => { + getJson({ engine, q, type, ll, start: 20 }, (page) => { placeIds.push(...page.local_results.map((r: any) => r.place_id)); if (placeIds.length < 40 && page.next) { page.next(); @@ -184,7 +186,7 @@ describe("google_maps", { await t.step("async/await", async () => { let page; let pageNum = 0; - page = await getJson(engine, { q, type, ll, start: 260 }); + page = await getJson({ engine, q, type, ll, start: 260 }); while (page && pageNum < 5) { pageNum++; page = await page.next?.(); @@ -195,7 +197,7 @@ describe("google_maps", { await t.step("callback", async () => { let pageNum = 0; await new Promise((done) => { - getJson(engine, { q, type, ll, start: 260 }, (page) => { + getJson({ engine, q, type, ll, start: 260 }, (page) => { pageNum++; if (pageNum < 5 && page.next) { page.next(); diff --git a/tests/engines/google_scholar_profiles_test.ts b/tests/engines/google_scholar_profiles_test.ts index d49f0fe..63bed96 100644 --- a/tests/engines/google_scholar_profiles_test.ts +++ b/tests/engines/google_scholar_profiles_test.ts @@ -60,7 +60,7 @@ describe("google_scholar_profiles", { await t.step("async/await", async () => { const authorIds: string[] = []; let page; - page = await getJson(engine, { mauthors }); + page = await getJson({ engine, mauthors }); while (page) { authorIds.push(...page.profiles.map((r: any) => r.author_id)); if (authorIds.length >= 20) break; @@ -72,7 +72,7 @@ describe("google_scholar_profiles", { await t.step("callback", async () => { const authorIds: string[] = []; await new Promise((done) => { - getJson(engine, { mauthors }, (page) => { + getJson({ engine, mauthors }, (page) => { authorIds.push(...page.profiles.map((r: any) => r.author_id)); if (authorIds.length < 20 && page.next) { page.next(); @@ -92,7 +92,8 @@ describe("google_scholar_profiles", { config.api_key = null; await t.step("async/await", async () => { - const page1 = await getJson(engine, { + const page1 = await getJson({ + engine, api_key: SERPAPI_TEST_KEY, no_cache: false, mauthors, @@ -109,7 +110,8 @@ describe("google_scholar_profiles", { await t.step("callback", async () => { const page1 = await new Promise>>( (res) => - getJson(engine, { + getJson({ + engine, api_key: SERPAPI_TEST_KEY, no_cache: false, mauthors, @@ -130,7 +132,7 @@ describe("google_scholar_profiles", { it("getJson pagination with token", { ignore: !HAS_API_KEY, }, async (t) => { - const firstPage = await getJson(engine, { mauthors }); + const firstPage = await getJson({ engine, mauthors }); const authorIdsOnFirstPage = firstPage.profiles.map((r: any) => r.author_id ); @@ -138,7 +140,7 @@ describe("google_scholar_profiles", { await t.step("async/await", async () => { const authorIds: string[] = []; let page; - page = await getJson(engine, { mauthors, after_author: "rZlDAYoq__8J" }); + page = await getJson({ engine, mauthors, after_author: "rZlDAYoq__8J" }); while (page) { authorIds.push(...page.profiles.map((r: any) => r.author_id)); if (authorIds.length >= 20) break; @@ -154,7 +156,7 @@ describe("google_scholar_profiles", { await t.step("callback", async () => { const authorIds: string[] = []; await new Promise((done) => { - getJson(engine, { mauthors, after_author: "rZlDAYoq__8J" }, (page) => { + getJson({ engine, mauthors, after_author: "rZlDAYoq__8J" }, (page) => { authorIds.push(...page.profiles.map((r: any) => r.author_id)); if (authorIds.length < 20 && page.next) { page.next(); @@ -177,7 +179,7 @@ describe("google_scholar_profiles", { await t.step("async/await", async () => { let page; let pageNum = 0; - page = await getJson(engine, { mauthors, after_author: "UXxiAf3___8J" }); + page = await getJson({ engine, mauthors, after_author: "UXxiAf3___8J" }); while (page && pageNum < 5) { pageNum++; page = await page.next?.(); @@ -188,7 +190,7 @@ describe("google_scholar_profiles", { await t.step("callback", async () => { let pageNum = 0; await new Promise((done) => { - getJson(engine, { mauthors, after_author: "UXxiAf3___8J" }, (page) => { + getJson({ engine, mauthors, after_author: "UXxiAf3___8J" }, (page) => { pageNum++; if (pageNum < 5 && page.next) { page.next(); diff --git a/tests/engines/google_test.ts b/tests/engines/google_test.ts index b1ab2b2..7978ed5 100644 --- a/tests/engines/google_test.ts +++ b/tests/engines/google_test.ts @@ -58,7 +58,8 @@ describe("google", { }); it("getJson for an unmetered query (async/await)", async () => { - const response = await getJson(engine, { + const response = await getJson({ + engine, api_key: null, // null to support the "coffee" unmetered query q: "coffee", }); @@ -74,11 +75,7 @@ describe("google", { it("getJson for an unmetered query (callback)", async () => { const response = await new Promise>>( - (res) => - getJson(engine, { - api_key: null, - q: "coffee", - }, res), + (res) => getJson({ engine, api_key: null, q: "coffee" }, res), ); assertArrayIncludes(Object.keys(response).sort(), [ "organic_results", @@ -94,10 +91,7 @@ describe("google", { const executeSpy = spy(_internals, "execute"); config.api_key = "test_initial_api_key"; try { - await getJson(engine, { - api_key: "test_override_api_key", - q: "coffee", - }); + await getJson({ engine, api_key: "test_override_api_key", q: "coffee" }); } finally { executeSpy.restore(); } @@ -111,29 +105,25 @@ describe("google", { }); it("getJson with no api key from params or config", () => { - assertRejects(async () => - await getJson(engine, { - api_key: "", - q: "coffee", - }), MissingApiKeyError); - assertRejects(async () => - await getJson(engine, { - q: "coffee", - }), MissingApiKeyError); - assertRejects(async () => - await getJson(engine, { - api_key: undefined, - q: "coffee", - }), MissingApiKeyError); + assertRejects( + async () => await getJson({ engine, api_key: "", q: "coffee" }), + MissingApiKeyError, + ); + assertRejects( + async () => await getJson({ engine, q: "coffee" }), + MissingApiKeyError, + ); + assertRejects( + async () => await getJson({ engine, api_key: undefined, q: "coffee" }), + MissingApiKeyError, + ); }); it("getJson with api key from config", { ignore: !HAS_API_KEY, }, async () => { config.api_key = SERPAPI_TEST_KEY; - const response = await getJson(engine, { - q: "serpapi", - }); + const response = await getJson({ engine, q: "serpapi" }); assertArrayIncludes(Object.keys(response).sort(), [ "organic_results", "pagination", @@ -145,10 +135,7 @@ describe("google", { }); it("getHtml for an unmetered query (async/await)", async () => { - const response = await getHtml(engine, { - api_key: null, - q: "coffee", - }); + const response = await getHtml({ engine, api_key: null, q: "coffee" }); assertStringIncludes(response, ""); @@ -157,11 +144,7 @@ describe("google", { it("getHtml for an unmetered query (callback)", async () => { const response = await new Promise>>( - (res) => - getHtml(engine, { - api_key: null, - q: "coffee", - }, res), + (res) => getHtml({ engine, api_key: null, q: "coffee" }, res), ); assertStringIncludes(response, " { - assertRejects(async () => - await getHtml(engine, { - api_key: "", - q: "coffee", - }), MissingApiKeyError); - assertRejects(async () => - await getHtml(engine, { - q: "coffee", - }), MissingApiKeyError); - assertRejects(async () => - await getHtml(engine, { - api_key: undefined, - q: "coffee", - }), MissingApiKeyError); + assertRejects( + async () => await getHtml({ engine, api_key: "", q: "coffee" }), + MissingApiKeyError, + ); + assertRejects( + async () => await getHtml({ engine, q: "coffee" }), + MissingApiKeyError, + ); + assertRejects( + async () => await getHtml({ engine, api_key: undefined, q: "coffee" }), + MissingApiKeyError, + ); }); it("getHtml with api key from config", { ignore: !HAS_API_KEY, }, async () => { config.api_key = SERPAPI_TEST_KEY; - const response = await getHtml(engine, { - q: "serpapi", - }); + const response = await getHtml({ engine, q: "serpapi" }); assertStringIncludes(response, ""); @@ -220,7 +196,8 @@ describe("google", { }); it("getHtml with async parameter returns json", async () => { - const response = await getHtml(engine, { + const response = await getHtml({ + engine, api_key: null, async: true, no_cache: true, @@ -241,7 +218,8 @@ describe("google", { let id: string; await t.step("initiate async request", async () => { - const response = await getJson(engine, { + const response = await getJson({ + engine, api_key: SERPAPI_TEST_KEY, async: true, no_cache: true, // Ensure a new request is sent so we don't get cached results diff --git a/tests/engines/home_depot_test.ts b/tests/engines/home_depot_test.ts index 15665d3..77c66a1 100644 --- a/tests/engines/home_depot_test.ts +++ b/tests/engines/home_depot_test.ts @@ -60,7 +60,7 @@ describe("home_depot", { await t.step("async/await", async () => { const ids: string[] = []; let page; - page = await getJson(engine, { q }); + page = await getJson({ engine, q }); while (page) { ids.push(...page.products.map((r: any) => r.product_id)); if (ids.length >= 48) break; @@ -72,7 +72,7 @@ describe("home_depot", { await t.step("callback", async () => { const ids: string[] = []; await new Promise((done) => { - getJson(engine, { q }, (page) => { + getJson({ engine, q }, (page) => { ids.push(...page.products.map((r: any) => r.product_id)); if (ids.length < 48 && page.next) { page.next(); @@ -92,7 +92,8 @@ describe("home_depot", { config.api_key = null; await t.step("async/await", async () => { - const page1 = await getJson(engine, { + const page1 = await getJson({ + engine, api_key: SERPAPI_TEST_KEY, no_cache: false, q, @@ -109,11 +110,10 @@ describe("home_depot", { await t.step("callback", async () => { const page1 = await new Promise>>( (res) => - getJson(engine, { - api_key: SERPAPI_TEST_KEY, - no_cache: false, - q, - }, res), + getJson( + { engine, api_key: SERPAPI_TEST_KEY, no_cache: false, q }, + res, + ), ); assertMatch(executeSpy.calls[0].args[1].api_key as string, /[a-z0-9]+/); assertEquals(executeSpy.calls[0].args[1].no_cache, false); @@ -130,13 +130,13 @@ describe("home_depot", { it("getJson pagination with offset + size", { ignore: !HAS_API_KEY, }, async (t) => { - const firstPage = await getJson(engine, { q, ps: 3 }); + const firstPage = await getJson({ engine, q, ps: 3 }); const idsOnFirstPage = firstPage.products.map((r: any) => r.product_id); await t.step("async/await", async () => { const ids: string[] = []; let page; - page = await getJson(engine, { q, nao: "6", ps: 3 }); + page = await getJson({ engine, q, nao: "6", ps: 3 }); while (page) { ids.push(...page.products.map((r: any) => r.product_id)); if (ids.length >= 6) break; @@ -152,7 +152,7 @@ describe("home_depot", { await t.step("callback", async () => { const ids: string[] = []; await new Promise((done) => { - getJson(engine, { q, nao: "6", ps: 3 }, (page) => { + getJson({ engine, q, nao: "6", ps: 3 }, (page) => { ids.push(...page.products.map((r: any) => r.product_id)); if (ids.length < 6 && page.next) { page.next(); @@ -172,13 +172,13 @@ describe("home_depot", { it("getJson pagination with page + size", { ignore: !HAS_API_KEY, }, async (t) => { - const firstPage = await getJson(engine, { q, ps: 3 }); + const firstPage = await getJson({ engine, q, ps: 3 }); const idsOnFirstPage = firstPage.products.map((r: any) => r.product_id); await t.step("async/await", async () => { const ids: string[] = []; let page; - page = await getJson(engine, { q, page: "3", ps: 3 }); + page = await getJson({ engine, q, page: "3", ps: 3 }); while (page) { ids.push(...page.products.map((r: any) => r.product_id)); if (ids.length >= 6) break; @@ -194,7 +194,7 @@ describe("home_depot", { await t.step("callback", async () => { const ids: string[] = []; await new Promise((done) => { - getJson(engine, { q, page: "3", ps: 3 }, (page) => { + getJson({ engine, q, page: "3", ps: 3 }, (page) => { ids.push(...page.products.map((r: any) => r.product_id)); if (ids.length < 6 && page.next) { page.next(); @@ -214,13 +214,13 @@ describe("home_depot", { it("getJson pagination with offset + page + size", { ignore: !HAS_API_KEY, }, async (t) => { - const firstPage = await getJson(engine, { q, ps: 3 }); + const firstPage = await getJson({ engine, q, ps: 3 }); const idsOnFirstPage = firstPage.products.map((r: any) => r.product_id); await t.step("async/await", async () => { const ids: string[] = []; let page; - page = await getJson(engine, { q, nao: "6", page: "3", ps: 3 }); + page = await getJson({ engine, q, nao: "6", page: "3", ps: 3 }); while (page) { ids.push(...page.products.map((r: any) => r.product_id)); if (ids.length >= 6) break; @@ -236,7 +236,7 @@ describe("home_depot", { await t.step("callback", async () => { const ids: string[] = []; await new Promise((done) => { - getJson(engine, { q, nao: "6", page: "3", ps: 3 }, (page) => { + getJson({ engine, q, nao: "6", page: "3", ps: 3 }, (page) => { ids.push(...page.products.map((r: any) => r.product_id)); if (ids.length < 6 && page.next) { page.next(); @@ -259,7 +259,7 @@ describe("home_depot", { await t.step("async/await", async () => { let page; let pageNum = 0; - page = await getJson(engine, { q, page: "17", ps: 48 }); + page = await getJson({ engine, q, page: "17", ps: 48 }); while (page && pageNum < 5) { pageNum++; page = await page.next?.(); @@ -270,7 +270,7 @@ describe("home_depot", { await t.step("callback", async () => { let pageNum = 0; await new Promise((done) => { - getJson(engine, { q, page: "17", ps: 48 }, (page) => { + getJson({ engine, q, page: "17", ps: 48 }, (page) => { pageNum++; if (pageNum < 5 && page.next) { page.next(); From 0b49cf718152d3c1d45fee4e31cae3bb7d451523 Mon Sep 17 00:00:00 2001 From: Sebastian Quek Date: Mon, 13 Feb 2023 17:05:11 +0800 Subject: [PATCH 04/61] Update jsdoc --- README.md | 28 ++++++++++++---------------- src/serpapi.ts | 22 ++++++++++------------ 2 files changed, 22 insertions(+), 28 deletions(-) diff --git a/README.md b/README.md index 469f571..e9fc321 100644 --- a/README.md +++ b/README.md @@ -134,9 +134,6 @@ Get a JSON response based on search parameters. #### Parameters -- `engine` - **[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** - engine name - `parameters` **[object](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Object)** search query parameters for the engine @@ -146,21 +143,21 @@ Get a JSON response based on search parameters. ```javascript // single call (async/await) -const json = await getJson("google", { api_key: API_KEY, q: "coffee" }); +const json = await getJson({ engine: "google", api_key: API_KEY, q: "coffee" }); // single call (callback) -getJson("google", { api_key: API_KEY, q: "coffee" }, console.log); +getJson({ engine: "google", api_key: API_KEY, q: "coffee" }, console.log); ``` ```javascript // pagination (async/await) -const page1 = await getJson("google", { q: "coffee", start: 15 }); +const page1 = await getJson({ engine: "google", q: "coffee", start: 15 }); const page2 = await page1.next?.(); ``` ```javascript // pagination (callback) -getJson("google", { q: "coffee", start: 15 }, (page1) => { +getJson({ engine: "google", q: "coffee", start: 15 }, (page1) => { page1.next?.((page2) => { console.log(page2); }); @@ -170,7 +167,7 @@ getJson("google", { q: "coffee", start: 15 }, (page1) => { ```javascript // pagination loop (async/await) const organicResults = []; -let page = await getJson("google", { api_key: API_KEY, q: "coffee" }); +let page = await getJson({ engine: "google", api_key: API_KEY, q: "coffee" }); while (page) { organicResults.push(...page.organic_results); if (organicResults.length >= 30) break; @@ -181,7 +178,7 @@ while (page) { ```javascript // pagination loop (callback) const organicResults = []; -getJson("google", { api_key: API_KEY, q: "coffee" }, (page) => { +getJson({ engine: "google", api_key: API_KEY, q: "coffee" }, (page) => { organicResults.push(...page.organic_results); if (organicResults.length < 30 && page.next) { page.next(); @@ -198,9 +195,6 @@ Get a HTML response based on search parameters. #### Parameters -- `engine` - **[string](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/String)** - engine name - `parameters` **[object](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Object)** search query parameters for the engine @@ -210,10 +204,10 @@ Get a HTML response based on search parameters. ```javascript // async/await -const html = await getHtml("google", { api_key: API_KEY, q: "coffee" }); +const html = await getHtml({ engine: "google", api_key: API_KEY, q: "coffee" }); // callback -getHtml("google", { api_key: API_KEY, q: "coffee" }, console.log); +getHtml({ engine: "google", api_key: API_KEY, q: "coffee" }, console.log); ``` ### getJsonBySearchId @@ -245,7 +239,8 @@ Get a JSON response given a search ID. #### Examples ```javascript -const response = await getJson("google", { +const response = await getJson({ + engine: "google", api_key: API_KEY, async: true, q: "coffee", @@ -290,7 +285,8 @@ Get a HTML response given a search ID. #### Examples ```javascript -const response = await getJson("google", { +const response = await getJson({ + engine: "google", api_key: API_KEY, async: true, q: "coffee", diff --git a/src/serpapi.ts b/src/serpapi.ts index 08c1786..1d3765e 100644 --- a/src/serpapi.ts +++ b/src/serpapi.ts @@ -27,24 +27,23 @@ const SEARCH_ARCHIVE_PATH = `/searches`; * - Accepts an optional callback. * - Get the next page of results by calling the `.next()` method on the returned response object. * - * @param {string} engine - engine name * @param {object} parameters - search query parameters for the engine * @param {fn=} callback - optional callback * @example * // single call (async/await) - * const json = await getJson("google", { api_key: API_KEY, q: "coffee" }); + * const json = await getJson({ engine: "google", api_key: API_KEY, q: "coffee" }); * * // single call (callback) - * getJson("google", { api_key: API_KEY, q: "coffee" }, console.log); + * getJson({ engine: "google", api_key: API_KEY, q: "coffee" }, console.log); * * @example * // pagination (async/await) - * const page1 = await getJson("google", { q: "coffee", start: 15 }); + * const page1 = await getJson({ engine: "google", q: "coffee", start: 15 }); * const page2 = await page1.next?.(); * * @example * // pagination (callback) - * getJson("google", { q: "coffee", start: 15 }, (page1) => { + * getJson({ engine: "google", q: "coffee", start: 15 }, (page1) => { * page1.next?.((page2) => { * console.log(page2); * }); @@ -53,7 +52,7 @@ const SEARCH_ARCHIVE_PATH = `/searches`; * @example * // pagination loop (async/await) * const organicResults = []; - * let page = await getJson("google", { api_key: API_KEY, q: "coffee" }); + * let page = await getJson({ engine: "google", api_key: API_KEY, q: "coffee" }); * while (page) { * organicResults.push(...page.organic_results); * if (organicResults.length >= 30) break; @@ -63,7 +62,7 @@ const SEARCH_ARCHIVE_PATH = `/searches`; * @example * // pagination loop (callback) * const organicResults = []; - * getJson("google", { api_key: API_KEY, q: "coffee" }, (page) => { + * getJson({ engine: "google", api_key: API_KEY, q: "coffee" }, (page) => { * organicResults.push(...page.organic_results); * if (organicResults.length < 30 && page.next) { * page.next(); @@ -114,15 +113,14 @@ export async function getJson< * - Accepts an optional callback. * - Responds with a JSON string if the search request hasn't completed. * - * @param {string} engine - engine name * @param {object} parameters - search query parameters for the engine * @param {fn=} callback - optional callback * @example * // async/await - * const html = await getHtml("google", { api_key: API_KEY, q: "coffee" }); + * const html = await getHtml({ engine: "google", api_key: API_KEY, q: "coffee" }); * * // callback - * getHtml("google", { api_key: API_KEY, q: "coffee" }, console.log); + * getHtml({ engine: "google", api_key: API_KEY, q: "coffee" }, console.log); */ export async function getHtml< E extends EngineName = EngineName, @@ -159,7 +157,7 @@ export async function getHtml< * @param {number=} [parameters.timeout] - timeout in milliseconds * @param {fn=} callback - optional callback * @example - * const response = await getJson("google", { api_key: API_KEY, async: true, q: "coffee" }); + * const response = await getJson({ engine: "google", api_key: API_KEY, async: true, q: "coffee" }); * const { id } = response.search_metadata; * await delay(1000); // wait for the request to be processed. * @@ -204,7 +202,7 @@ export async function getJsonBySearchId< * @param {number=} [parameters.timeout] - timeout in milliseconds * @param {fn=} callback - optional callback * @example - * const response = await getJson("google", { api_key: API_KEY, async: true, q: "coffee" }); + * const response = await getJson({ engine: "google", api_key: API_KEY, async: true, q: "coffee" }); * const { id } = response.search_metadata; * await delay(1000); // wait for the request to be processed. * From a60cf2879b19713e60ef46650ea9ab2278105d89 Mon Sep 17 00:00:00 2001 From: Sebastian Quek Date: Mon, 13 Feb 2023 17:08:30 +0800 Subject: [PATCH 05/61] Update docs --- README.md | 9 +++++---- docs/migrating_from_google_search_results_nodejs.md | 8 ++++---- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index e9fc321..5d1080d 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,8 @@ npm install serpapi ```js import { getJson } from "serpapi"; -const response = await getJson("google", { +const response = await getJson({ + engine: "google", api_key: API_KEY, // Get your API_KEY from https://serpapi.com/manage-api-key q: "coffee", location: "Austin, Texas", @@ -68,8 +69,8 @@ import { config, getJson } from "serpapi"; config.api_key = API_KEY; config.timeout = 60000; -await getJson("google", { q: "coffee" }); // uses the API key defined in the config -await getJson("google", { api_key: API_KEY_2, q: "coffee" }); // API_KEY_2 will be used +await getJson({ engine: "google", q: "coffee" }); // uses the API key defined in the config +await getJson({ engine: "google", api_key: API_KEY_2, q: "coffee" }); // API_KEY_2 will be used ``` ## Pagination @@ -86,7 +87,7 @@ pagination is not supported for the search engine or there are no more pages to be retrieved. ```js -const page1 = await getJson("google", { q: "coffee", start: 15 }); +const page1 = await getJson({ engine: "google", q: "coffee", start: 15 }); const page2 = await page1.next?.(); ``` diff --git a/docs/migrating_from_google_search_results_nodejs.md b/docs/migrating_from_google_search_results_nodejs.md index 2e869ab..de3fae0 100644 --- a/docs/migrating_from_google_search_results_nodejs.md +++ b/docs/migrating_from_google_search_results_nodejs.md @@ -17,7 +17,7 @@ migrate over to the `serpapi` npm package. // ✅ New way, import and use functions directly. import { getJson } from "serpapi"; - getJson("google", { api_key: API_KEY, ... }) + getJson({ engine: "google", api_key: API_KEY, ... }) ``` - The `search_archive` method is replaced by `getJsonBySearchId` and @@ -63,7 +63,7 @@ migrate over to the `serpapi` npm package. engine.json({ q: "coffee", api_key: undefined }); // ✅ Now, no error is thrown when api_key is null - getJson("google", { q: "coffee", api_key: null }); + getJson({ engine: "google", q: "coffee", api_key: null }); ``` ## Added @@ -71,7 +71,7 @@ migrate over to the `serpapi` npm package. - TypeScript types for supported parameters. - First-class Promises support. ```js - const json = await getJson("google", { q: "coffee" }); + const json = await getJson({ engine: "google", q: "coffee" }); ``` - `config` object to configure global `api_key` and `timeout` values. ```js @@ -81,6 +81,6 @@ migrate over to the `serpapi` npm package. ``` - Error classes (`MissingApiKeyError` and `InvalidTimeoutError`). ```js - getJson("google", { api_key: "" }); // Throws `MissingApiKeyError` + getJson({ engine: "google", api_key: "" }); // Throws `MissingApiKeyError` getAccount({ api_key: API_KEY, timeout: 0 }); // Throws `InvalidTimeoutError` ``` From 743e942731a81ba20366e7845f5dc650ba5ccc1c Mon Sep 17 00:00:00 2001 From: Sebastian Quek Date: Mon, 13 Feb 2023 17:08:37 +0800 Subject: [PATCH 06/61] Update examples --- examples/deno/basic_ts/example.ts | 11 ++++++----- examples/deno/pagination_ts/example.ts | 15 ++++++++------- examples/node/basic_js_commonjs/example.js | 7 ++++--- examples/node/basic_js_esm/example.js | 7 ++++--- examples/node/basic_ts_esm/example.ts | 11 ++++++----- examples/node/pagination_js_commonjs/example.js | 11 ++++++----- examples/node/pagination_js_esm/example.js | 11 ++++++----- examples/node/pagination_ts_esm/example.ts | 15 ++++++++------- 8 files changed, 48 insertions(+), 40 deletions(-) diff --git a/examples/deno/basic_ts/example.ts b/examples/deno/basic_ts/example.ts index 70d8c4e..5b5c411 100644 --- a/examples/deno/basic_ts/example.ts +++ b/examples/deno/basic_ts/example.ts @@ -2,24 +2,25 @@ import { loadSync } from "https://deno.land/std@0.173.0/dotenv/mod.ts"; import { AllowArbitraryParams, config, + EngineParameters, getJson, - GoogleParameters, } from "../../../mod.ts"; const { API_KEY: apiKey } = loadSync(); const params = { + engine: "google", q: "Coffee", api_key: apiKey, -} satisfies AllowArbitraryParams; +} satisfies AllowArbitraryParams>; // Show result as JSON (async/await) -const response1 = await getJson("google", params); +const response1 = await getJson(params); console.log(response1["organic_results"]); // Show result as JSON (callback) -getJson("google", params, (json) => console.log(json["organic_results"])); +getJson(params, (json) => console.log(json["organic_results"])); // Use global config config.api_key = apiKey; -const response2 = await getJson("google", { q: "Coffee" }); +const response2 = await getJson({ engine: "google", q: "Coffee" }); console.log(response2["organic_results"]); diff --git a/examples/deno/pagination_ts/example.ts b/examples/deno/pagination_ts/example.ts index d3a9ee8..24f79fb 100644 --- a/examples/deno/pagination_ts/example.ts +++ b/examples/deno/pagination_ts/example.ts @@ -2,8 +2,8 @@ import { loadSync } from "https://deno.land/std@0.173.0/dotenv/mod.ts"; import { AllowArbitraryParams, config, + EngineParameters, getJson, - GoogleParameters, } from "../../../mod.ts"; const { API_KEY: apiKey } = loadSync(); @@ -12,12 +12,13 @@ const extractLinks = (results: { link: string }[]) => results.map((r) => r.link); const params = { + engine: "google", q: "Coffee", api_key: apiKey, -} satisfies AllowArbitraryParams; +} satisfies AllowArbitraryParams>; // Pagination (async/await) -let page1 = await getJson("google", params); +let page1 = await getJson(params); console.log( "First page links", extractLinks(page1.organic_results), @@ -29,7 +30,7 @@ console.log( ); // Pagination (callback) -getJson("google", params, (page1) => { +getJson(params, (page1) => { console.log( "First page links", extractLinks(page1.organic_results), @@ -44,7 +45,7 @@ getJson("google", params, (page1) => { // Use global config config.api_key = apiKey; -page1 = await getJson("google", { q: "Coffee" }); +page1 = await getJson({ engine: "google", q: "Coffee" }); page2 = await page1.next?.(); console.log( "Second page links", @@ -54,7 +55,7 @@ console.log( // Pagination loop (async/await) let links: string[] = []; let page; -page = await getJson("google", { q: "Coffee" }); +page = await getJson({ engine: "google", q: "Coffee" }); while (page) { links.push(...extractLinks(page.organic_results)); if (links.length >= 30) break; @@ -64,7 +65,7 @@ console.log(links); // Pagination loop (callback) links = []; -getJson("google", { q: "Coffee" }, (page) => { +getJson({ engine: "google", q: "Coffee" }, (page) => { links.push(...extractLinks(page.organic_results)); if (links.length < 30 && page.next) { page.next(); diff --git a/examples/node/basic_js_commonjs/example.js b/examples/node/basic_js_commonjs/example.js index 43136a8..c216c64 100644 --- a/examples/node/basic_js_commonjs/example.js +++ b/examples/node/basic_js_commonjs/example.js @@ -6,20 +6,21 @@ const apiKey = process.env.API_KEY; const run = async () => { const params = { + engine: "google", q: "Coffee", api_key: apiKey, }; // Show result as JSON (async/await) - const response1 = await getJson("google", params); + const response1 = await getJson(params); console.log(response1["organic_results"]); // Show result as JSON (callback) - getJson("google", params, (json) => console.log(json["organic_results"])); + getJson(params, (json) => console.log(json["organic_results"])); // Use global config config.api_key = apiKey; - const response2 = await getJson("google", { q: "Coffee" }); + const response2 = await getJson({ engine: "google", q: "Coffee" }); console.log(response2["organic_results"]); }; diff --git a/examples/node/basic_js_esm/example.js b/examples/node/basic_js_esm/example.js index caab132..cb0e1a8 100644 --- a/examples/node/basic_js_esm/example.js +++ b/examples/node/basic_js_esm/example.js @@ -5,18 +5,19 @@ Dotenv.config(); const apiKey = process.env.API_KEY; const params = { + engine: "google", q: "Coffee", api_key: apiKey, }; // Show result as JSON (async/await) -const response1 = await getJson("google", params); +const response1 = await getJson(params); console.log(response1["organic_results"]); // Show result as JSON (callback) -getJson("google", params, (json) => console.log(json["organic_results"])); +getJson(params, (json) => console.log(json["organic_results"])); // Use global config config.api_key = apiKey; -const response2 = await getJson("google", { q: "Coffee" }); +const response2 = await getJson({ engine: "google", q: "Coffee" }); console.log(response2["organic_results"]); diff --git a/examples/node/basic_ts_esm/example.ts b/examples/node/basic_ts_esm/example.ts index f5b645d..85c35f7 100644 --- a/examples/node/basic_ts_esm/example.ts +++ b/examples/node/basic_ts_esm/example.ts @@ -1,22 +1,23 @@ import * as Dotenv from "dotenv"; -import { AllowArbitraryParams, config, getJson, GoogleParameters } from "serpapi"; +import { AllowArbitraryParams, config, EngineParameters, getJson } from "serpapi"; Dotenv.config(); const apiKey = process.env.API_KEY; const params = { + engine: "google", q: "Coffee", api_key: apiKey, -} satisfies AllowArbitraryParams; +} satisfies AllowArbitraryParams>; // Show result as JSON (async/await) -const response1 = await getJson("google", params); +const response1 = await getJson(params); console.log(response1["organic_results"]); // Show result as JSON (callback) -getJson("google", params, (json) => console.log(json["organic_results"])); +getJson(params, (json) => console.log(json["organic_results"])); // Use global config config.api_key = apiKey; -const response2 = await getJson("google", { q: "Coffee" }); +const response2 = await getJson({ engine: "google", q: "Coffee" }); console.log(response2["organic_results"]); diff --git a/examples/node/pagination_js_commonjs/example.js b/examples/node/pagination_js_commonjs/example.js index c0b5065..efba943 100644 --- a/examples/node/pagination_js_commonjs/example.js +++ b/examples/node/pagination_js_commonjs/example.js @@ -8,12 +8,13 @@ const extractLinks = (results) => results.map((r) => r.link); const run = async () => { const params = { + engine: "google", q: "Coffee", api_key: apiKey, }; // Pagination (async/await) - let page1 = await getJson("google", params); + let page1 = await getJson(params); console.log( "First page links", extractLinks(page1.organic_results), @@ -25,7 +26,7 @@ const run = async () => { ); // Pagination (callback) - getJson("google", params, (page1) => { + getJson(params, (page1) => { console.log( "First page links", extractLinks(page1.organic_results), @@ -40,7 +41,7 @@ const run = async () => { // Use global config config.api_key = apiKey; - page1 = await getJson("google", { q: "Coffee" }); + page1 = await getJson({ engine: "google", q: "Coffee" }); page2 = await page1.next?.(); console.log( "Second page links", @@ -49,7 +50,7 @@ const run = async () => { // Pagination loop (async/await) let links = []; - let page = await getJson("google", { q: "Coffee" }); + let page = await getJson({ engine: "google", q: "Coffee" }); while (page) { links.push(...extractLinks(page.organic_results)); if (links.length >= 30) break; @@ -59,7 +60,7 @@ const run = async () => { // Pagination loop (callback) links = []; - getJson("google", { q: "Coffee" }, (page) => { + getJson({ engine: "google", q: "Coffee" }, (page) => { links.push(...extractLinks(page.organic_results)); if (links.length < 30 && page.next) { page.next(); diff --git a/examples/node/pagination_js_esm/example.js b/examples/node/pagination_js_esm/example.js index d5bf869..62c154f 100644 --- a/examples/node/pagination_js_esm/example.js +++ b/examples/node/pagination_js_esm/example.js @@ -7,12 +7,13 @@ const apiKey = process.env.API_KEY; const extractLinks = (results) => results.map((r) => r.link); const params = { + engine: "google", q: "Coffee", api_key: apiKey, }; // Pagination (async/await) -let page1 = await getJson("google", params); +let page1 = await getJson(params); console.log( "First page links", extractLinks(page1.organic_results), @@ -24,7 +25,7 @@ console.log( ); // Pagination (callback) -getJson("google", params, (page1) => { +getJson(params, (page1) => { console.log( "First page links", extractLinks(page1.organic_results), @@ -39,7 +40,7 @@ getJson("google", params, (page1) => { // Use global config config.api_key = apiKey; -page1 = await getJson("google", { q: "Coffee" }); +page1 = await getJson({ engine: "google", q: "Coffee" }); page2 = await page1.next?.(); console.log( "Second page links", @@ -48,7 +49,7 @@ console.log( // Pagination loop (async/await) let links = []; -let page = await getJson("google", { q: "Coffee" }); +let page = await getJson({ engine: "google", q: "Coffee" }); while (page) { links.push(...extractLinks(page.organic_results)); if (links.length >= 30) break; @@ -58,7 +59,7 @@ console.log(links); // Pagination loop (callback) links = []; -getJson("google", { q: "Coffee" }, (page) => { +getJson({ engine: "google", q: "Coffee" }, (page) => { links.push(...extractLinks(page.organic_results)); if (links.length < 30 && page.next) { page.next(); diff --git a/examples/node/pagination_ts_esm/example.ts b/examples/node/pagination_ts_esm/example.ts index 8a91163..de98846 100644 --- a/examples/node/pagination_ts_esm/example.ts +++ b/examples/node/pagination_ts_esm/example.ts @@ -1,5 +1,5 @@ import * as Dotenv from "dotenv"; -import { AllowArbitraryParams, config, getJson, GoogleParameters } from "serpapi"; +import { AllowArbitraryParams, config, EngineParameters, getJson } from "serpapi"; Dotenv.config(); const apiKey = process.env.API_KEY; @@ -8,12 +8,13 @@ const extractLinks = (results: { link: string }[]) => results.map((r) => r.link); const params = { + engine: "google", q: "Coffee", api_key: apiKey, -} satisfies AllowArbitraryParams; +} satisfies AllowArbitraryParams>; // Pagination (async/await) -let page1 = await getJson("google", params); +let page1 = await getJson(params); console.log( "First page links", extractLinks(page1.organic_results), @@ -25,7 +26,7 @@ console.log( ); // Pagination (callback) -getJson("google", params, (page1) => { +getJson(params, (page1) => { console.log( "First page links", extractLinks(page1.organic_results), @@ -40,7 +41,7 @@ getJson("google", params, (page1) => { // Use global config config.api_key = apiKey; -page1 = await getJson("google", { q: "Coffee" }); +page1 = await getJson({ engine: "google", q: "Coffee" }); page2 = await page1.next?.(); console.log( "Second page links", @@ -50,7 +51,7 @@ console.log( // Pagination loop (async/await) let links: string[] = []; let page; -page = await getJson("google", { q: "Coffee" }); +page = await getJson({ engine: "google", q: "Coffee" }); while (page) { links.push(...extractLinks(page.organic_results)); if (links.length >= 30) break; @@ -60,7 +61,7 @@ console.log(links); // Pagination loop (callback) links = []; -getJson("google", { q: "Coffee" }, (page) => { +getJson({ engine: "google", q: "Coffee" }, (page) => { links.push(...extractLinks(page.organic_results)); if (links.length < 30 && page.next) { page.next(); From 84550e9969413ac80479d53ce3df0a14a7fae73c Mon Sep 17 00:00:00 2001 From: Sebastian Quek Date: Tue, 14 Feb 2023 17:04:54 +0800 Subject: [PATCH 07/61] Fix params showing as unknown type --- src/types.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/types.ts b/src/types.ts index 186d92c..77a2a99 100644 --- a/src/types.ts +++ b/src/types.ts @@ -56,8 +56,7 @@ export type EngineParameters< > = { [K in E]: & { engine: K } - & (K extends keyof EngineMap ? EngineMap[K]["parameters"] - : BaseParameters & Record); + & (K extends keyof EngineMap ? EngineMap[K]["parameters"] : BaseParameters); }[E]; export type BaseResponse = { From c0f4f2387ae798dbc0c5c2e5313433f8089ad8df Mon Sep 17 00:00:00 2001 From: Sebastian Quek Date: Fri, 17 Feb 2023 19:27:32 +0800 Subject: [PATCH 08/61] Support node 12 --- deno.json | 6 +++ .../node/pagination_js_commonjs/example.js | 40 +++++++++++-------- scripts/build_npm.ts | 8 +++- src/utils.ts | 17 ++++++-- tests/utils_test.ts | 1 + 5 files changed, 50 insertions(+), 22 deletions(-) diff --git a/deno.json b/deno.json index 0aadab2..e5c46d3 100644 --- a/deno.json +++ b/deno.json @@ -16,5 +16,11 @@ "files": { "exclude": ["npm/", "examples/node"] } + }, + "compilerOptions": { + "lib": [ + "dom", + "deno.ns" + ] } } diff --git a/examples/node/pagination_js_commonjs/example.js b/examples/node/pagination_js_commonjs/example.js index c0b5065..5b6d02a 100644 --- a/examples/node/pagination_js_commonjs/example.js +++ b/examples/node/pagination_js_commonjs/example.js @@ -18,11 +18,13 @@ const run = async () => { "First page links", extractLinks(page1.organic_results), ); - let page2 = await page1.next?.(); - console.log( - "Second page links", - extractLinks(page2?.organic_results), - ); + if (page1.next) { + let page2 = await page1.next(); + console.log( + "Second page links", + extractLinks(page2.organic_results), + ); + } // Pagination (callback) getJson("google", params, (page1) => { @@ -30,22 +32,26 @@ const run = async () => { "First page links", extractLinks(page1.organic_results), ); - page1.next?.((page2) => { - console.log( - "Second page links", - extractLinks(page2.organic_results), - ); - }); + if (page1.next) { + page1.next((page2) => { + console.log( + "Second page links", + extractLinks(page2.organic_results), + ); + }); + } }); // Use global config config.api_key = apiKey; page1 = await getJson("google", { q: "Coffee" }); - page2 = await page1.next?.(); - console.log( - "Second page links", - extractLinks(page2?.organic_results), - ); + if (page1.next) { + page2 = await page1.next(); + console.log( + "Second page links", + extractLinks(page2.organic_results), + ); + } // Pagination loop (async/await) let links = []; @@ -53,7 +59,7 @@ const run = async () => { while (page) { links.push(...extractLinks(page.organic_results)); if (links.length >= 30) break; - page = await page.next?.(); + page = page.next ? await page.next(): undefined; } console.log(links); diff --git a/scripts/build_npm.ts b/scripts/build_npm.ts index 3bfb545..b145b90 100644 --- a/scripts/build_npm.ts +++ b/scripts/build_npm.ts @@ -4,6 +4,7 @@ import { version } from "../version.ts"; await emptyDir("./npm"); await build({ + test: false, // Turned off to avoid publishing tests entryPoints: ["./mod.ts"], rootTestDir: "./tests", outDir: "./npm", @@ -16,8 +17,11 @@ await build({ // https://deno.land/std/async/delay.ts relies on DOMException. // This is only used in tests. domException: "dev", // Only used in tests. - - undici: true, // Required for `fetch` + }, + compilerOptions: { + // https://github.com/microsoft/TypeScript/wiki/Node-Target-Mapping + lib: ["es2019"], + target: "ES2019", }, package: { name: "serpapi", diff --git a/src/utils.ts b/src/utils.ts index 855125f..084f6f9 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -1,5 +1,6 @@ import type { EngineName, EngineParameters } from "./types.ts"; import { version } from "../version.ts"; +import fetch from "npm:cross-fetch@3.1.4"; type UrlParameters = Record< string, @@ -110,7 +111,7 @@ export function buildUrl

( return `${_internals.getBaseUrl()}${path}?${searchParams}`; } -export async function execute

( +export function execute

( path: string, parameters: P, timeout: number, @@ -119,7 +120,17 @@ export async function execute

( ...parameters, source: getSource(), }); - return await _internals.fetch(url, { - signal: AbortSignal.timeout(timeout), + // https://github.com/github/fetch/issues/175#issuecomment-216791333 + return new Promise((resolve, reject) => { + const timer = setTimeout(() => reject(new Error("Timeout")), timeout); + _internals.fetch(url) + .then((res) => { + clearTimeout(timer); + resolve(res); + }) + .catch((err) => { + clearTimeout(timer); + reject(err); + }); }); } diff --git a/tests/utils_test.ts b/tests/utils_test.ts index 8e27b11..f12c2b3 100644 --- a/tests/utils_test.ts +++ b/tests/utils_test.ts @@ -23,6 +23,7 @@ import { extractNextParameters, haveParametersChanged, } from "../src/utils.ts"; +import { Response } from "npm:cross-fetch@3.1.4"; loadSync({ export: true }); const BASE_URL = Deno.env.get("ENV_TYPE") === "local" From c1f2180f8613a08bf7c6b3f44636b7f2c7bbd8e7 Mon Sep 17 00:00:00 2001 From: Sebastian Quek Date: Thu, 16 Feb 2023 16:58:02 +0800 Subject: [PATCH 09/61] Support node 10 --- scripts/build_npm.ts | 4 ++-- src/utils.ts | 7 +++++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/scripts/build_npm.ts b/scripts/build_npm.ts index b145b90..7820f8f 100644 --- a/scripts/build_npm.ts +++ b/scripts/build_npm.ts @@ -20,8 +20,8 @@ await build({ }, compilerOptions: { // https://github.com/microsoft/TypeScript/wiki/Node-Target-Mapping - lib: ["es2019"], - target: "ES2019", + lib: ["es2018"], + target: "ES2018", }, package: { name: "serpapi", diff --git a/src/utils.ts b/src/utils.ts index 084f6f9..e65de92 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -53,8 +53,11 @@ export function extractNextParameters(json: { if (nextUrlString) { const nextUrl = new URL(nextUrlString); - const nextParameters = Object.fromEntries(nextUrl.searchParams.entries()); - delete nextParameters["engine"]; + const nextParameters: Record = {}; + for (const [k, v] of nextUrl.searchParams.entries()) { + if (k === "engine") continue; + nextParameters[k] = v; + } return nextParameters as NextParameters; } } From eef0119c3ffe4ec0e837c9055a824c1afaa6cb08 Mon Sep 17 00:00:00 2001 From: Sebastian Quek Date: Thu, 16 Feb 2023 20:02:29 +0800 Subject: [PATCH 10/61] Support node 7 --- examples/node/pagination_js_commonjs/example.js | 10 +++++----- scripts/build_npm.ts | 5 +++-- src/utils.ts | 2 ++ 3 files changed, 10 insertions(+), 7 deletions(-) diff --git a/examples/node/pagination_js_commonjs/example.js b/examples/node/pagination_js_commonjs/example.js index 5b6d02a..f3b048f 100644 --- a/examples/node/pagination_js_commonjs/example.js +++ b/examples/node/pagination_js_commonjs/example.js @@ -16,13 +16,13 @@ const run = async () => { let page1 = await getJson("google", params); console.log( "First page links", - extractLinks(page1.organic_results), + extractLinks(page1.organic_results) ); if (page1.next) { let page2 = await page1.next(); console.log( "Second page links", - extractLinks(page2.organic_results), + extractLinks(page2.organic_results) ); } @@ -30,13 +30,13 @@ const run = async () => { getJson("google", params, (page1) => { console.log( "First page links", - extractLinks(page1.organic_results), + extractLinks(page1.organic_results) ); if (page1.next) { page1.next((page2) => { console.log( "Second page links", - extractLinks(page2.organic_results), + extractLinks(page2.organic_results) ); }); } @@ -49,7 +49,7 @@ const run = async () => { page2 = await page1.next(); console.log( "Second page links", - extractLinks(page2.organic_results), + extractLinks(page2.organic_results) ); } diff --git a/scripts/build_npm.ts b/scripts/build_npm.ts index 7820f8f..2a268da 100644 --- a/scripts/build_npm.ts +++ b/scripts/build_npm.ts @@ -5,6 +5,7 @@ await emptyDir("./npm"); await build({ test: false, // Turned off to avoid publishing tests + typeCheck: false, entryPoints: ["./mod.ts"], rootTestDir: "./tests", outDir: "./npm", @@ -20,8 +21,8 @@ await build({ }, compilerOptions: { // https://github.com/microsoft/TypeScript/wiki/Node-Target-Mapping - lib: ["es2018"], - target: "ES2018", + lib: ["es2017"], + target: "ES2017", }, package: { name: "serpapi", diff --git a/src/utils.ts b/src/utils.ts index e65de92..82a692b 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -1,6 +1,8 @@ import type { EngineName, EngineParameters } from "./types.ts"; import { version } from "../version.ts"; import fetch from "npm:cross-fetch@3.1.4"; +import core from "npm:core-js-pure@3.28.0"; +const { globalThis, URL, URLSearchParams } = core; type UrlParameters = Record< string, From 7e8868046beb0c227c91c89bdf33e42f1222c26c Mon Sep 17 00:00:00 2001 From: Sebastian Quek Date: Fri, 17 Feb 2023 18:35:04 +0800 Subject: [PATCH 11/61] Update examples --- .../.env.example | 0 .../{basic_js_esm => basic_js_node_14_up}/README.md | 4 ++-- .../{basic_js_esm => basic_js_node_14_up}/example.js | 8 ++++++++ .../{basic_js_esm => basic_js_node_14_up}/package.json | 0 .../{basic_js_esm => basic_js_node_7_up}/.env.example | 0 .../README.md | 4 ++-- .../example.js | 4 ++++ .../package.json | 0 .../{basic_ts_esm => basic_ts_node_14_up}/.env.example | 0 .../README.md | 4 ++-- .../{basic_ts_esm => basic_ts_node_14_up}/example.ts | 8 ++++++++ .../{basic_ts_esm => basic_ts_node_14_up}/package.json | 0 .../tsconfig.json | 0 .../.env.example | 0 .../README.md | 4 ++-- .../example.js | 10 ++++++++++ .../package.json | 0 .../.env.example | 0 .../README.md | 4 ++-- .../example.js | 4 ++++ .../package.json | 0 .../.env.example | 0 .../README.md | 4 ++-- .../example.ts | 10 ++++++++++ .../package.json | 0 .../tsconfig.json | 0 26 files changed, 56 insertions(+), 12 deletions(-) rename examples/node/{basic_js_commonjs => basic_js_node_14_up}/.env.example (100%) rename examples/node/{basic_js_esm => basic_js_node_14_up}/README.md (88%) rename examples/node/{basic_js_esm => basic_js_node_14_up}/example.js (59%) rename examples/node/{basic_js_esm => basic_js_node_14_up}/package.json (100%) rename examples/node/{basic_js_esm => basic_js_node_7_up}/.env.example (100%) rename examples/node/{basic_js_commonjs => basic_js_node_7_up}/README.md (88%) rename examples/node/{basic_js_commonjs => basic_js_node_7_up}/example.js (92%) rename examples/node/{basic_js_commonjs => basic_js_node_7_up}/package.json (100%) rename examples/node/{basic_ts_esm => basic_ts_node_14_up}/.env.example (100%) rename examples/node/{pagination_ts_esm => basic_ts_node_14_up}/README.md (89%) rename examples/node/{basic_ts_esm => basic_ts_node_14_up}/example.ts (62%) rename examples/node/{basic_ts_esm => basic_ts_node_14_up}/package.json (100%) rename examples/node/{basic_ts_esm => basic_ts_node_14_up}/tsconfig.json (100%) rename examples/node/{pagination_js_commonjs => pagination_js_node_14_up}/.env.example (100%) rename examples/node/{pagination_js_esm => pagination_js_node_14_up}/README.md (87%) rename examples/node/{pagination_js_esm => pagination_js_node_14_up}/example.js (71%) rename examples/node/{pagination_js_esm => pagination_js_node_14_up}/package.json (100%) rename examples/node/{pagination_js_esm => pagination_js_node_7_up}/.env.example (100%) rename examples/node/{pagination_js_commonjs => pagination_js_node_7_up}/README.md (86%) rename examples/node/{pagination_js_commonjs => pagination_js_node_7_up}/example.js (97%) rename examples/node/{pagination_js_commonjs => pagination_js_node_7_up}/package.json (100%) rename examples/node/{pagination_ts_esm => pagination_ts_node_14_up}/.env.example (100%) rename examples/node/{basic_ts_esm => pagination_ts_node_14_up}/README.md (88%) rename examples/node/{pagination_ts_esm => pagination_ts_node_14_up}/example.ts (73%) rename examples/node/{pagination_ts_esm => pagination_ts_node_14_up}/package.json (100%) rename examples/node/{pagination_ts_esm => pagination_ts_node_14_up}/tsconfig.json (100%) diff --git a/examples/node/basic_js_commonjs/.env.example b/examples/node/basic_js_node_14_up/.env.example similarity index 100% rename from examples/node/basic_js_commonjs/.env.example rename to examples/node/basic_js_node_14_up/.env.example diff --git a/examples/node/basic_js_esm/README.md b/examples/node/basic_js_node_14_up/README.md similarity index 88% rename from examples/node/basic_js_esm/README.md rename to examples/node/basic_js_node_14_up/README.md index aed73cb..2818395 100644 --- a/examples/node/basic_js_esm/README.md +++ b/examples/node/basic_js_node_14_up/README.md @@ -1,4 +1,4 @@ -# Node.js basic JavaScript (ESM) example +# Basic JavaScript example for Node.js 14 and newer ## Usage @@ -17,7 +17,7 @@ deno task npm 3. Install dependencies and run example ```bash -cd examples/node/basic_js_esm +cd examples/node/basic_js_node_14_up npm i npm start ``` diff --git a/examples/node/basic_js_esm/example.js b/examples/node/basic_js_node_14_up/example.js similarity index 59% rename from examples/node/basic_js_esm/example.js rename to examples/node/basic_js_node_14_up/example.js index caab132..bb171f7 100644 --- a/examples/node/basic_js_esm/example.js +++ b/examples/node/basic_js_node_14_up/example.js @@ -1,3 +1,11 @@ +/** + * Example works for Node.js 14 and newer. + * - Uses ESM imports which is supported from Node.js 13.2.0. + * https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Statements/import#browser_compatibility + * - Uses top-level await which is supported from Node.js 14.8.0. + * https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Operators/await#browser_compatibility + */ + import * as Dotenv from "dotenv"; import { config, getJson } from "serpapi"; diff --git a/examples/node/basic_js_esm/package.json b/examples/node/basic_js_node_14_up/package.json similarity index 100% rename from examples/node/basic_js_esm/package.json rename to examples/node/basic_js_node_14_up/package.json diff --git a/examples/node/basic_js_esm/.env.example b/examples/node/basic_js_node_7_up/.env.example similarity index 100% rename from examples/node/basic_js_esm/.env.example rename to examples/node/basic_js_node_7_up/.env.example diff --git a/examples/node/basic_js_commonjs/README.md b/examples/node/basic_js_node_7_up/README.md similarity index 88% rename from examples/node/basic_js_commonjs/README.md rename to examples/node/basic_js_node_7_up/README.md index 2ded14b..3b73b0c 100644 --- a/examples/node/basic_js_commonjs/README.md +++ b/examples/node/basic_js_node_7_up/README.md @@ -1,4 +1,4 @@ -# Node.js basic JavaScript (CommonJS) example +# Basic JavaScript example for Node.js 7 and newer ## Usage @@ -17,7 +17,7 @@ deno task npm 3. Install dependencies and run example ```bash -cd examples/node/basic_js_commonjs +cd examples/node/basic_js_node_7_up npm i npm start ``` diff --git a/examples/node/basic_js_commonjs/example.js b/examples/node/basic_js_node_7_up/example.js similarity index 92% rename from examples/node/basic_js_commonjs/example.js rename to examples/node/basic_js_node_7_up/example.js index 43136a8..301baf4 100644 --- a/examples/node/basic_js_commonjs/example.js +++ b/examples/node/basic_js_node_7_up/example.js @@ -1,3 +1,7 @@ +/** + * Example works for Node.js 7 and newer. + */ + const Dotenv = require("dotenv"); const { config, getJson } = require("serpapi"); diff --git a/examples/node/basic_js_commonjs/package.json b/examples/node/basic_js_node_7_up/package.json similarity index 100% rename from examples/node/basic_js_commonjs/package.json rename to examples/node/basic_js_node_7_up/package.json diff --git a/examples/node/basic_ts_esm/.env.example b/examples/node/basic_ts_node_14_up/.env.example similarity index 100% rename from examples/node/basic_ts_esm/.env.example rename to examples/node/basic_ts_node_14_up/.env.example diff --git a/examples/node/pagination_ts_esm/README.md b/examples/node/basic_ts_node_14_up/README.md similarity index 89% rename from examples/node/pagination_ts_esm/README.md rename to examples/node/basic_ts_node_14_up/README.md index 96e2230..44fd48a 100644 --- a/examples/node/pagination_ts_esm/README.md +++ b/examples/node/basic_ts_node_14_up/README.md @@ -1,4 +1,4 @@ -# Node.js pagination TypeScript (ESM) example +# Basic TypeScript example for Node.js 14 and newer ## Usage @@ -17,7 +17,7 @@ deno task npm 3. Install dependencies and run example ```bash -cd examples/node/pagination_ts_esm +cd examples/node/basic_ts_node_14_up npm i npm start ``` diff --git a/examples/node/basic_ts_esm/example.ts b/examples/node/basic_ts_node_14_up/example.ts similarity index 62% rename from examples/node/basic_ts_esm/example.ts rename to examples/node/basic_ts_node_14_up/example.ts index f5b645d..6097c09 100644 --- a/examples/node/basic_ts_esm/example.ts +++ b/examples/node/basic_ts_node_14_up/example.ts @@ -1,3 +1,11 @@ +/** + * Example works for Node.js 14 and newer. + * - Uses ESM imports which is supported from Node.js 13.2.0. + * https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Statements/import#browser_compatibility + * - Uses top-level await which is supported from Node.js 14.8.0. + * https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Operators/await#browser_compatibility + */ + import * as Dotenv from "dotenv"; import { AllowArbitraryParams, config, getJson, GoogleParameters } from "serpapi"; diff --git a/examples/node/basic_ts_esm/package.json b/examples/node/basic_ts_node_14_up/package.json similarity index 100% rename from examples/node/basic_ts_esm/package.json rename to examples/node/basic_ts_node_14_up/package.json diff --git a/examples/node/basic_ts_esm/tsconfig.json b/examples/node/basic_ts_node_14_up/tsconfig.json similarity index 100% rename from examples/node/basic_ts_esm/tsconfig.json rename to examples/node/basic_ts_node_14_up/tsconfig.json diff --git a/examples/node/pagination_js_commonjs/.env.example b/examples/node/pagination_js_node_14_up/.env.example similarity index 100% rename from examples/node/pagination_js_commonjs/.env.example rename to examples/node/pagination_js_node_14_up/.env.example diff --git a/examples/node/pagination_js_esm/README.md b/examples/node/pagination_js_node_14_up/README.md similarity index 87% rename from examples/node/pagination_js_esm/README.md rename to examples/node/pagination_js_node_14_up/README.md index 4e4ebe2..81a9eb1 100644 --- a/examples/node/pagination_js_esm/README.md +++ b/examples/node/pagination_js_node_14_up/README.md @@ -1,4 +1,4 @@ -# Node.js pagination JavaScript (ESM) example +# Pagination JavaScript example for Node.js 14 and newer ## Usage @@ -17,7 +17,7 @@ deno task npm 3. Install dependencies and run example ```bash -cd examples/node/pagination_js_esm +cd examples/node/pagination_js_node_14_up npm i npm start ``` diff --git a/examples/node/pagination_js_esm/example.js b/examples/node/pagination_js_node_14_up/example.js similarity index 71% rename from examples/node/pagination_js_esm/example.js rename to examples/node/pagination_js_node_14_up/example.js index d5bf869..650a3e5 100644 --- a/examples/node/pagination_js_esm/example.js +++ b/examples/node/pagination_js_node_14_up/example.js @@ -1,3 +1,13 @@ +/** + * Example works for Node.js 14 and newer. + * - Uses ESM imports which is supported from Node.js 13.2.0. + * https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Statements/import#browser_compatibility + * - Uses top-level await which is supported from Node.js 14.8.0. + * https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Operators/await#browser_compatibility + * - Uses optional chaining which is supported from Node.js 14.0.0. + * https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Operators/Optional_chaining#browser_compatibility + */ + import * as Dotenv from "dotenv"; import { config, getJson } from "serpapi"; diff --git a/examples/node/pagination_js_esm/package.json b/examples/node/pagination_js_node_14_up/package.json similarity index 100% rename from examples/node/pagination_js_esm/package.json rename to examples/node/pagination_js_node_14_up/package.json diff --git a/examples/node/pagination_js_esm/.env.example b/examples/node/pagination_js_node_7_up/.env.example similarity index 100% rename from examples/node/pagination_js_esm/.env.example rename to examples/node/pagination_js_node_7_up/.env.example diff --git a/examples/node/pagination_js_commonjs/README.md b/examples/node/pagination_js_node_7_up/README.md similarity index 86% rename from examples/node/pagination_js_commonjs/README.md rename to examples/node/pagination_js_node_7_up/README.md index 0afbe7a..c5e9810 100644 --- a/examples/node/pagination_js_commonjs/README.md +++ b/examples/node/pagination_js_node_7_up/README.md @@ -1,4 +1,4 @@ -# Node.js pagination JavaScript (CommonJS) example +# Pagination JavaScript example for Node.js 7 and newer ## Usage @@ -17,7 +17,7 @@ deno task npm 3. Install dependencies and run example ```bash -cd examples/node/pagination_js_commonjs +cd examples/node/pagination_js_node_7_up npm i npm start ``` diff --git a/examples/node/pagination_js_commonjs/example.js b/examples/node/pagination_js_node_7_up/example.js similarity index 97% rename from examples/node/pagination_js_commonjs/example.js rename to examples/node/pagination_js_node_7_up/example.js index f3b048f..f998b37 100644 --- a/examples/node/pagination_js_commonjs/example.js +++ b/examples/node/pagination_js_node_7_up/example.js @@ -1,3 +1,7 @@ +/** + * Example works for Node.js 7 and newer. + */ + const Dotenv = require("dotenv"); const { config, getJson } = require("serpapi"); diff --git a/examples/node/pagination_js_commonjs/package.json b/examples/node/pagination_js_node_7_up/package.json similarity index 100% rename from examples/node/pagination_js_commonjs/package.json rename to examples/node/pagination_js_node_7_up/package.json diff --git a/examples/node/pagination_ts_esm/.env.example b/examples/node/pagination_ts_node_14_up/.env.example similarity index 100% rename from examples/node/pagination_ts_esm/.env.example rename to examples/node/pagination_ts_node_14_up/.env.example diff --git a/examples/node/basic_ts_esm/README.md b/examples/node/pagination_ts_node_14_up/README.md similarity index 88% rename from examples/node/basic_ts_esm/README.md rename to examples/node/pagination_ts_node_14_up/README.md index 3c2ec7e..c95d849 100644 --- a/examples/node/basic_ts_esm/README.md +++ b/examples/node/pagination_ts_node_14_up/README.md @@ -1,4 +1,4 @@ -# Node.js basic TypeScript (ESM) example +# Pagination TypeScript example for Node.js 14 and newer ## Usage @@ -17,7 +17,7 @@ deno task npm 3. Install dependencies and run example ```bash -cd examples/node/basic_ts_esm +cd examples/node/pagination_ts_node_14_up npm i npm start ``` diff --git a/examples/node/pagination_ts_esm/example.ts b/examples/node/pagination_ts_node_14_up/example.ts similarity index 73% rename from examples/node/pagination_ts_esm/example.ts rename to examples/node/pagination_ts_node_14_up/example.ts index 8a91163..fb486c5 100644 --- a/examples/node/pagination_ts_esm/example.ts +++ b/examples/node/pagination_ts_node_14_up/example.ts @@ -1,3 +1,13 @@ +/** + * Example works for Node.js 14 and newer. + * - Uses ESM imports which is supported from Node.js 13.2.0. + * https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Statements/import#browser_compatibility + * - Uses top-level await which is supported from Node.js 14.8.0. + * https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Operators/await#browser_compatibility + * - Uses optional chaining which is supported from Node.js 14.0.0. + * https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Operators/Optional_chaining#browser_compatibility + */ + import * as Dotenv from "dotenv"; import { AllowArbitraryParams, config, getJson, GoogleParameters } from "serpapi"; diff --git a/examples/node/pagination_ts_esm/package.json b/examples/node/pagination_ts_node_14_up/package.json similarity index 100% rename from examples/node/pagination_ts_esm/package.json rename to examples/node/pagination_ts_node_14_up/package.json diff --git a/examples/node/pagination_ts_esm/tsconfig.json b/examples/node/pagination_ts_node_14_up/tsconfig.json similarity index 100% rename from examples/node/pagination_ts_esm/tsconfig.json rename to examples/node/pagination_ts_node_14_up/tsconfig.json From 1f4f4ed39963ae0f09c6e0230054f31b1be53834 Mon Sep 17 00:00:00 2001 From: Sebastian Quek Date: Fri, 24 Feb 2023 19:42:18 +0800 Subject: [PATCH 12/61] Update docs --- CHANGELOG.md | 1 + README.md | 33 +++++++++++++++++-- ...ating_from_google_search_results_nodejs.md | 2 -- 3 files changed, 32 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 14cf866..5508284 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ and this project adheres to ### Added - Expose `EngineName`, `EngineParameters` and `AllowArbitraryParams` types. +- Add support for Node.js 7.10.1 and newer. ### Changed diff --git a/README.md b/README.md index 469f571..9056db9 100644 --- a/README.md +++ b/README.md @@ -17,12 +17,39 @@ more. ### Node.js -Ensure you're running at least Node.js v16.14. +- Supports Node.js 7.10.1 and newer. +- Refer to [this example](examples/node/basic_js_node_7_up) for help. ```bash npm install serpapi ``` +```js +const { getJson } = require("serpapi"); +getJson("google", { + api_key: API_KEY, // Get your API_KEY from https://serpapi.com/manage-api-key + q: "coffee", + location: "Austin, Texas", +}, (json) => { + console.log(json["organic_results"]); +}); +``` + +### Node.js with ES Modules (ESM) and top-level await + +- If you prefer using the `import` syntax and top-level `await`, you need to use + at least Node.js 14.8.0. +- Refer to [this example](examples/node/basic_js_node_14_up) for help. + +You will need to add `"type": "module"` to your `package.json`: + +```js +{ + "type": "module", + // rest of package.json +} +``` + ```js import { getJson } from "serpapi"; const response = await getJson("google", { @@ -35,7 +62,9 @@ console.log(response); ### Deno -Import directly from deno.land. Usage is otherwise the same as above. +- Import directly from deno.land. +- Usage is otherwise the same as above. +- Refer to [this example](examples/deno/basic_ts) for help. ```ts import { getJson } from "https://deno.land/x/serpapi/mod.ts"; diff --git a/docs/migrating_from_google_search_results_nodejs.md b/docs/migrating_from_google_search_results_nodejs.md index 2e869ab..6ef7f5d 100644 --- a/docs/migrating_from_google_search_results_nodejs.md +++ b/docs/migrating_from_google_search_results_nodejs.md @@ -51,8 +51,6 @@ migrate over to the `serpapi` npm package. - The `buildUrl`, `execute` and `search` methods are removed. Use `getJson` and `getHtml` functions instead. - The `SerpApiSearch` class is removed as a public class. -- Dropped support for Node.js 16.13 and below. This module supports Node.js - 16.14 and above. ## Fixed From 1365e13a9586944a34d2d0d41f832903cbacc7ff Mon Sep 17 00:00:00 2001 From: Sebastian Quek Date: Mon, 27 Feb 2023 12:45:42 +0800 Subject: [PATCH 13/61] Update examples to depend on published npm module --- examples/node/basic_js_node_14_up/README.md | 29 ++-------------- .../node/basic_js_node_14_up/package.json | 2 +- examples/node/basic_js_node_7_up/README.md | 28 ++-------------- examples/node/basic_js_node_7_up/package.json | 2 +- examples/node/basic_ts_node_14_up/README.md | 33 ++----------------- .../node/basic_ts_node_14_up/package.json | 2 +- .../node/pagination_js_node_14_up/README.md | 29 ++-------------- .../pagination_js_node_14_up/package.json | 2 +- .../node/pagination_js_node_7_up/README.md | 28 ++-------------- .../node/pagination_js_node_7_up/package.json | 2 +- .../node/pagination_ts_node_14_up/README.md | 33 ++----------------- .../pagination_ts_node_14_up/package.json | 2 +- 12 files changed, 18 insertions(+), 174 deletions(-) diff --git a/examples/node/basic_js_node_14_up/README.md b/examples/node/basic_js_node_14_up/README.md index 2818395..3b36e40 100644 --- a/examples/node/basic_js_node_14_up/README.md +++ b/examples/node/basic_js_node_14_up/README.md @@ -2,39 +2,14 @@ ## Usage -1. Build module - -```bash -cd ../../../ # Navigate to the root folder -deno task npm -``` - -2. Setup environment variables +1. Setup environment variables - Duplicate `.env.example` and name it `.env`. - Replace `YOUR_API_KEY` with your SerpApi API key. -3. Install dependencies and run example +2. Install dependencies and run example ```bash -cd examples/node/basic_js_node_14_up npm i npm start ``` - -## Notes - -- If you want to run the example without building the module, you can update - `package.json` to depend on the published `serpapi` npm module instead: - ```json - { - "type": "module", - "dependencies": { - "dotenv": "*", - "serpapi": "*" // Relies on the npm module - }, - "scripts": { - "start": "node example.js" - } - } - ``` diff --git a/examples/node/basic_js_node_14_up/package.json b/examples/node/basic_js_node_14_up/package.json index ddd9788..055dcb9 100644 --- a/examples/node/basic_js_node_14_up/package.json +++ b/examples/node/basic_js_node_14_up/package.json @@ -2,7 +2,7 @@ "type": "module", "dependencies": { "dotenv": "*", - "serpapi": "../../../npm" + "serpapi": "*" }, "scripts": { "start": "node example.js" diff --git a/examples/node/basic_js_node_7_up/README.md b/examples/node/basic_js_node_7_up/README.md index 3b73b0c..9085457 100644 --- a/examples/node/basic_js_node_7_up/README.md +++ b/examples/node/basic_js_node_7_up/README.md @@ -2,38 +2,14 @@ ## Usage -1. Build module - -```bash -cd ../../../ # Navigate to the root folder -deno task npm -``` - -2. Setup environment variables +1. Setup environment variables - Duplicate `.env.example` and name it `.env`. - Replace `YOUR_API_KEY` with your SerpApi API key. -3. Install dependencies and run example +2. Install dependencies and run example ```bash -cd examples/node/basic_js_node_7_up npm i npm start ``` - -## Notes - -- If you want to run the example without building the module, you can update - `package.json` to depend on the published `serpapi` npm module instead: - ```json - { - "dependencies": { - "dotenv": "*", - "serpapi": "*" // Relies on the npm module - }, - "scripts": { - "start": "node example.js" - } - } - ``` diff --git a/examples/node/basic_js_node_7_up/package.json b/examples/node/basic_js_node_7_up/package.json index 6a5d1e4..f7ceb03 100644 --- a/examples/node/basic_js_node_7_up/package.json +++ b/examples/node/basic_js_node_7_up/package.json @@ -1,7 +1,7 @@ { "dependencies": { "dotenv": "*", - "serpapi": "../../../npm" + "serpapi": "*" }, "scripts": { "start": "node example.js" diff --git a/examples/node/basic_ts_node_14_up/README.md b/examples/node/basic_ts_node_14_up/README.md index 44fd48a..20e12ba 100644 --- a/examples/node/basic_ts_node_14_up/README.md +++ b/examples/node/basic_ts_node_14_up/README.md @@ -2,43 +2,14 @@ ## Usage -1. Build module - -```bash -cd ../../../ # Navigate to the root folder -deno task npm -``` - -2. Setup environment variables +1. Setup environment variables - Duplicate `.env.example` and name it `.env`. - Replace `YOUR_API_KEY` with your SerpApi API key. -3. Install dependencies and run example +2. Install dependencies and run example ```bash -cd examples/node/basic_ts_node_14_up npm i npm start ``` - -## Notes - -- If you want to run the example without building the module, you can update - `package.json` to depend on the published `serpapi` npm module instead: - ```json - { - "type": "module", - "dependencies": { - "dotenv": "*", - "serpapi": "*" // Relies on the npm module - }, - "devDependencies": { - "@types/node": "*", - "typescript": "*" - }, - "scripts": { - "start": "npx ts-node example.ts" - } - } - ``` diff --git a/examples/node/basic_ts_node_14_up/package.json b/examples/node/basic_ts_node_14_up/package.json index dee2799..e1ef349 100644 --- a/examples/node/basic_ts_node_14_up/package.json +++ b/examples/node/basic_ts_node_14_up/package.json @@ -2,7 +2,7 @@ "type": "module", "dependencies": { "dotenv": "*", - "serpapi": "../../../npm" + "serpapi": "*" }, "devDependencies": { "@types/node": "*", diff --git a/examples/node/pagination_js_node_14_up/README.md b/examples/node/pagination_js_node_14_up/README.md index 81a9eb1..d49b924 100644 --- a/examples/node/pagination_js_node_14_up/README.md +++ b/examples/node/pagination_js_node_14_up/README.md @@ -2,39 +2,14 @@ ## Usage -1. Build module - -```bash -cd ../../../ # Navigate to the root folder -deno task npm -``` - -2. Setup environment variables +1. Setup environment variables - Duplicate `.env.example` and name it `.env`. - Replace `YOUR_API_KEY` with your SerpApi API key. -3. Install dependencies and run example +2. Install dependencies and run example ```bash -cd examples/node/pagination_js_node_14_up npm i npm start ``` - -## Notes - -- If you want to run the example without building the module, you can update - `package.json` to depend on the published `serpapi` npm module instead: - ```json - { - "type": "module", - "dependencies": { - "dotenv": "*", - "serpapi": "*" // Relies on the npm module - }, - "scripts": { - "start": "node example.js" - } - } - ``` diff --git a/examples/node/pagination_js_node_14_up/package.json b/examples/node/pagination_js_node_14_up/package.json index ddd9788..055dcb9 100644 --- a/examples/node/pagination_js_node_14_up/package.json +++ b/examples/node/pagination_js_node_14_up/package.json @@ -2,7 +2,7 @@ "type": "module", "dependencies": { "dotenv": "*", - "serpapi": "../../../npm" + "serpapi": "*" }, "scripts": { "start": "node example.js" diff --git a/examples/node/pagination_js_node_7_up/README.md b/examples/node/pagination_js_node_7_up/README.md index c5e9810..4542a22 100644 --- a/examples/node/pagination_js_node_7_up/README.md +++ b/examples/node/pagination_js_node_7_up/README.md @@ -2,38 +2,14 @@ ## Usage -1. Build module - -```bash -cd ../../../ # Navigate to the root folder -deno task npm -``` - -2. Setup environment variables +1. Setup environment variables - Duplicate `.env.example` and name it `.env`. - Replace `YOUR_API_KEY` with your SerpApi API key. -3. Install dependencies and run example +2. Install dependencies and run example ```bash -cd examples/node/pagination_js_node_7_up npm i npm start ``` - -## Notes - -- If you want to run the example without building the module, you can update - `package.json` to depend on the published `serpapi` npm module instead: - ```json - { - "dependencies": { - "dotenv": "*", - "serpapi": "*" // Relies on the npm module - }, - "scripts": { - "start": "node example.js" - } - } - ``` diff --git a/examples/node/pagination_js_node_7_up/package.json b/examples/node/pagination_js_node_7_up/package.json index 6a5d1e4..f7ceb03 100644 --- a/examples/node/pagination_js_node_7_up/package.json +++ b/examples/node/pagination_js_node_7_up/package.json @@ -1,7 +1,7 @@ { "dependencies": { "dotenv": "*", - "serpapi": "../../../npm" + "serpapi": "*" }, "scripts": { "start": "node example.js" diff --git a/examples/node/pagination_ts_node_14_up/README.md b/examples/node/pagination_ts_node_14_up/README.md index c95d849..2f6cb33 100644 --- a/examples/node/pagination_ts_node_14_up/README.md +++ b/examples/node/pagination_ts_node_14_up/README.md @@ -2,43 +2,14 @@ ## Usage -1. Build module - -```bash -cd ../../../ # Navigate to the root folder -deno task npm -``` - -2. Setup environment variables +1. Setup environment variables - Duplicate `.env.example` and name it `.env`. - Replace `YOUR_API_KEY` with your SerpApi API key. -3. Install dependencies and run example +2. Install dependencies and run example ```bash -cd examples/node/pagination_ts_node_14_up npm i npm start ``` - -## Notes - -- If you want to run the example without building the module, you can update - `package.json` to depend on the published `serpapi` npm module instead: - ```json - { - "type": "module", - "dependencies": { - "dotenv": "*", - "serpapi": "*" // Relies on the npm module - }, - "devDependencies": { - "@types/node": "*", - "typescript": "*" - }, - "scripts": { - "start": "npx ts-node example.ts" - } - } - ``` diff --git a/examples/node/pagination_ts_node_14_up/package.json b/examples/node/pagination_ts_node_14_up/package.json index dee2799..e1ef349 100644 --- a/examples/node/pagination_ts_node_14_up/package.json +++ b/examples/node/pagination_ts_node_14_up/package.json @@ -2,7 +2,7 @@ "type": "module", "dependencies": { "dotenv": "*", - "serpapi": "../../../npm" + "serpapi": "*" }, "devDependencies": { "@types/node": "*", From 79cad31fd93679daf938acc2e3816ff9c82592f3 Mon Sep 17 00:00:00 2001 From: Sebastian Quek Date: Mon, 27 Feb 2023 12:46:58 +0800 Subject: [PATCH 14/61] Add docs on running examples on local files --- CONTRIBUTING.md | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 29bbfa4..bb18f40 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -96,6 +96,25 @@ deno task test:cov # Get test coverage by running tests that hit "localhost" deno task test:ci # Run tests that hit "https://serpapi.com" ``` +## Run examples on local source files + +To run [examples](./examples/) on your local source files, follow these steps. + +1. Run `deno task npm` to build the files. +2. Update the respective example's `package.json` to depend on the local + `serpapi` module instead, + ```json + { + "dependencies": { + "dotenv": "*", + "serpapi": "../../../npm" + }, + "scripts": { + "start": "node example.js" + } + } + ``` + ## Update documentation - Every exposed function must have associated JSDoc comments. @@ -112,7 +131,8 @@ deno task docs:gen TypeScript types are generated from the backend code. Follow these steps to update the types. -1. Run `bundle exec rails sdk:generate_ts_types` in the backend repository. +1. Run `bundle exec rails libraries:generate_ts_types` in the backend + repository. 2. Replace everything in `src/engines` with the generated files from `tmp/ts/engines`. 3. Update `mod.ts` with the new engine exports from `tmp/ts/mod.ts`. From e83736f5220797f09a3b01437e76f61faa3d2302 Mon Sep 17 00:00:00 2001 From: Sebastian Quek Date: Thu, 2 Mar 2023 13:15:06 +0800 Subject: [PATCH 15/61] Add automated smoke tests --- .github/workflows/build.yml | 81 ++++++++++++++++- CONTRIBUTING.md | 17 +++- deno.json | 10 ++- tests/smoke/.gitignore | 3 + tests/smoke/commonjs/.env.example | 1 + tests/smoke/commonjs/commonjs.js | 139 ++++++++++++++++++++++++++++++ tests/smoke/commonjs/package.json | 9 ++ tests/smoke/esm/.env.example | 1 + tests/smoke/esm/esm.js | 130 ++++++++++++++++++++++++++++ tests/smoke/esm/package.json | 10 +++ 10 files changed, 396 insertions(+), 5 deletions(-) create mode 100644 tests/smoke/.gitignore create mode 100644 tests/smoke/commonjs/.env.example create mode 100644 tests/smoke/commonjs/commonjs.js create mode 100644 tests/smoke/commonjs/package.json create mode 100644 tests/smoke/esm/.env.example create mode 100644 tests/smoke/esm/esm.js create mode 100644 tests/smoke/esm/package.json diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 880d6f8..8e8ce83 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -4,6 +4,7 @@ on: push jobs: build: + name: "Deno tests and build npm files" runs-on: ubuntu-22.04 steps: - name: Checkout repo @@ -28,8 +29,84 @@ jobs: - name: Setup Node uses: actions/setup-node@v3 with: - node-version: '18.x' + node-version: '18.x' # Build files using a fixed node version registry-url: 'https://registry.npmjs.org' - - name: Test building of npm files + - name: Build npm files run: deno task npm + + - name: Zip build files + run: zip npm.zip ./npm -r + + - name: Upload build files for smoke tests + uses: actions/upload-artifact@v3 + with: + name: npm + path: npm.zip + retention-days: 1 + + smoke-tests-commonjs: + name: "Smoke tests (CommonJS)" + needs: build + runs-on: ubuntu-22.04 + strategy: + matrix: + node-version: [7.x, 8.x, 9.x, 10.x, 11.x, 12.x, 13.x, 14.x, 15.x, 16.x, 17.x, 18.x, 19.x] + steps: + - name: Checkout repo + uses: actions/checkout@v3 + + - name: Setup Node + uses: actions/setup-node@v3 + with: + node-version: ${{ matrix.node-version }} + registry-url: 'https://registry.npmjs.org' + + - name: Download build files + uses: actions/download-artifact@v3 + with: + name: npm + + - name: Unzip build files + run: unzip npm.zip + + - name: Run smoke tests + env: + SERPAPI_TEST_KEY: ${{ secrets.SERPAPI_TEST_KEY }} + run: | + cd tests/smoke/commonjs + npm i + npm test + + smoke-tests-esm: + name: "Smoke tests (ESM)" + needs: build + runs-on: ubuntu-22.04 + strategy: + matrix: + node-version: [14.x, 15.x, 16.x, 17.x, 18.x, 19.x] + steps: + - name: Checkout repo + uses: actions/checkout@v3 + + - name: Setup Node + uses: actions/setup-node@v3 + with: + node-version: ${{ matrix.node-version }} + registry-url: 'https://registry.npmjs.org' + + - name: Download build files + uses: actions/download-artifact@v3 + with: + name: npm + + - name: Unzip build files + run: unzip npm.zip + + - name: Run smoke tests + env: + SERPAPI_TEST_KEY: ${{ secrets.SERPAPI_TEST_KEY }} + run: | + cd tests/smoke/esm + npm i + npm test diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index bb18f40..9fb98dd 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -26,7 +26,8 @@ If you use VSCode, use the following settings (`.vscode/settings.json`): "mod.ts", "version.ts", "src", - "tests", + "tests/*.ts", + "tests/engines/", "scripts", "examples/deno" ], @@ -115,6 +116,20 @@ To run [examples](./examples/) on your local source files, follow these steps. } ``` +## Run smoke tests + +These test key functionality on different Node.js versions. They are ran on +GitHub Actions, see the [build workflow](.github/workflows/build.yml) for more +details. + +To run these locally, follow these steps. + +1. Run `deno task npm` to build the files. +2. Change directory to either the `commonjs` or `esm` folder. +3. Setup the intended Node.js version. For example, if you're using `nvm`, you + can run `nvm use 14` to run Node.js 14 for the current shell. +4. Run `npm i`, then `npm test`. + ## Update documentation - Every exposed function must have associated JSDoc comments. diff --git a/deno.json b/deno.json index e5c46d3..6e8fb32 100644 --- a/deno.json +++ b/deno.json @@ -9,12 +9,18 @@ }, "fmt": { "files": { - "exclude": ["npm/", "examples/node"] + "exclude": ["npm/", "examples/node", "tests/smoke/"] } }, "lint": { "files": { - "exclude": ["npm/", "examples/node"] + "exclude": ["npm/", "examples/node", "tests/smoke/"] + } + }, + "test": { + "files": { + "include": ["tests/"], + "exclude": ["tests/smoke/"] } }, "compilerOptions": { diff --git a/tests/smoke/.gitignore b/tests/smoke/.gitignore new file mode 100644 index 0000000..217965e --- /dev/null +++ b/tests/smoke/.gitignore @@ -0,0 +1,3 @@ +# Smoke tests run on different node versions which generate different +# package-lock.json files. +package-lock.json \ No newline at end of file diff --git a/tests/smoke/commonjs/.env.example b/tests/smoke/commonjs/.env.example new file mode 100644 index 0000000..6debcc7 --- /dev/null +++ b/tests/smoke/commonjs/.env.example @@ -0,0 +1 @@ +SERPAPI_TEST_KEY=YOUR_API_KEY \ No newline at end of file diff --git a/tests/smoke/commonjs/commonjs.js b/tests/smoke/commonjs/commonjs.js new file mode 100644 index 0000000..9cf60b1 --- /dev/null +++ b/tests/smoke/commonjs/commonjs.js @@ -0,0 +1,139 @@ +/** + * Smoke test works for Node.js 7 and newer. + */ + +const Dotenv = require("dotenv"); +const { config, getJson, getHtml, getJsonBySearchId, getHtmlBySearchId, getAccount, getLocations } = require("serpapi"); + +Dotenv.config(); +const apiKey = process.env.SERPAPI_TEST_KEY; + +const run = async () => { + console.log("running", process.versions.node); + + const params = { + q: "Coffee", + api_key: apiKey, + }; + + let searchId; + + { + console.log("getJson async await"); + const page1 = await getJson("google", params); + searchId = page1["search_metadata"]["id"]; + if (!page1["organic_results"]) throw new Error("No organic results"); + if (page1.next) { + const page2 = await page1.next(); + if (!page2["organic_results"]) throw new Error("No organic results"); + } + } + + { + console.log("getJson callback"); + getJson("google", params, (page1) => { + if (!page1["organic_results"]) throw new Error("No organic results"); + if (page1.next) { + page1.next((page2) => { + if (!page2["organic_results"]) throw new Error("No organic results"); + }); + } + }); + } + + { + console.log("getJson using global config"); + config.api_key = apiKey; + const page1 = await getJson("google", { q: "Coffee" }); + if (!page1["organic_results"]) throw new Error("No organic results"); + if (page1.next) { + const page2 = await page1.next(); + if (!page2["organic_results"]) throw new Error("No organic results"); + } + } + + { + console.log("getJson pagination loop (async/await)"); + const links = []; + let page = await getJson("google", params); + while (page) { + links.push(...page.organic_results.map(r => r.link)); + if (links.length >= 30) break; + page = page.next ? await page.next(): undefined; + } + if (links.length < 30) throw new Error("Incorrect number of links"); + } + + { + console.log("getJson pagination loop (callback)"); + const links = []; + getJson("google", params, (page) => { + links.push(...page.organic_results.map(r => r.link)); + if (links.length < 30 && page.next) { + page.next(); + } else { + if (links.length < 30) throw new Error("Incorrect number of links"); + } + }); + } + + { + console.log("getHtml"); + const html = await getHtml("google", params); + if (html.length < 1000) throw new Error("Incorrect HTML"); + + getHtml("google", params, html => { + if (html.length < 1000) throw new Error("Incorrect HTML"); + }); + } + + { + console.log("getJsonBySearchId"); + config.api_key = apiKey; + const json = await getJsonBySearchId(searchId); + if (!json["organic_results"]) throw new Error("No organic results"); + + getJsonBySearchId(searchId, {}, (json) => { + if (!json["organic_results"]) throw new Error("No organic results"); + }); + } + + { + console.log("getHtmlBySearchId"); + config.api_key = apiKey; + const html = await getHtmlBySearchId(searchId); + if (html.length < 1000) throw new Error("Incorrect HTML"); + + getHtmlBySearchId(searchId, {}, (html) => { + if (html.length < 1000) throw new Error("Incorrect HTML"); + }); + } + + { + console.log("getAccount"); + config.api_key = apiKey; + const info = await getAccount(); + if (!info["account_email"]) throw new Error("Incorrect account info"); + + getAccount({}, (info) => { + if (!info["account_email"]) throw new Error("Incorrect account info"); + }); + } + + { + console.log("getLocations"); + const locations = await getLocations({ limit: 3 }); + if (locations.length !== 3) throw new Error("Incorrect locations length"); + + getLocations({ limit: 3 }, (locations) => { + if (locations.length !== 3) throw new Error("Incorrect locations length"); + }); + } + + console.log("success", process.versions.node); +}; + +run().catch((e) => { + console.error(e); + process.exitCode = 1; +}); \ No newline at end of file diff --git a/tests/smoke/commonjs/package.json b/tests/smoke/commonjs/package.json new file mode 100644 index 0000000..1d5cecb --- /dev/null +++ b/tests/smoke/commonjs/package.json @@ -0,0 +1,9 @@ +{ + "dependencies": { + "dotenv": "*", + "serpapi": "../../../npm" + }, + "scripts": { + "test": "node commonjs.js" + } +} diff --git a/tests/smoke/esm/.env.example b/tests/smoke/esm/.env.example new file mode 100644 index 0000000..6debcc7 --- /dev/null +++ b/tests/smoke/esm/.env.example @@ -0,0 +1 @@ +SERPAPI_TEST_KEY=YOUR_API_KEY \ No newline at end of file diff --git a/tests/smoke/esm/esm.js b/tests/smoke/esm/esm.js new file mode 100644 index 0000000..914cfc2 --- /dev/null +++ b/tests/smoke/esm/esm.js @@ -0,0 +1,130 @@ +/** + * Smoke test works for Node.js 14 and newer. + * - Uses ESM imports which is supported from Node.js 13.2.0. + * https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Statements/import#browser_compatibility + * - Uses top-level await which is supported from Node.js 14.8.0. + * https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Operators/await#browser_compatibility + */ + +import Dotenv from "dotenv"; +import { config, getJson, getHtml, getJsonBySearchId, getHtmlBySearchId, getAccount, getLocations } from "serpapi"; + +Dotenv.config(); +const apiKey = process.env.SERPAPI_TEST_KEY; + +console.log("running", process.versions.node); + +const params = { + q: "Coffee", + api_key: apiKey, +}; + +let searchId; + +{ + console.log("getJson async await") + const page1 = await getJson("google", params); + searchId = page1["search_metadata"]["id"]; + if (!page1["organic_results"]) throw new Error("No organic results"); + const page2 = await page1.next?.(); + if (!page2["organic_results"]) throw new Error("No organic results"); +} + +{ + console.log("getJson callback") + getJson("google", params, (page1) => { + if (!page1["organic_results"]) throw new Error("No organic results"); + page1.next?.((page2) => { + if (!page2["organic_results"]) throw new Error("No organic results"); + }); + }); +} + +{ + console.log("getJson using global config") + config.api_key = apiKey; + const page1 = await getJson("google", { q: "Coffee" }); + if (!page1["organic_results"]) throw new Error("No organic results"); + const page2 = await page1.next?.(); + if (!page2["organic_results"]) throw new Error("No organic results"); +} + +{ + console.log("getJson pagination loop (async/await)") + const links = []; + let page = await getJson("google", params); + while (page) { + links.push(...page.organic_results.map(r => r.link)); + if (links.length >= 30) break; + page = await page.next?.(); + } + if (links.length < 30) throw new Error("Incorrect number of links"); +} + +{ + console.log("getJson pagination loop (callback)") + const links = []; + getJson("google", params, (page) => { + links.push(...page.organic_results.map(r => r.link)); + if (links.length < 30 && page.next) { + page.next(); + } else { + if (links.length < 30) throw new Error("Incorrect number of links"); + } + }); +} + +{ + console.log("getHtml") + const html = await getHtml("google", params); + if (html.length < 1000) throw new Error("Incorrect HTML"); + + getHtml("google", params, html => { + if (html.length < 1000) throw new Error("Incorrect HTML"); + }); +} + +{ + console.log("getJsonBySearchId") + config.api_key = apiKey; + const json = await getJsonBySearchId(searchId); + if (!json["organic_results"]) throw new Error("No organic results"); + + getJsonBySearchId(searchId, {}, (json) => { + if (!json["organic_results"]) throw new Error("No organic results"); + }); +} + +{ + console.log("getHtmlBySearchId") + config.api_key = apiKey; + const html = await getHtmlBySearchId(searchId); + if (html.length < 1000) throw new Error("Incorrect HTML"); + + getHtmlBySearchId(searchId, {}, (html) => { + if (html.length < 1000) throw new Error("Incorrect HTML"); + }); +} + +{ + console.log("getAccount") + config.api_key = apiKey; + const info = await getAccount(); + if (!info["account_email"]) throw new Error("Incorrect account info"); + + getAccount({}, (info) => { + if (!info["account_email"]) throw new Error("Incorrect account info"); + }); +} + +{ + console.log("getLocations") + const locations = await getLocations({ limit: 3 }); + if (locations.length !== 3) throw new Error("Incorrect locations length"); + + getLocations({ limit: 3 }, (locations) => { + if (locations.length !== 3) throw new Error("Incorrect locations length"); + }); +} + +console.log("success", process.versions.node); diff --git a/tests/smoke/esm/package.json b/tests/smoke/esm/package.json new file mode 100644 index 0000000..f80c76b --- /dev/null +++ b/tests/smoke/esm/package.json @@ -0,0 +1,10 @@ +{ + "type": "module", + "dependencies": { + "dotenv": "*", + "serpapi": "../../../npm" + }, + "scripts": { + "test": "node esm.js" + } +} From 43fc75926387edc5d10659785ece365ea31cc905 Mon Sep 17 00:00:00 2001 From: Sebastian Quek Date: Tue, 7 Mar 2023 13:55:49 +0800 Subject: [PATCH 16/61] Temporarily disable home depot tests --- tests/engines/home_depot_test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/engines/home_depot_test.ts b/tests/engines/home_depot_test.ts index 15665d3..918e0a1 100644 --- a/tests/engines/home_depot_test.ts +++ b/tests/engines/home_depot_test.ts @@ -33,6 +33,7 @@ const BASE_URL = Deno.env.get("ENV_TYPE") === "local" describe("home_depot", { sanitizeOps: false, // TODO(seb): look into how we can avoid setting these to false sanitizeResources: false, + ignore: true, }, () => { let urlStub: Stub; const engine = "home_depot"; From 226458364b8bdf7bde9e62433d1e9772c220dead Mon Sep 17 00:00:00 2001 From: Sebastian Quek Date: Tue, 7 Mar 2023 14:56:54 +0800 Subject: [PATCH 17/61] Use polyfills only if they don't exist in the runtime --- deno.json | 1 + src/serpapi.ts | 2 +- src/utils.ts | 107 +++++++++++++++++++++++++++----------------- tests/utils_test.ts | 71 +++++++++++++---------------- 4 files changed, 98 insertions(+), 83 deletions(-) diff --git a/deno.json b/deno.json index 6e8fb32..cd7448c 100644 --- a/deno.json +++ b/deno.json @@ -26,6 +26,7 @@ "compilerOptions": { "lib": [ "dom", + "dom.iterable", "deno.ns" ] } diff --git a/src/serpapi.ts b/src/serpapi.ts index 242019d..6a9b8a9 100644 --- a/src/serpapi.ts +++ b/src/serpapi.ts @@ -91,7 +91,7 @@ export async function getJson< timeout, ); const json = await response.json() as BaseResponse; - const nextParametersFromResponse = extractNextParameters(json); + const nextParametersFromResponse = await extractNextParameters(json); if ( // https://github.com/serpapi/public-roadmap/issues/562 // https://github.com/serpapi/public-roadmap/issues/563 diff --git a/src/utils.ts b/src/utils.ts index 82a692b..537879a 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -1,8 +1,5 @@ import type { EngineName, EngineParameters } from "./types.ts"; import { version } from "../version.ts"; -import fetch from "npm:cross-fetch@3.1.4"; -import core from "npm:core-js-pure@3.28.0"; -const { globalThis, URL, URLSearchParams } = core; type UrlParameters = Record< string, @@ -11,26 +8,48 @@ type UrlParameters = Record< /** * This `_internals` object is needed to support stubbing/spying of - * fetch, execute and getBaseUrl. + * certain functions in this file. * https://deno.land/manual@v1.28.3/basics/testing/mocking * - * If fetch is stubbed via `globalThis`, the test phase of the npm build fails. - * ```ts - * const fetchStub = stub(globalThis, "fetch", resolvesNext([new Response("data")])); - * ``` - * - * [`dnt`](https://github.com/denoland/dnt) shims `fetch` by relying on the - * `undici` package. It replaces all references to `fetch` with `dntShim.fetch`. - * As a side effect, stubbing `globalThis.fetch` becomes incorrect; we want to - * stub `dntShim.fetch` instead. - * - * As a workaround, the `_internals` object serves as an indirection and we - * stub the `fetch` key of this object instead. + * It's also useful to encapsulate functions that are polyfilled. */ export const _internals = { - fetch: fetch, + get fetch() { + return (async () => { + // Use runtime's `fetch` if it exists, otherwise fallback to `cross-fetch`. + return typeof fetch === "function" + ? Promise.resolve(fetch) + : (await import("npm:cross-fetch@3.1.4")).default; + })(); + }, execute: execute, getBaseUrl: getBaseUrl, + get globalThis() { + return (async () => { + // Use runtime's `globalThis` if it exists, otherwise fallback to `core-js`'s implementation. + // dnt-shim-ignore + const gt = typeof globalThis !== "undefined" ? globalThis : undefined; + return gt !== undefined + ? Promise.resolve(gt) + : (await import("npm:core-js-pure@3.28.0")).default.globalThis; + })(); + }, + get URL(): Promise { + return (async () => { + // Use runtime's `URL` if it exists, otherwise fallback to `core-js`'s implementation. + return typeof URL !== "undefined" + ? Promise.resolve(URL) + : (await import("npm:core-js-pure@3.28.0")).default.URL; + })(); + }, + get URLSearchParams(): Promise { + return (async () => { + // Use runtime's `URLSearchParams` if it exists, otherwise fallback to `core-js`'s implementation. + return typeof URLSearchParams !== "undefined" + ? Promise.resolve(URLSearchParams) + : (await import("npm:core-js-pure@3.28.0")).default.URLSearchParams; + })(); + }, }; /** Facilitates stubbing in tests, e.g. localhost as the base url */ @@ -46,14 +65,17 @@ type NextParameters = { > ]: string; }; -export function extractNextParameters(json: { - serpapi_pagination?: { next: string }; - pagination?: { next: string }; -}) { +export async function extractNextParameters( + json: { + serpapi_pagination?: { next: string }; + pagination?: { next: string }; + }, +) { const nextUrlString = json["serpapi_pagination"]?.["next"] || json["pagination"]?.["next"]; if (nextUrlString) { + const URL = await _internals.URL; const nextUrl = new URL(nextUrlString); const nextParameters: Record = {}; for (const [k, v] of nextUrl.searchParams.entries()) { @@ -78,22 +100,20 @@ export function haveParametersChanged( ); } -function getSource() { +export async function getSource() { const moduleSource = `serpapi@${version}`; try { + const gt = await _internals.globalThis; + // Check if running in Node.js - // dnt-shim-ignore - // deno-lint-ignore no-explicit-any - const nodeVersion = (globalThis as any).process?.versions?.node; + const nodeVersion = gt.process?.versions?.node; if (nodeVersion) { return `nodejs@${nodeVersion},${moduleSource}`; } // Assumes running in Deno instead. https://deno.land/api?s=Deno.version // Deno.version is not shimmed since it's not used when ran in a Node env. - // dnt-shim-ignore - // deno-lint-ignore no-explicit-any - const denoVersion = (globalThis as any).Deno?.version?.deno; + const denoVersion = gt.Deno?.version?.deno; if (denoVersion) { return `deno@${denoVersion},${moduleSource}`; } @@ -105,37 +125,40 @@ function getSource() { } } -export function buildUrl

( +export async function buildUrl

( path: string, parameters: P, -): string { +): Promise { const nonUndefinedParams: [string, string][] = Object.entries(parameters) .filter(([_, value]) => value !== undefined) .map(([key, value]) => [key, `${value}`]); + const URLSearchParams = await _internals.URLSearchParams; const searchParams = new URLSearchParams(nonUndefinedParams); return `${_internals.getBaseUrl()}${path}?${searchParams}`; } -export function execute

( +export async function execute

( path: string, parameters: P, timeout: number, ): Promise { - const url = buildUrl(path, { + const url = await buildUrl(path, { ...parameters, - source: getSource(), + source: await getSource(), }); // https://github.com/github/fetch/issues/175#issuecomment-216791333 return new Promise((resolve, reject) => { const timer = setTimeout(() => reject(new Error("Timeout")), timeout); - _internals.fetch(url) - .then((res) => { - clearTimeout(timer); - resolve(res); - }) - .catch((err) => { - clearTimeout(timer); - reject(err); - }); + _internals.fetch.then((fetch) => + fetch(url) + .then((res) => { + clearTimeout(timer); + resolve(res); + }) + .catch((err) => { + clearTimeout(timer); + reject(err); + }) + ); }); } diff --git a/tests/utils_test.ts b/tests/utils_test.ts index f12c2b3..ff689d2 100644 --- a/tests/utils_test.ts +++ b/tests/utils_test.ts @@ -5,12 +5,7 @@ import { describe, it, } from "https://deno.land/std@0.170.0/testing/bdd.ts"; -import { - assertSpyCalls, - resolvesNext, - Stub, - stub, -} from "https://deno.land/std@0.170.0/testing/mock.ts"; +import { Stub, stub } from "https://deno.land/std@0.170.0/testing/mock.ts"; import { assertEquals, assertMatch, @@ -21,9 +16,9 @@ import { buildUrl, execute, extractNextParameters, + getSource, haveParametersChanged, } from "../src/utils.ts"; -import { Response } from "npm:cross-fetch@3.1.4"; loadSync({ export: true }); const BASE_URL = Deno.env.get("ENV_TYPE") === "local" @@ -31,9 +26,9 @@ const BASE_URL = Deno.env.get("ENV_TYPE") === "local" : "https://serpapi.com"; describe("extractNextParameters", () => { - it("with serpapi_pagination property", () => { + it("with serpapi_pagination property", async () => { assertEquals( - extractNextParameters<"google">({ + await extractNextParameters<"google">({ serpapi_pagination: { next: "https://serpapi.com/search.json?device=desktop&engine=google&gl=us&google_domain=google.com&hl=en&location=Austin%2C+Texas%2C+United+States&q=coffee&start=10", @@ -51,9 +46,9 @@ describe("extractNextParameters", () => { ); }); - it("with pagination property", () => { + it("with pagination property", async () => { assertEquals( - extractNextParameters<"google_scholar_profiles">( + await extractNextParameters<"google_scholar_profiles">( { pagination: { next: @@ -170,6 +165,15 @@ describe("haveParametersChanged", () => { }); }); +describe("getSource", () => { + it("use runtime version", async () => { + assertMatch( + await getSource(), + /(nodejs|deno)@\d+\.\d+\.\d+,serpapi@\d+\.\d+\.\d+$/, + ); + }); +}); + describe("buildUrl", () => { let urlStub: Stub; @@ -181,24 +185,32 @@ describe("buildUrl", () => { urlStub.restore(); }); - it("with blank path and empty parameters", () => { - assertEquals(buildUrl("", {}), `${BASE_URL}?`); + it("with blank path and empty parameters", async () => { + assertEquals(await buildUrl("", {}), `${BASE_URL}?`); }); - it("with path and empty parameters", () => { - assertEquals(buildUrl("/", {}), `${BASE_URL}/?`); + it("with path and empty parameters", async () => { + assertEquals(await buildUrl("/", {}), `${BASE_URL}/?`); }); - it("with path and parameters", () => { + it("with path and parameters", async () => { assertEquals( - buildUrl("/search", { q: "coffee", gl: "us" }), + await buildUrl("/search", { q: "coffee", gl: "us" }), `${BASE_URL}/search?q=coffee&gl=us`, ); }); - it("with undefined parameters", () => { + it("with source", async () => { + const url = await buildUrl("/search", { source: await getSource() }); + assertMatch( + url, + /source=(nodejs|deno)%40\d+\.\d+\.\d+%2Cserpapi%40\d+\.\d+\.\d+$/, + ); + }); + + it("with undefined parameters", async () => { assertEquals( - buildUrl("/search", { q: "coffee", gl: undefined, hl: null }), + await buildUrl("/search", { q: "coffee", gl: undefined, hl: null }), `${BASE_URL}/search?q=coffee&hl=null`, ); }); @@ -218,27 +230,6 @@ describe("execute", { urlStub.restore(); }); - it("with path and parameters calls fetch with source appended", async () => { - const fetchStub = stub( - _internals, - "fetch", - resolvesNext([new Response("data")]), - ); - try { - await execute("/search", { q: "coffee", gl: "us" }, 4000); - } finally { - fetchStub.restore(); - } - - assertSpyCalls(fetchStub, 1); - const url = fetchStub.calls[0].args[0] as string; - // e.g. deno@1.28.2,serpapi@1.0.0 - assertMatch( - url, - /source=(nodejs|deno)%40\d+\.\d+\.\d+%2Cserpapi%40\d+\.\d+\.\d+$/, - ); - }); - it("with short timeout", () => { assertRejects(async () => await execute("/search", { q: "coffee", gl: "us" }, 1) From c2db13f7254033c4d2a9f5a5006ba2af72412278 Mon Sep 17 00:00:00 2001 From: Sebastian Quek Date: Tue, 7 Mar 2023 17:10:16 +0800 Subject: [PATCH 18/61] Move smoke tests + format --- .github/workflows/build.yml | 4 +- deno.json | 7 ++-- {tests/smoke => smoke_tests}/.gitignore | 0 .../commonjs/.env.example | 0 .../commonjs/commonjs.js | 30 +++++++++----- .../commonjs/package.json | 2 +- {tests/smoke => smoke_tests}/esm/.env.example | 0 {tests/smoke => smoke_tests}/esm/esm.js | 40 +++++++++++-------- {tests/smoke => smoke_tests}/esm/package.json | 2 +- 9 files changed, 50 insertions(+), 35 deletions(-) rename {tests/smoke => smoke_tests}/.gitignore (100%) rename {tests/smoke => smoke_tests}/commonjs/.env.example (100%) rename {tests/smoke => smoke_tests}/commonjs/commonjs.js (90%) rename {tests/smoke => smoke_tests}/commonjs/package.json (76%) rename {tests/smoke => smoke_tests}/esm/.env.example (100%) rename {tests/smoke => smoke_tests}/esm/esm.js (82%) rename {tests/smoke => smoke_tests}/esm/package.json (79%) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 8e8ce83..4ca5687 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -74,7 +74,7 @@ jobs: env: SERPAPI_TEST_KEY: ${{ secrets.SERPAPI_TEST_KEY }} run: | - cd tests/smoke/commonjs + cd smoke_tests/commonjs npm i npm test @@ -107,6 +107,6 @@ jobs: env: SERPAPI_TEST_KEY: ${{ secrets.SERPAPI_TEST_KEY }} run: | - cd tests/smoke/esm + cd smoke_tests/esm npm i npm test diff --git a/deno.json b/deno.json index cd7448c..93afa4a 100644 --- a/deno.json +++ b/deno.json @@ -9,18 +9,17 @@ }, "fmt": { "files": { - "exclude": ["npm/", "examples/node", "tests/smoke/"] + "exclude": ["npm/", "examples/node", "smoke_tests/"] } }, "lint": { "files": { - "exclude": ["npm/", "examples/node", "tests/smoke/"] + "exclude": ["npm/", "examples/node", "smoke_tests/"] } }, "test": { "files": { - "include": ["tests/"], - "exclude": ["tests/smoke/"] + "include": ["tests/"] } }, "compilerOptions": { diff --git a/tests/smoke/.gitignore b/smoke_tests/.gitignore similarity index 100% rename from tests/smoke/.gitignore rename to smoke_tests/.gitignore diff --git a/tests/smoke/commonjs/.env.example b/smoke_tests/commonjs/.env.example similarity index 100% rename from tests/smoke/commonjs/.env.example rename to smoke_tests/commonjs/.env.example diff --git a/tests/smoke/commonjs/commonjs.js b/smoke_tests/commonjs/commonjs.js similarity index 90% rename from tests/smoke/commonjs/commonjs.js rename to smoke_tests/commonjs/commonjs.js index 9cf60b1..9c5fef0 100644 --- a/tests/smoke/commonjs/commonjs.js +++ b/smoke_tests/commonjs/commonjs.js @@ -3,21 +3,29 @@ */ const Dotenv = require("dotenv"); -const { config, getJson, getHtml, getJsonBySearchId, getHtmlBySearchId, getAccount, getLocations } = require("serpapi"); +const { + config, + getJson, + getHtml, + getJsonBySearchId, + getHtmlBySearchId, + getAccount, + getLocations, +} = require("serpapi"); Dotenv.config(); const apiKey = process.env.SERPAPI_TEST_KEY; const run = async () => { console.log("running", process.versions.node); - + const params = { q: "Coffee", api_key: apiKey, }; let searchId; - + { console.log("getJson async await"); const page1 = await getJson("google", params); @@ -57,18 +65,18 @@ const run = async () => { const links = []; let page = await getJson("google", params); while (page) { - links.push(...page.organic_results.map(r => r.link)); + links.push(...page.organic_results.map((r) => r.link)); if (links.length >= 30) break; - page = page.next ? await page.next(): undefined; + page = page.next ? await page.next() : undefined; } if (links.length < 30) throw new Error("Incorrect number of links"); } - + { console.log("getJson pagination loop (callback)"); const links = []; getJson("google", params, (page) => { - links.push(...page.organic_results.map(r => r.link)); + links.push(...page.organic_results.map((r) => r.link)); if (links.length < 30 && page.next) { page.next(); } else { @@ -81,8 +89,8 @@ const run = async () => { console.log("getHtml"); const html = await getHtml("google", params); if (html.length < 1000) throw new Error("Incorrect HTML"); - - getHtml("google", params, html => { + + getHtml("google", params, (html) => { if (html.length < 1000) throw new Error("Incorrect HTML"); }); } @@ -124,7 +132,7 @@ const run = async () => { console.log("getLocations"); const locations = await getLocations({ limit: 3 }); if (locations.length !== 3) throw new Error("Incorrect locations length"); - + getLocations({ limit: 3 }, (locations) => { if (locations.length !== 3) throw new Error("Incorrect locations length"); }); @@ -136,4 +144,4 @@ const run = async () => { run().catch((e) => { console.error(e); process.exitCode = 1; -}); \ No newline at end of file +}); diff --git a/tests/smoke/commonjs/package.json b/smoke_tests/commonjs/package.json similarity index 76% rename from tests/smoke/commonjs/package.json rename to smoke_tests/commonjs/package.json index 1d5cecb..f229cf1 100644 --- a/tests/smoke/commonjs/package.json +++ b/smoke_tests/commonjs/package.json @@ -1,7 +1,7 @@ { "dependencies": { "dotenv": "*", - "serpapi": "../../../npm" + "serpapi": "../../npm" }, "scripts": { "test": "node commonjs.js" diff --git a/tests/smoke/esm/.env.example b/smoke_tests/esm/.env.example similarity index 100% rename from tests/smoke/esm/.env.example rename to smoke_tests/esm/.env.example diff --git a/tests/smoke/esm/esm.js b/smoke_tests/esm/esm.js similarity index 82% rename from tests/smoke/esm/esm.js rename to smoke_tests/esm/esm.js index 914cfc2..1dd8c69 100644 --- a/tests/smoke/esm/esm.js +++ b/smoke_tests/esm/esm.js @@ -7,7 +7,15 @@ */ import Dotenv from "dotenv"; -import { config, getJson, getHtml, getJsonBySearchId, getHtmlBySearchId, getAccount, getLocations } from "serpapi"; +import { + config, + getAccount, + getHtml, + getHtmlBySearchId, + getJson, + getJsonBySearchId, + getLocations, +} from "serpapi"; Dotenv.config(); const apiKey = process.env.SERPAPI_TEST_KEY; @@ -22,7 +30,7 @@ const params = { let searchId; { - console.log("getJson async await") + console.log("getJson async await"); const page1 = await getJson("google", params); searchId = page1["search_metadata"]["id"]; if (!page1["organic_results"]) throw new Error("No organic results"); @@ -31,7 +39,7 @@ let searchId; } { - console.log("getJson callback") + console.log("getJson callback"); getJson("google", params, (page1) => { if (!page1["organic_results"]) throw new Error("No organic results"); page1.next?.((page2) => { @@ -41,7 +49,7 @@ let searchId; } { - console.log("getJson using global config") + console.log("getJson using global config"); config.api_key = apiKey; const page1 = await getJson("google", { q: "Coffee" }); if (!page1["organic_results"]) throw new Error("No organic results"); @@ -50,11 +58,11 @@ let searchId; } { - console.log("getJson pagination loop (async/await)") + console.log("getJson pagination loop (async/await)"); const links = []; let page = await getJson("google", params); while (page) { - links.push(...page.organic_results.map(r => r.link)); + links.push(...page.organic_results.map((r) => r.link)); if (links.length >= 30) break; page = await page.next?.(); } @@ -62,10 +70,10 @@ let searchId; } { - console.log("getJson pagination loop (callback)") + console.log("getJson pagination loop (callback)"); const links = []; getJson("google", params, (page) => { - links.push(...page.organic_results.map(r => r.link)); + links.push(...page.organic_results.map((r) => r.link)); if (links.length < 30 && page.next) { page.next(); } else { @@ -75,17 +83,17 @@ let searchId; } { - console.log("getHtml") + console.log("getHtml"); const html = await getHtml("google", params); if (html.length < 1000) throw new Error("Incorrect HTML"); - - getHtml("google", params, html => { + + getHtml("google", params, (html) => { if (html.length < 1000) throw new Error("Incorrect HTML"); }); } { - console.log("getJsonBySearchId") + console.log("getJsonBySearchId"); config.api_key = apiKey; const json = await getJsonBySearchId(searchId); if (!json["organic_results"]) throw new Error("No organic results"); @@ -96,7 +104,7 @@ let searchId; } { - console.log("getHtmlBySearchId") + console.log("getHtmlBySearchId"); config.api_key = apiKey; const html = await getHtmlBySearchId(searchId); if (html.length < 1000) throw new Error("Incorrect HTML"); @@ -107,7 +115,7 @@ let searchId; } { - console.log("getAccount") + console.log("getAccount"); config.api_key = apiKey; const info = await getAccount(); if (!info["account_email"]) throw new Error("Incorrect account info"); @@ -118,10 +126,10 @@ let searchId; } { - console.log("getLocations") + console.log("getLocations"); const locations = await getLocations({ limit: 3 }); if (locations.length !== 3) throw new Error("Incorrect locations length"); - + getLocations({ limit: 3 }, (locations) => { if (locations.length !== 3) throw new Error("Incorrect locations length"); }); diff --git a/tests/smoke/esm/package.json b/smoke_tests/esm/package.json similarity index 79% rename from tests/smoke/esm/package.json rename to smoke_tests/esm/package.json index f80c76b..aaae393 100644 --- a/tests/smoke/esm/package.json +++ b/smoke_tests/esm/package.json @@ -2,7 +2,7 @@ "type": "module", "dependencies": { "dotenv": "*", - "serpapi": "../../../npm" + "serpapi": "../../npm" }, "scripts": { "test": "node esm.js" From cfddfc36e7dde4afcff36f090afe25877f22246a Mon Sep 17 00:00:00 2001 From: Sebastian Quek Date: Thu, 16 Mar 2023 18:15:35 +0800 Subject: [PATCH 19/61] Revert "Temporarily disable home depot tests" This reverts commit 43fc75926387edc5d10659785ece365ea31cc905. --- tests/engines/home_depot_test.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/engines/home_depot_test.ts b/tests/engines/home_depot_test.ts index 918e0a1..15665d3 100644 --- a/tests/engines/home_depot_test.ts +++ b/tests/engines/home_depot_test.ts @@ -33,7 +33,6 @@ const BASE_URL = Deno.env.get("ENV_TYPE") === "local" describe("home_depot", { sanitizeOps: false, // TODO(seb): look into how we can avoid setting these to false sanitizeResources: false, - ignore: true, }, () => { let urlStub: Stub; const engine = "home_depot"; From a6c20f7e85f4d70154df1aa9acf75fae77e4edc3 Mon Sep 17 00:00:00 2001 From: Sebastian Quek Date: Mon, 3 Apr 2023 17:57:18 +0800 Subject: [PATCH 20/61] Polymorphic getJson --- CHANGELOG.md | 1 + mod.ts | 6 +++++- src/errors.ts | 7 +++++++ src/serpapi.ts | 36 ++++++++++++++++++++++++++++++++++-- src/types.ts | 3 ++- 5 files changed, 49 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 14cf866..14274de 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ and this project adheres to ### Added - Expose `EngineName`, `EngineParameters` and `AllowArbitraryParams` types. +- Expose `InvalidArgumentTypesError` error. ### Changed diff --git a/mod.ts b/mod.ts index a603485..f6a909a 100644 --- a/mod.ts +++ b/mod.ts @@ -1,7 +1,11 @@ export type { Config } from "./src/config.ts"; export { config } from "./src/config.ts"; -export { InvalidTimeoutError, MissingApiKeyError } from "./src/errors.ts"; +export { + InvalidArgumentTypesError, + InvalidTimeoutError, + MissingApiKeyError, +} from "./src/errors.ts"; export type { AccountApiParameters, diff --git a/src/errors.ts b/src/errors.ts index f225872..63ecf3a 100644 --- a/src/errors.ts +++ b/src/errors.ts @@ -1,3 +1,10 @@ +export class InvalidArgumentTypesError extends Error { + constructor() { + super("Provide the arguments with the correct type"); + Object.setPrototypeOf(this, InvalidArgumentTypesError.prototype); + } +} + export class MissingApiKeyError extends Error { constructor() { super( diff --git a/src/serpapi.ts b/src/serpapi.ts index 1d3765e..7663fbe 100644 --- a/src/serpapi.ts +++ b/src/serpapi.ts @@ -1,3 +1,4 @@ +import { InvalidArgumentTypesError } from "./errors.ts"; import { AccountApiParameters, AccountInformation, @@ -69,13 +70,44 @@ const SEARCH_ARCHIVE_PATH = `/searches`; * } * }); */ -export async function getJson< +export function getJson< + E extends EngineName = EngineName, + P1 extends AllowArbitraryParams> = EngineParameters, + P2 extends AllowArbitraryParams> = + EngineParameters, +>( + ...args: + | [parameters: P1, callback?: (json: BaseResponse) => void] + | [ + engine: string, // intentionally kept as a string to support arbitrary params + parameters: P2, + callback?: (json: BaseResponse) => void, + ] +): Promise> { + if ( + typeof args[0] === "string" && + typeof args[1] === "object" + ) { + const [engine, parameters, callback] = args; + return _getJson({ ...parameters, engine: engine as E }, callback); + } else if ( + typeof args[0] === "object" && + (typeof args[1] === "undefined" || typeof args[1] === "function") + ) { + const [parameters, callback] = args; + return _getJson(parameters, callback); + } else { + throw new InvalidArgumentTypesError(); + } +} + +async function _getJson< E extends EngineName = EngineName, P extends AllowArbitraryParams> = EngineParameters, >( parameters: P, callback?: (json: BaseResponse) => void, -) { +): Promise> { const key = validateApiKey(parameters.api_key, true); const timeout = validateTimeout(parameters.timeout); const response = await _internals.execute( diff --git a/src/types.ts b/src/types.ts index 77a2a99..7407b36 100644 --- a/src/types.ts +++ b/src/types.ts @@ -53,9 +53,10 @@ type AnyEngineName = string & {}; export type EngineName = (keyof EngineMap) | AnyEngineName; export type EngineParameters< E extends EngineName = EngineName, + W = true, // flag for whether `engine` is a required param > = { [K in E]: - & { engine: K } + & (W extends true ? { engine: K } : { engine?: K }) & (K extends keyof EngineMap ? EngineMap[K]["parameters"] : BaseParameters); }[E]; From 47d65b42c061b025382c93f117129b327ac39730 Mon Sep 17 00:00:00 2001 From: Sebastian Quek Date: Mon, 3 Apr 2023 18:01:20 +0800 Subject: [PATCH 21/61] Polymorphic getHtml --- src/serpapi.ts | 35 +++++++++++++++++++++++++++++++++-- 1 file changed, 33 insertions(+), 2 deletions(-) diff --git a/src/serpapi.ts b/src/serpapi.ts index 7663fbe..171c167 100644 --- a/src/serpapi.ts +++ b/src/serpapi.ts @@ -154,13 +154,44 @@ async function _getJson< * // callback * getHtml({ engine: "google", api_key: API_KEY, q: "coffee" }, console.log); */ -export async function getHtml< +export function getHtml< + E extends EngineName = EngineName, + P1 extends AllowArbitraryParams> = EngineParameters, + P2 extends AllowArbitraryParams> = + EngineParameters, +>( + ...args: + | [parameters: P1, callback?: (html: string) => void] + | [ + engine: string, // intentionally kept as a string to support arbitrary params + parameters: P2, + callback?: (html: string) => void, + ] +): Promise { + if ( + typeof args[0] === "string" && + typeof args[1] === "object" + ) { + const [engine, parameters, callback] = args; + return _getHtml({ ...parameters, engine: engine as E }, callback); + } else if ( + typeof args[0] === "object" && + (typeof args[1] === "undefined" || typeof args[1] === "function") + ) { + const [parameters, callback] = args; + return _getHtml(parameters, callback); + } else { + throw new InvalidArgumentTypesError(); + } +} + +async function _getHtml< E extends EngineName = EngineName, P extends AllowArbitraryParams> = EngineParameters, >( parameters: P, callback?: (html: string) => void, -) { +): Promise { const key = validateApiKey(parameters.api_key, true); const timeout = validateTimeout(parameters.timeout); const response = await _internals.execute( From 82bd536fd43489f0adab85c3536bdd3a3b90dfdf Mon Sep 17 00:00:00 2001 From: Sebastian Quek Date: Mon, 3 Apr 2023 18:25:23 +0800 Subject: [PATCH 22/61] Fix TS2590 error --- src/serpapi.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/serpapi.ts b/src/serpapi.ts index 171c167..b30585b 100644 --- a/src/serpapi.ts +++ b/src/serpapi.ts @@ -89,7 +89,8 @@ export function getJson< typeof args[1] === "object" ) { const [engine, parameters, callback] = args; - return _getJson({ ...parameters, engine: engine as E }, callback); + const newParameters = { engine, ...parameters } as P1; + return _getJson(newParameters, callback); } else if ( typeof args[0] === "object" && (typeof args[1] === "undefined" || typeof args[1] === "function") @@ -173,7 +174,8 @@ export function getHtml< typeof args[1] === "object" ) { const [engine, parameters, callback] = args; - return _getHtml({ ...parameters, engine: engine as E }, callback); + const newParameters = { engine, ...parameters } as P1; + return _getHtml(newParameters, callback); } else if ( typeof args[0] === "object" && (typeof args[1] === "undefined" || typeof args[1] === "function") From c61499c402c5cf44d808ee343c9bfc9b3057b114 Mon Sep 17 00:00:00 2001 From: Sebastian Quek Date: Tue, 4 Apr 2023 16:48:33 +0800 Subject: [PATCH 23/61] Add tests for when engine is the first param --- tests/engines/google_test.ts | 47 ++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/tests/engines/google_test.ts b/tests/engines/google_test.ts index 7978ed5..0808605 100644 --- a/tests/engines/google_test.ts +++ b/tests/engines/google_test.ts @@ -134,6 +134,35 @@ describe("google", { ]); }); + it("getJson with engine as first parameter (async/await)", async () => { + const response = await getJson(engine, { + api_key: null, // null to support the "coffee" unmetered query + q: "coffee", + }); + assertArrayIncludes(Object.keys(response).sort(), [ + "organic_results", + "pagination", + "search_information", + "search_metadata", + "search_parameters", + "serpapi_pagination", + ]); + }); + + it("getJson with engine as first parameter (callback)", async () => { + const response = await new Promise>>( + (res) => getJson(engine, { api_key: null, q: "coffee" }, res), + ); + assertArrayIncludes(Object.keys(response).sort(), [ + "organic_results", + "pagination", + "search_information", + "search_metadata", + "search_parameters", + "serpapi_pagination", + ]); + }); + it("getHtml for an unmetered query (async/await)", async () => { const response = await getHtml({ engine, api_key: null, q: "coffee" }); assertStringIncludes(response, " { + const response = await getHtml(engine, { api_key: null, q: "coffee" }); + assertStringIncludes(response, ""); + assertStringIncludes(response, ""); + }); + + it("getHtml with engine as first parameter (callback)", async () => { + const response = await new Promise>>( + (res) => getHtml(engine, { api_key: null, q: "coffee" }, res), + ); + assertStringIncludes(response, ""); + assertStringIncludes(response, ""); + }); + // get(Json|Html)BySearchId always require a valid API key even for unmetered queries it("get(Json|Html)BySearchId", { ignore: !HAS_API_KEY, From 4687abd95983eda89395cac000676428adc9be79 Mon Sep 17 00:00:00 2001 From: zyc9012 Date: Mon, 10 Apr 2023 12:14:05 +0800 Subject: [PATCH 24/61] Simplify types and fix intelligence --- mod.ts | 1 - src/serpapi.ts | 47 ++++++++++++++++++++++------------------------- src/types.ts | 23 +++++++++++------------ src/utils.ts | 7 ++++--- 4 files changed, 37 insertions(+), 41 deletions(-) diff --git a/mod.ts b/mod.ts index f6a909a..8f153d5 100644 --- a/mod.ts +++ b/mod.ts @@ -13,7 +13,6 @@ export type { AllowArbitraryParams, BaseParameters, BaseResponse, - EngineName, EngineParameters, GetBySearchIdParameters, Location, diff --git a/src/serpapi.ts b/src/serpapi.ts index b30585b..101bfbe 100644 --- a/src/serpapi.ts +++ b/src/serpapi.ts @@ -1,10 +1,9 @@ +import { EngineMap } from "./engines/engine_map.ts"; import { InvalidArgumentTypesError } from "./errors.ts"; import { AccountApiParameters, AccountInformation, - AllowArbitraryParams, BaseResponse, - EngineName, EngineParameters, GetBySearchIdParameters, Locations, @@ -71,16 +70,16 @@ const SEARCH_ARCHIVE_PATH = `/searches`; * }); */ export function getJson< - E extends EngineName = EngineName, - P1 extends AllowArbitraryParams> = EngineParameters, - P2 extends AllowArbitraryParams> = - EngineParameters, + E extends keyof EngineMap, >( ...args: - | [parameters: P1, callback?: (json: BaseResponse) => void] | [ - engine: string, // intentionally kept as a string to support arbitrary params - parameters: P2, + parameters: EngineParameters, + callback?: (json: BaseResponse) => void, + ] + | [ + engine: E | string, + parameters: EngineParameters, callback?: (json: BaseResponse) => void, ] ): Promise> { @@ -89,7 +88,7 @@ export function getJson< typeof args[1] === "object" ) { const [engine, parameters, callback] = args; - const newParameters = { engine, ...parameters } as P1; + const newParameters = { ...parameters, engine } as EngineParameters; return _getJson(newParameters, callback); } else if ( typeof args[0] === "object" && @@ -103,10 +102,9 @@ export function getJson< } async function _getJson< - E extends EngineName = EngineName, - P extends AllowArbitraryParams> = EngineParameters, + E extends keyof EngineMap, >( - parameters: P, + parameters: EngineParameters, callback?: (json: BaseResponse) => void, ): Promise> { const key = validateApiKey(parameters.api_key, true); @@ -156,16 +154,16 @@ async function _getJson< * getHtml({ engine: "google", api_key: API_KEY, q: "coffee" }, console.log); */ export function getHtml< - E extends EngineName = EngineName, - P1 extends AllowArbitraryParams> = EngineParameters, - P2 extends AllowArbitraryParams> = - EngineParameters, + E extends keyof EngineMap, >( ...args: - | [parameters: P1, callback?: (html: string) => void] | [ - engine: string, // intentionally kept as a string to support arbitrary params - parameters: P2, + parameters: EngineParameters, + callback?: (html: string) => void, + ] + | [ + engine: E | string, + parameters: EngineParameters, callback?: (html: string) => void, ] ): Promise { @@ -174,7 +172,7 @@ export function getHtml< typeof args[1] === "object" ) { const [engine, parameters, callback] = args; - const newParameters = { engine, ...parameters } as P1; + const newParameters = { ...parameters, engine } as EngineParameters; return _getHtml(newParameters, callback); } else if ( typeof args[0] === "object" && @@ -188,10 +186,9 @@ export function getHtml< } async function _getHtml< - E extends EngineName = EngineName, - P extends AllowArbitraryParams> = EngineParameters, + E extends keyof EngineMap, >( - parameters: P, + parameters: EngineParameters, callback?: (html: string) => void, ): Promise { const key = validateApiKey(parameters.api_key, true); @@ -233,7 +230,7 @@ async function _getHtml< * getJsonBySearchId(id, { api_key: API_KEY }, console.log); */ export async function getJsonBySearchId< - R extends BaseResponse, + R extends BaseResponse, >( searchId: string, parameters: GetBySearchIdParameters = {}, diff --git a/src/types.ts b/src/types.ts index 7407b36..3c22fb1 100644 --- a/src/types.ts +++ b/src/types.ts @@ -47,20 +47,19 @@ export type BaseParameters = { timeout?: number; }; -// https://github.com/microsoft/TypeScript/issues/29729 -// deno-lint-ignore ban-types -type AnyEngineName = string & {}; -export type EngineName = (keyof EngineMap) | AnyEngineName; export type EngineParameters< - E extends EngineName = EngineName, - W = true, // flag for whether `engine` is a required param -> = { - [K in E]: - & (W extends true ? { engine: K } : { engine?: K }) - & (K extends keyof EngineMap ? EngineMap[K]["parameters"] : BaseParameters); -}[E]; + E extends keyof EngineMap, + EngineRequired = true, +> = + // https://github.com/microsoft/TypeScript/issues/29729 + // deno-lint-ignore ban-types + & (EngineRequired extends true ? { engine: E | (string & {}) } + // deno-lint-ignore ban-types + : { engine?: E | (string & {}) }) + & EngineMap[E]["parameters"] + & Record; -export type BaseResponse = { +export type BaseResponse = { search_metadata: { id: string; status: "Queued" | "Processing" | "Success"; diff --git a/src/utils.ts b/src/utils.ts index 68d0653..796b241 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -1,5 +1,6 @@ -import type { EngineName, EngineParameters } from "./types.ts"; +import type { EngineParameters } from "./types.ts"; import { version } from "../version.ts"; +import { EngineMap } from "./engines/engine_map.ts"; type UrlParameters = Record< string, @@ -35,7 +36,7 @@ function getBaseUrl() { return "https://serpapi.com"; } -type NextParameters = { +type NextParameters = { [ K in keyof Omit< EngineParameters, @@ -43,7 +44,7 @@ type NextParameters = { > ]: string; }; -export function extractNextParameters(json: { +export function extractNextParameters(json: { serpapi_pagination?: { next: string }; pagination?: { next: string }; }) { From fd6d52d9cadb45aa6a21f55b0cf802ed828438a3 Mon Sep 17 00:00:00 2001 From: zyc9012 Date: Mon, 10 Apr 2023 12:18:45 +0800 Subject: [PATCH 25/61] Remove AllowArbitraryParams --- CHANGELOG.md | 2 +- examples/deno/basic_ts/README.md | 1 - examples/deno/basic_ts/example.ts | 9 ++------- examples/deno/pagination_ts/README.md | 1 - examples/deno/pagination_ts/example.ts | 9 ++------- examples/node/basic_ts_esm/example.ts | 4 ++-- examples/node/pagination_ts_esm/example.ts | 4 ++-- mod.ts | 1 - src/types.ts | 5 ----- 9 files changed, 9 insertions(+), 27 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 14274de..a28fb19 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,7 +10,7 @@ and this project adheres to ### Added -- Expose `EngineName`, `EngineParameters` and `AllowArbitraryParams` types. +- Expose `EngineParameters` type. - Expose `InvalidArgumentTypesError` error. ### Changed diff --git a/examples/deno/basic_ts/README.md b/examples/deno/basic_ts/README.md index 78456e3..811641a 100644 --- a/examples/deno/basic_ts/README.md +++ b/examples/deno/basic_ts/README.md @@ -20,7 +20,6 @@ deno run example.ts following: ```ts import { - AllowArbitraryParams, config, getJson, GoogleParameters, diff --git a/examples/deno/basic_ts/example.ts b/examples/deno/basic_ts/example.ts index 5b5c411..df6c117 100644 --- a/examples/deno/basic_ts/example.ts +++ b/examples/deno/basic_ts/example.ts @@ -1,17 +1,12 @@ import { loadSync } from "https://deno.land/std@0.173.0/dotenv/mod.ts"; -import { - AllowArbitraryParams, - config, - EngineParameters, - getJson, -} from "../../../mod.ts"; +import { config, EngineParameters, getJson } from "../../../mod.ts"; const { API_KEY: apiKey } = loadSync(); const params = { engine: "google", q: "Coffee", api_key: apiKey, -} satisfies AllowArbitraryParams>; +} satisfies EngineParameters<"google">; // Show result as JSON (async/await) const response1 = await getJson(params); diff --git a/examples/deno/pagination_ts/README.md b/examples/deno/pagination_ts/README.md index cb46b9b..4402df1 100644 --- a/examples/deno/pagination_ts/README.md +++ b/examples/deno/pagination_ts/README.md @@ -20,7 +20,6 @@ deno run example.ts following: ```ts import { - AllowArbitraryParams, config, getJson, GoogleParameters, diff --git a/examples/deno/pagination_ts/example.ts b/examples/deno/pagination_ts/example.ts index 24f79fb..6eabcb7 100644 --- a/examples/deno/pagination_ts/example.ts +++ b/examples/deno/pagination_ts/example.ts @@ -1,10 +1,5 @@ import { loadSync } from "https://deno.land/std@0.173.0/dotenv/mod.ts"; -import { - AllowArbitraryParams, - config, - EngineParameters, - getJson, -} from "../../../mod.ts"; +import { config, EngineParameters, getJson } from "../../../mod.ts"; const { API_KEY: apiKey } = loadSync(); @@ -15,7 +10,7 @@ const params = { engine: "google", q: "Coffee", api_key: apiKey, -} satisfies AllowArbitraryParams>; +} satisfies EngineParameters<"google">; // Pagination (async/await) let page1 = await getJson(params); diff --git a/examples/node/basic_ts_esm/example.ts b/examples/node/basic_ts_esm/example.ts index 85c35f7..6416f31 100644 --- a/examples/node/basic_ts_esm/example.ts +++ b/examples/node/basic_ts_esm/example.ts @@ -1,5 +1,5 @@ import * as Dotenv from "dotenv"; -import { AllowArbitraryParams, config, EngineParameters, getJson } from "serpapi"; +import { config, EngineParameters, getJson } from "serpapi"; Dotenv.config(); const apiKey = process.env.API_KEY; @@ -8,7 +8,7 @@ const params = { engine: "google", q: "Coffee", api_key: apiKey, -} satisfies AllowArbitraryParams>; +} satisfies EngineParameters<"google">; // Show result as JSON (async/await) const response1 = await getJson(params); diff --git a/examples/node/pagination_ts_esm/example.ts b/examples/node/pagination_ts_esm/example.ts index de98846..dbecfb6 100644 --- a/examples/node/pagination_ts_esm/example.ts +++ b/examples/node/pagination_ts_esm/example.ts @@ -1,5 +1,5 @@ import * as Dotenv from "dotenv"; -import { AllowArbitraryParams, config, EngineParameters, getJson } from "serpapi"; +import { config, EngineParameters, getJson } from "serpapi"; Dotenv.config(); const apiKey = process.env.API_KEY; @@ -11,7 +11,7 @@ const params = { engine: "google", q: "Coffee", api_key: apiKey, -} satisfies AllowArbitraryParams>; +} satisfies EngineParameters<"google">; // Pagination (async/await) let page1 = await getJson(params); diff --git a/mod.ts b/mod.ts index 8f153d5..4d1e7f5 100644 --- a/mod.ts +++ b/mod.ts @@ -10,7 +10,6 @@ export { export type { AccountApiParameters, AccountInformation, - AllowArbitraryParams, BaseParameters, BaseResponse, EngineParameters, diff --git a/src/types.ts b/src/types.ts index 3c22fb1..3ce6c47 100644 --- a/src/types.ts +++ b/src/types.ts @@ -1,10 +1,5 @@ import { EngineMap } from "./engines/engine_map.ts"; -/** - * Allow arbitrary parameters in addition to parameters in T. - */ -export type AllowArbitraryParams = T & Record; - export type BaseParameters = { /** * Parameter defines the device to use to get the results. It can be set to From 1c24395e72cff0a39a827de017192b1457ca8070 Mon Sep 17 00:00:00 2001 From: zyc9012 Date: Tue, 11 Apr 2023 16:41:20 +0800 Subject: [PATCH 26/61] Fix test --- tests/serpapi_test.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/serpapi_test.ts b/tests/serpapi_test.ts index ea920e6..9265d02 100644 --- a/tests/serpapi_test.ts +++ b/tests/serpapi_test.ts @@ -90,6 +90,7 @@ describe("getAccount", { "extra_credits", "last_hour_searches", "plan_id", + "plan_monthly_price", "plan_name", "plan_searches_left", "searches_per_month", @@ -113,6 +114,7 @@ describe("getAccount", { "extra_credits", "last_hour_searches", "plan_id", + "plan_monthly_price", "plan_name", "plan_searches_left", "searches_per_month", @@ -135,6 +137,7 @@ describe("getAccount", { "extra_credits", "last_hour_searches", "plan_id", + "plan_monthly_price", "plan_name", "plan_searches_left", "searches_per_month", From ac62022b5ec05e65568c58f8afc29e5a7daa8be3 Mon Sep 17 00:00:00 2001 From: zyc9012 Date: Mon, 17 Apr 2023 15:30:13 +0800 Subject: [PATCH 27/61] Remove polyfills for fetch, globalThis, URL and URLSearchParams We are not targeting browsers, we don't need polyfills for them. This also removes extra dependencies, which improves compatibility. --- scripts/build_npm.ts | 2 +- src/serpapi.ts | 19 +++---- src/utils.ts | 132 +++++++++++++++---------------------------- 3 files changed, 55 insertions(+), 98 deletions(-) diff --git a/scripts/build_npm.ts b/scripts/build_npm.ts index 2a268da..82e1be0 100644 --- a/scripts/build_npm.ts +++ b/scripts/build_npm.ts @@ -1,4 +1,4 @@ -import { build, emptyDir } from "https://deno.land/x/dnt@0.32.1/mod.ts"; +import { build, emptyDir } from "https://deno.land/x/dnt@0.34.0/mod.ts"; import { version } from "../version.ts"; await emptyDir("./npm"); diff --git a/src/serpapi.ts b/src/serpapi.ts index 6a9b8a9..cba0801 100644 --- a/src/serpapi.ts +++ b/src/serpapi.ts @@ -11,7 +11,6 @@ import { } from "./types.ts"; import { _internals, - execute, extractNextParameters, haveParametersChanged, } from "./utils.ts"; @@ -90,7 +89,7 @@ export async function getJson< }, timeout, ); - const json = await response.json() as BaseResponse; + const json = JSON.parse(response) as BaseResponse; const nextParametersFromResponse = await extractNextParameters(json); if ( // https://github.com/serpapi/public-roadmap/issues/562 @@ -137,7 +136,7 @@ export async function getHtml< ) { const key = validateApiKey(parameters.api_key, true); const timeout = validateTimeout(parameters.timeout); - const response = await _internals.execute( + const html = await _internals.execute( SEARCH_PATH, { ...parameters, @@ -147,7 +146,6 @@ export async function getHtml< }, timeout, ); - const html = await response.text(); callback?.(html); return html; } @@ -191,7 +189,7 @@ export async function getJsonBySearchId< }, timeout, ); - const json = await response.json() as R; + const json = JSON.parse(response) as R; callback?.(json); return json; } @@ -226,7 +224,7 @@ export async function getHtmlBySearchId( ) { const key = validateApiKey(parameters.api_key); const timeout = validateTimeout(parameters.timeout); - const response = await _internals.execute( + const html = await _internals.execute( `${SEARCH_ARCHIVE_PATH}/${searchId}`, { api_key: key, @@ -234,7 +232,6 @@ export async function getHtmlBySearchId( }, timeout, ); - const html = await response.text(); callback?.(html); return html; } @@ -260,10 +257,10 @@ export async function getAccount( ) { const key = validateApiKey(parameters.api_key); const timeout = validateTimeout(parameters.timeout); - const response = await execute(ACCOUNT_PATH, { + const response = await _internals.execute(ACCOUNT_PATH, { api_key: key, }, timeout); - const info = await response.json() as AccountInformation; + const info = JSON.parse(response) as AccountInformation; callback?.(info); return info; } @@ -289,12 +286,12 @@ export async function getLocations( callback?: (locations: Locations) => void, ) { const timeout = validateTimeout(parameters.timeout); - const response = await execute( + const response = await _internals.execute( LOCATIONS_PATH, parameters, timeout, ); - const locations = await response.json() as Locations; + const locations = JSON.parse(response) as Locations; callback?.(locations); return locations; } diff --git a/src/utils.ts b/src/utils.ts index 537879a..dab3f32 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -1,10 +1,7 @@ import type { EngineName, EngineParameters } from "./types.ts"; import { version } from "../version.ts"; - -type UrlParameters = Record< - string, - string | number | boolean | undefined | null ->; +import https from "node:https"; +import qs from "node:querystring"; /** * This `_internals` object is needed to support stubbing/spying of @@ -14,42 +11,8 @@ type UrlParameters = Record< * It's also useful to encapsulate functions that are polyfilled. */ export const _internals = { - get fetch() { - return (async () => { - // Use runtime's `fetch` if it exists, otherwise fallback to `cross-fetch`. - return typeof fetch === "function" - ? Promise.resolve(fetch) - : (await import("npm:cross-fetch@3.1.4")).default; - })(); - }, execute: execute, getBaseUrl: getBaseUrl, - get globalThis() { - return (async () => { - // Use runtime's `globalThis` if it exists, otherwise fallback to `core-js`'s implementation. - // dnt-shim-ignore - const gt = typeof globalThis !== "undefined" ? globalThis : undefined; - return gt !== undefined - ? Promise.resolve(gt) - : (await import("npm:core-js-pure@3.28.0")).default.globalThis; - })(); - }, - get URL(): Promise { - return (async () => { - // Use runtime's `URL` if it exists, otherwise fallback to `core-js`'s implementation. - return typeof URL !== "undefined" - ? Promise.resolve(URL) - : (await import("npm:core-js-pure@3.28.0")).default.URL; - })(); - }, - get URLSearchParams(): Promise { - return (async () => { - // Use runtime's `URLSearchParams` if it exists, otherwise fallback to `core-js`'s implementation. - return typeof URLSearchParams !== "undefined" - ? Promise.resolve(URLSearchParams) - : (await import("npm:core-js-pure@3.28.0")).default.URLSearchParams; - })(); - }, }; /** Facilitates stubbing in tests, e.g. localhost as the base url */ @@ -65,7 +28,7 @@ type NextParameters = { > ]: string; }; -export async function extractNextParameters( +export function extractNextParameters( json: { serpapi_pagination?: { next: string }; pagination?: { next: string }; @@ -75,7 +38,6 @@ export async function extractNextParameters( json["pagination"]?.["next"]; if (nextUrlString) { - const URL = await _internals.URL; const nextUrl = new URL(nextUrlString); const nextParameters: Record = {}; for (const [k, v] of nextUrl.searchParams.entries()) { @@ -100,65 +62,63 @@ export function haveParametersChanged( ); } -export async function getSource() { +export function getSource() { const moduleSource = `serpapi@${version}`; - try { - const gt = await _internals.globalThis; - - // Check if running in Node.js - const nodeVersion = gt.process?.versions?.node; - if (nodeVersion) { - return `nodejs@${nodeVersion},${moduleSource}`; - } - - // Assumes running in Deno instead. https://deno.land/api?s=Deno.version - // Deno.version is not shimmed since it's not used when ran in a Node env. - const denoVersion = gt.Deno?.version?.deno; + if (typeof Deno == "object") { + const denoVersion = Deno.version?.deno; if (denoVersion) { return `deno@${denoVersion},${moduleSource}`; } - - return `nodejs,${moduleSource}`; - } catch { - // If something unexpectedly occurs, revert to "nodejs". - return `nodejs,${moduleSource}`; + // @ts-ignore: scope of nodejs + } else if (typeof process == "object") { + // @ts-ignore: scope of nodejs + const nodeVersion = process.versions?.node; + if (nodeVersion) { + return `nodejs@${nodeVersion},${moduleSource}`; + } } + return `nodejs,${moduleSource}`; } -export async function buildUrl

( +export function buildUrl( path: string, - parameters: P, -): Promise { - const nonUndefinedParams: [string, string][] = Object.entries(parameters) - .filter(([_, value]) => value !== undefined) - .map(([key, value]) => [key, `${value}`]); - const URLSearchParams = await _internals.URLSearchParams; - const searchParams = new URLSearchParams(nonUndefinedParams); - return `${_internals.getBaseUrl()}${path}?${searchParams}`; + parameters: qs.ParsedUrlQueryInput, +): string { + return `${_internals.getBaseUrl()}${path}?${qs.stringify(parameters)}`; } -export async function execute

( +export function execute( path: string, - parameters: P, + parameters: qs.ParsedUrlQueryInput, timeout: number, -): Promise { - const url = await buildUrl(path, { +): Promise { + const url = buildUrl(path, { ...parameters, - source: await getSource(), + source: getSource(), }); - // https://github.com/github/fetch/issues/175#issuecomment-216791333 return new Promise((resolve, reject) => { - const timer = setTimeout(() => reject(new Error("Timeout")), timeout); - _internals.fetch.then((fetch) => - fetch(url) - .then((res) => { - clearTimeout(timer); - resolve(res); - }) - .catch((err) => { - clearTimeout(timer); - reject(err); - }) - ); + https.get(url, { timeout }, (resp) => { + let data = ""; + + // A chunk of data has been recieved. + resp.on("data", (chunk) => { + data += chunk; + }); + + // The whole response has been received. Print out the result. + resp.on("end", () => { + try { + if (resp.statusCode == 200) { + resolve(data); + } else { + reject(data); + } + } catch (e) { + reject(e); + } + }); + }).on("error", (err) => { + reject(err); + }); }); } From d9a14e82430d8613cedcd89f6ea89c01709e57ed Mon Sep 17 00:00:00 2001 From: zyc9012 Date: Mon, 17 Apr 2023 16:01:13 +0800 Subject: [PATCH 28/61] Fix tests --- src/utils.ts | 8 +++++++- tests/engines/google_test.ts | 4 ++++ tests/serpapi_test.ts | 3 +++ tests/utils_test.ts | 2 +- 4 files changed, 15 insertions(+), 2 deletions(-) diff --git a/src/utils.ts b/src/utils.ts index dab3f32..9593c5d 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -84,7 +84,13 @@ export function buildUrl( path: string, parameters: qs.ParsedUrlQueryInput, ): string { - return `${_internals.getBaseUrl()}${path}?${qs.stringify(parameters)}`; + const clonedParams = { ...parameters }; + for (const k in clonedParams) { + if (clonedParams[k] === undefined) { + delete clonedParams[k]; + } + } + return `${_internals.getBaseUrl()}${path}?${qs.stringify(clonedParams)}`; } export function execute( diff --git a/tests/engines/google_test.ts b/tests/engines/google_test.ts index b1ab2b2..1bfc199 100644 --- a/tests/engines/google_test.ts +++ b/tests/engines/google_test.ts @@ -98,6 +98,8 @@ describe("google", { api_key: "test_override_api_key", q: "coffee", }); + } catch { + // pass } finally { executeSpy.restore(); } @@ -177,6 +179,8 @@ describe("google", { api_key: "test_override_api_key", q: "coffee", }); + } catch { + // pass } finally { executeSpy.restore(); } diff --git a/tests/serpapi_test.ts b/tests/serpapi_test.ts index ea920e6..9265d02 100644 --- a/tests/serpapi_test.ts +++ b/tests/serpapi_test.ts @@ -90,6 +90,7 @@ describe("getAccount", { "extra_credits", "last_hour_searches", "plan_id", + "plan_monthly_price", "plan_name", "plan_searches_left", "searches_per_month", @@ -113,6 +114,7 @@ describe("getAccount", { "extra_credits", "last_hour_searches", "plan_id", + "plan_monthly_price", "plan_name", "plan_searches_left", "searches_per_month", @@ -135,6 +137,7 @@ describe("getAccount", { "extra_credits", "last_hour_searches", "plan_id", + "plan_monthly_price", "plan_name", "plan_searches_left", "searches_per_month", diff --git a/tests/utils_test.ts b/tests/utils_test.ts index ff689d2..6b0383d 100644 --- a/tests/utils_test.ts +++ b/tests/utils_test.ts @@ -211,7 +211,7 @@ describe("buildUrl", () => { it("with undefined parameters", async () => { assertEquals( await buildUrl("/search", { q: "coffee", gl: undefined, hl: null }), - `${BASE_URL}/search?q=coffee&hl=null`, + `${BASE_URL}/search?q=coffee&hl=`, ); }); }); From 8d37edbe61dd85342c7f41e19cfb9ec80e0e5672 Mon Sep 17 00:00:00 2001 From: zyc9012 Date: Mon, 17 Apr 2023 18:02:30 +0800 Subject: [PATCH 29/61] Fix timeout --- src/errors.ts | 7 +++++++ src/utils.ts | 10 +++++++++- tests/utils_test.ts | 13 ++++++++----- 3 files changed, 24 insertions(+), 6 deletions(-) diff --git a/src/errors.ts b/src/errors.ts index f225872..982ebf9 100644 --- a/src/errors.ts +++ b/src/errors.ts @@ -13,3 +13,10 @@ export class InvalidTimeoutError extends Error { Object.setPrototypeOf(this, InvalidTimeoutError.prototype); } } + +export class RequestTimeoutError extends Error { + constructor() { + super("The request was timed out"); + Object.setPrototypeOf(this, RequestTimeoutError.prototype); + } +} diff --git a/src/utils.ts b/src/utils.ts index 9593c5d..29eba94 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -2,6 +2,7 @@ import type { EngineName, EngineParameters } from "./types.ts"; import { version } from "../version.ts"; import https from "node:https"; import qs from "node:querystring"; +import { RequestTimeoutError } from "./errors.ts"; /** * This `_internals` object is needed to support stubbing/spying of @@ -103,7 +104,7 @@ export function execute( source: getSource(), }); return new Promise((resolve, reject) => { - https.get(url, { timeout }, (resp) => { + const req = https.get(url, (resp) => { let data = ""; // A chunk of data has been recieved. @@ -126,5 +127,12 @@ export function execute( }).on("error", (err) => { reject(err); }); + + if (timeout > 0) { + setTimeout(() => { + reject(new RequestTimeoutError()); + req.destroy(); + }, timeout); + } }); } diff --git a/tests/utils_test.ts b/tests/utils_test.ts index 6b0383d..44ba392 100644 --- a/tests/utils_test.ts +++ b/tests/utils_test.ts @@ -8,8 +8,8 @@ import { import { Stub, stub } from "https://deno.land/std@0.170.0/testing/mock.ts"; import { assertEquals, + assertInstanceOf, assertMatch, - assertRejects, } from "https://deno.land/std@0.170.0/testing/asserts.ts"; import { _internals, @@ -19,6 +19,7 @@ import { getSource, haveParametersChanged, } from "../src/utils.ts"; +import { RequestTimeoutError } from "../src/errors.ts"; loadSync({ export: true }); const BASE_URL = Deno.env.get("ENV_TYPE") === "local" @@ -230,9 +231,11 @@ describe("execute", { urlStub.restore(); }); - it("with short timeout", () => { - assertRejects(async () => - await execute("/search", { q: "coffee", gl: "us" }, 1) - ); + it("with short timeout", async () => { + try { + await execute("/search", { q: "coffee", gl: "us" }, 1); + } catch (e) { + assertInstanceOf(e, RequestTimeoutError); + } }); }); From dec0ff02f0c24bc59eaa4978fee4f7d268a875ae Mon Sep 17 00:00:00 2001 From: zyc9012 Date: Mon, 17 Apr 2023 19:15:52 +0800 Subject: [PATCH 30/61] Fix URL on Node <= 9 --- src/utils.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/utils.ts b/src/utils.ts index 29eba94..5a3995b 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -2,6 +2,7 @@ import type { EngineName, EngineParameters } from "./types.ts"; import { version } from "../version.ts"; import https from "node:https"; import qs from "node:querystring"; +import url from "node:url"; import { RequestTimeoutError } from "./errors.ts"; /** @@ -39,7 +40,7 @@ export function extractNextParameters( json["pagination"]?.["next"]; if (nextUrlString) { - const nextUrl = new URL(nextUrlString); + const nextUrl = new url.URL(nextUrlString); const nextParameters: Record = {}; for (const [k, v] of nextUrl.searchParams.entries()) { if (k === "engine") continue; From 0a04ae4fd738e8ae1713fc82f0a57a0dc7dbffbd Mon Sep 17 00:00:00 2001 From: zyc9012 Date: Mon, 17 Apr 2023 20:30:24 +0800 Subject: [PATCH 31/61] Clear timeout on finish --- src/utils.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/utils.ts b/src/utils.ts index 5a3995b..9bd82bf 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -105,6 +105,7 @@ export function execute( source: getSource(), }); return new Promise((resolve, reject) => { + let timer: number; const req = https.get(url, (resp) => { let data = ""; @@ -123,14 +124,17 @@ export function execute( } } catch (e) { reject(e); + } finally { + if (timer) clearTimeout(timer); } }); }).on("error", (err) => { reject(err); + if (timer) clearTimeout(timer); }); if (timeout > 0) { - setTimeout(() => { + timer = setTimeout(() => { reject(new RequestTimeoutError()); req.destroy(); }, timeout); From 316de24bee0f8318212dbbee68f7a2e58052d37f Mon Sep 17 00:00:00 2001 From: zyc9012 Date: Fri, 19 May 2023 14:39:07 +0800 Subject: [PATCH 32/61] Test getJson and getHtml with old and new API --- tests/serpapi_test.ts | 260 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 260 insertions(+) diff --git a/tests/serpapi_test.ts b/tests/serpapi_test.ts index 9265d02..a4b3965 100644 --- a/tests/serpapi_test.ts +++ b/tests/serpapi_test.ts @@ -8,15 +8,20 @@ import { } from "https://deno.land/std@0.170.0/testing/bdd.ts"; import { assertEquals, + assertExists, assertInstanceOf, assertRejects, } from "https://deno.land/std@0.170.0/testing/asserts.ts"; import { Stub, stub } from "https://deno.land/std@0.170.0/testing/mock.ts"; import { _internals } from "../src/utils.ts"; import { + BaseResponse, config, getAccount, + getHtml, + getJson, getLocations, + InvalidArgumentTypesError, InvalidTimeoutError, MissingApiKeyError, } from "../mod.ts"; @@ -194,3 +199,258 @@ describe("getLocations", { assertInstanceOf(locations, Array); }); }); + +describe("getJson", { + sanitizeOps: false, + sanitizeResources: false, +}, () => { + let urlStub: Stub; + + beforeAll(() => { + urlStub = stub(_internals, "getBaseUrl", () => BASE_URL); + }); + + afterEach(() => { + config.api_key = null; + }); + + afterAll(() => { + urlStub.restore(); + }); + + it("with no api_key", () => { + assertRejects( + async () => await getJson({ engine: "google", q: "Paris" }), + MissingApiKeyError, + ); + assertRejects( + async () => await getJson("google", { q: "Paris" }), + MissingApiKeyError, + ); + assertRejects( + // @ts-ignore testing invalid usage + async () => await getJson({}), + MissingApiKeyError, + ); + }); + + it("with invalid arguments", () => { + assertRejects( + // @ts-ignore testing invalid usage + async () => await getJson("google"), + InvalidArgumentTypesError, + ); + assertRejects( + // @ts-ignore testing invalid usage + async () => await getJson(), + InvalidArgumentTypesError, + ); + }); + + it("with invalid timeout", { + ignore: !HAS_API_KEY, + }, () => { + config.api_key = SERPAPI_TEST_KEY; + assertRejects( + async () => await getJson({ engine: "google", q: "Paris", timeout: 0 }), + InvalidTimeoutError, + ); + assertRejects( + async () => await getJson({ engine: "google", q: "Paris", timeout: -10 }), + InvalidTimeoutError, + ); + assertRejects( + async () => await getJson("google", { q: "Paris", timeout: 0 }), + InvalidTimeoutError, + ); + assertRejects( + async () => await getJson("google", { q: "Paris", timeout: -10 }), + InvalidTimeoutError, + ); + }); + + it("async/await", { + ignore: !HAS_API_KEY, + }, async () => { + const json = await getJson({ + engine: "google", + q: "Paris", + api_key: SERPAPI_TEST_KEY, + timeout: 10000, + }); + assertEquals(json.search_metadata["status"], "Success"); + assertExists(json.organic_results); + + // old API + const json2 = await getJson("google", { + q: "Paris", + api_key: SERPAPI_TEST_KEY, + timeout: 10000, + }); + assertEquals(json2.search_metadata.id, json.search_metadata.id); + }); + + it("callback", { + ignore: !HAS_API_KEY, + }, async () => { + const json = await new Promise>((done) => { + getJson({ + engine: "google", + q: "Paris", + api_key: SERPAPI_TEST_KEY, + timeout: 10000, + }, done); + }); + assertEquals(json.search_metadata["status"], "Success"); + assertExists(json.organic_results); + + // old API + const json2 = await new Promise>((done) => { + getJson("google", { + q: "Paris", + api_key: SERPAPI_TEST_KEY, + timeout: 10000, + }, done); + }); + assertEquals(json2.search_metadata.id, json.search_metadata.id); + }); + + it("rely on global config", { + ignore: !HAS_API_KEY, + }, async () => { + config.api_key = SERPAPI_TEST_KEY; + const json = await getJson({ + engine: "google", + q: "Paris", + timeout: 10000, + }); + assertEquals(json.search_metadata["status"], "Success"); + assertExists(json.organic_results); + }); +}); + +describe("getHtml", { + sanitizeOps: false, + sanitizeResources: false, +}, () => { + let urlStub: Stub; + + beforeAll(() => { + urlStub = stub(_internals, "getBaseUrl", () => BASE_URL); + }); + + afterEach(() => { + config.api_key = null; + }); + + afterAll(() => { + urlStub.restore(); + }); + + it("with no api_key", () => { + assertRejects( + async () => await getHtml({ engine: "google", q: "Paris" }), + MissingApiKeyError, + ); + assertRejects( + async () => await getHtml("google", { q: "Paris" }), + MissingApiKeyError, + ); + assertRejects( + // @ts-ignore testing invalid usage + async () => await getHtml({}), + MissingApiKeyError, + ); + }); + + it("with invalid arguments", () => { + assertRejects( + // @ts-ignore testing invalid usage + async () => await getHtml("google"), + InvalidArgumentTypesError, + ); + assertRejects( + // @ts-ignore testing invalid usage + async () => await getHtml(), + InvalidArgumentTypesError, + ); + }); + + it("with invalid timeout", { + ignore: !HAS_API_KEY, + }, () => { + config.api_key = SERPAPI_TEST_KEY; + assertRejects( + async () => await getHtml({ engine: "google", q: "Paris", timeout: 0 }), + InvalidTimeoutError, + ); + assertRejects( + async () => await getHtml({ engine: "google", q: "Paris", timeout: -10 }), + InvalidTimeoutError, + ); + assertRejects( + async () => await getHtml("google", { q: "Paris", timeout: 0 }), + InvalidTimeoutError, + ); + assertRejects( + async () => await getHtml("google", { q: "Paris", timeout: -10 }), + InvalidTimeoutError, + ); + }); + + it("async/await", { + ignore: !HAS_API_KEY, + }, async () => { + const html = await getHtml({ + engine: "google", + q: "Paris", + api_key: SERPAPI_TEST_KEY, + timeout: 10000, + }); + assertEquals(html.includes("Paris"), true); + + // old API + const html2 = await getHtml("google", { + q: "Paris", + api_key: SERPAPI_TEST_KEY, + timeout: 10000, + }); + assertEquals(html2, html); + }); + + it("callback", { + ignore: !HAS_API_KEY, + }, async () => { + const html = await new Promise((done) => { + getHtml({ + engine: "google", + q: "Paris", + api_key: SERPAPI_TEST_KEY, + timeout: 10000, + }, done); + }); + assertEquals(html.includes("Paris"), true); + + // old API + const html2 = await new Promise((done) => { + getHtml("google", { + q: "Paris", + api_key: SERPAPI_TEST_KEY, + timeout: 10000, + }, done); + }); + assertEquals(html2, html); + }); + + it("rely on global config", { + ignore: !HAS_API_KEY, + }, async () => { + config.api_key = SERPAPI_TEST_KEY; + const html = await getHtml({ + engine: "google", + q: "Paris", + timeout: 10000, + }); + assertEquals(html.includes("Paris"), true); + }); +}); From 91ad1ec59a9631e0b0e1be30b57f9757d3dd4ed6 Mon Sep 17 00:00:00 2001 From: zyc9012 Date: Fri, 19 May 2023 14:51:28 +0800 Subject: [PATCH 33/61] Rename to InvalidArgumentError --- mod.ts | 2 +- src/errors.ts | 6 +++--- src/serpapi.ts | 6 +++--- tests/serpapi_test.ts | 10 +++++----- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/mod.ts b/mod.ts index 4d1e7f5..81018bb 100644 --- a/mod.ts +++ b/mod.ts @@ -2,7 +2,7 @@ export type { Config } from "./src/config.ts"; export { config } from "./src/config.ts"; export { - InvalidArgumentTypesError, + InvalidArgumentError, InvalidTimeoutError, MissingApiKeyError, } from "./src/errors.ts"; diff --git a/src/errors.ts b/src/errors.ts index 63ecf3a..0d84991 100644 --- a/src/errors.ts +++ b/src/errors.ts @@ -1,7 +1,7 @@ -export class InvalidArgumentTypesError extends Error { +export class InvalidArgumentError extends Error { constructor() { - super("Provide the arguments with the correct type"); - Object.setPrototypeOf(this, InvalidArgumentTypesError.prototype); + super("Arguments are missing or of incorrect type"); + Object.setPrototypeOf(this, InvalidArgumentError.prototype); } } diff --git a/src/serpapi.ts b/src/serpapi.ts index 101bfbe..5934cce 100644 --- a/src/serpapi.ts +++ b/src/serpapi.ts @@ -1,5 +1,5 @@ import { EngineMap } from "./engines/engine_map.ts"; -import { InvalidArgumentTypesError } from "./errors.ts"; +import { InvalidArgumentError } from "./errors.ts"; import { AccountApiParameters, AccountInformation, @@ -97,7 +97,7 @@ export function getJson< const [parameters, callback] = args; return _getJson(parameters, callback); } else { - throw new InvalidArgumentTypesError(); + throw new InvalidArgumentError(); } } @@ -181,7 +181,7 @@ export function getHtml< const [parameters, callback] = args; return _getHtml(parameters, callback); } else { - throw new InvalidArgumentTypesError(); + throw new InvalidArgumentError(); } } diff --git a/tests/serpapi_test.ts b/tests/serpapi_test.ts index a4b3965..465af98 100644 --- a/tests/serpapi_test.ts +++ b/tests/serpapi_test.ts @@ -21,7 +21,7 @@ import { getHtml, getJson, getLocations, - InvalidArgumentTypesError, + InvalidArgumentError, InvalidTimeoutError, MissingApiKeyError, } from "../mod.ts"; @@ -238,12 +238,12 @@ describe("getJson", { assertRejects( // @ts-ignore testing invalid usage async () => await getJson("google"), - InvalidArgumentTypesError, + InvalidArgumentError, ); assertRejects( // @ts-ignore testing invalid usage async () => await getJson(), - InvalidArgumentTypesError, + InvalidArgumentError, ); }); @@ -367,12 +367,12 @@ describe("getHtml", { assertRejects( // @ts-ignore testing invalid usage async () => await getHtml("google"), - InvalidArgumentTypesError, + InvalidArgumentError, ); assertRejects( // @ts-ignore testing invalid usage async () => await getHtml(), - InvalidArgumentTypesError, + InvalidArgumentError, ); }); From 545da2169c729ba1b6a56c7c4a01908cbbef10ed Mon Sep 17 00:00:00 2001 From: zyc9012 Date: Mon, 22 May 2023 17:24:10 +0800 Subject: [PATCH 34/61] Add smoke test for new API --- smoke_tests/commonjs/commonjs.js | 48 ++++++++++++++++++++++++++++++-- smoke_tests/esm/esm.js | 42 ++++++++++++++++++++++++++-- tests/engines/google_test.ts | 11 ++------ 3 files changed, 89 insertions(+), 12 deletions(-) diff --git a/smoke_tests/commonjs/commonjs.js b/smoke_tests/commonjs/commonjs.js index 9c5fef0..7dc7bde 100644 --- a/smoke_tests/commonjs/commonjs.js +++ b/smoke_tests/commonjs/commonjs.js @@ -28,7 +28,7 @@ const run = async () => { { console.log("getJson async await"); - const page1 = await getJson("google", params); + const page1 = await getJson({ engine: "google", ...params }); searchId = page1["search_metadata"]["id"]; if (!page1["organic_results"]) throw new Error("No organic results"); if (page1.next) { @@ -39,7 +39,7 @@ const run = async () => { { console.log("getJson callback"); - getJson("google", params, (page1) => { + getJson({ engine: "google", ...params }, (page1) => { if (!page1["organic_results"]) throw new Error("No organic results"); if (page1.next) { page1.next((page2) => { @@ -52,6 +52,40 @@ const run = async () => { { console.log("getJson using global config"); config.api_key = apiKey; + const page1 = await getJson({ engine: "google", q: "Coffee" }); + if (!page1["organic_results"]) throw new Error("No organic results"); + if (page1.next) { + const page2 = await page1.next(); + if (!page2["organic_results"]) throw new Error("No organic results"); + } + } + + { + console.log("getJson (old API) async await"); + const page1 = await getJson("google", params); + searchId = page1["search_metadata"]["id"]; + if (!page1["organic_results"]) throw new Error("No organic results"); + if (page1.next) { + const page2 = await page1.next(); + if (!page2["organic_results"]) throw new Error("No organic results"); + } + } + + { + console.log("getJson (old API) callback"); + getJson("google", params, (page1) => { + if (!page1["organic_results"]) throw new Error("No organic results"); + if (page1.next) { + page1.next((page2) => { + if (!page2["organic_results"]) throw new Error("No organic results"); + }); + } + }); + } + + { + console.log("getJson (old API) using global config"); + config.api_key = apiKey; const page1 = await getJson("google", { q: "Coffee" }); if (!page1["organic_results"]) throw new Error("No organic results"); if (page1.next) { @@ -87,6 +121,16 @@ const run = async () => { { console.log("getHtml"); + const html = await getHtml({ engine: "google", ...params }); + if (html.length < 1000) throw new Error("Incorrect HTML"); + + getHtml({ engine: "google", ...params }, (html) => { + if (html.length < 1000) throw new Error("Incorrect HTML"); + }); + } + + { + console.log("getHtml (old API)"); const html = await getHtml("google", params); if (html.length < 1000) throw new Error("Incorrect HTML"); diff --git a/smoke_tests/esm/esm.js b/smoke_tests/esm/esm.js index 1dd8c69..d8503b7 100644 --- a/smoke_tests/esm/esm.js +++ b/smoke_tests/esm/esm.js @@ -31,7 +31,7 @@ let searchId; { console.log("getJson async await"); - const page1 = await getJson("google", params); + const page1 = await getJson({ engine: "google", ...params }); searchId = page1["search_metadata"]["id"]; if (!page1["organic_results"]) throw new Error("No organic results"); const page2 = await page1.next?.(); @@ -40,7 +40,7 @@ let searchId; { console.log("getJson callback"); - getJson("google", params, (page1) => { + getJson({ engine: "google", ...params }, (page1) => { if (!page1["organic_results"]) throw new Error("No organic results"); page1.next?.((page2) => { if (!page2["organic_results"]) throw new Error("No organic results"); @@ -51,6 +51,34 @@ let searchId; { console.log("getJson using global config"); config.api_key = apiKey; + const page1 = await getJson({ engine: "google", q: "Coffee" }); + if (!page1["organic_results"]) throw new Error("No organic results"); + const page2 = await page1.next?.(); + if (!page2["organic_results"]) throw new Error("No organic results"); +} + +{ + console.log("getJson (old API) async await"); + const page1 = await getJson("google", params); + searchId = page1["search_metadata"]["id"]; + if (!page1["organic_results"]) throw new Error("No organic results"); + const page2 = await page1.next?.(); + if (!page2["organic_results"]) throw new Error("No organic results"); +} + +{ + console.log("getJson (old API) callback"); + getJson("google", params, (page1) => { + if (!page1["organic_results"]) throw new Error("No organic results"); + page1.next?.((page2) => { + if (!page2["organic_results"]) throw new Error("No organic results"); + }); + }); +} + +{ + console.log("getJson (old API) using global config"); + config.api_key = apiKey; const page1 = await getJson("google", { q: "Coffee" }); if (!page1["organic_results"]) throw new Error("No organic results"); const page2 = await page1.next?.(); @@ -84,6 +112,16 @@ let searchId; { console.log("getHtml"); + const html = await getHtml({ engine: "google", ...params }); + if (html.length < 1000) throw new Error("Incorrect HTML"); + + getHtml({ engine: "google", ...params }, (html) => { + if (html.length < 1000) throw new Error("Incorrect HTML"); + }); +} + +{ + console.log("getHtml (old API)"); const html = await getHtml("google", params); if (html.length < 1000) throw new Error("Incorrect HTML"); diff --git a/tests/engines/google_test.ts b/tests/engines/google_test.ts index be5d91b..145403b 100644 --- a/tests/engines/google_test.ts +++ b/tests/engines/google_test.ts @@ -92,6 +92,8 @@ describe("google", { config.api_key = "test_initial_api_key"; try { await getJson({ engine, api_key: "test_override_api_key", q: "coffee" }); + } catch { + // pass } finally { executeSpy.restore(); } @@ -185,16 +187,9 @@ describe("google", { const executeSpy = spy(_internals, "execute"); config.api_key = "test_initial_api_key"; try { - // HEAD - await getHtml(engine, { - api_key: "test_override_api_key", - q: "coffee", - }); + await getHtml({ engine, api_key: "test_override_api_key", q: "coffee" }); } catch { // pass - // - await getHtml({ engine, api_key: "test_override_api_key", q: "coffee" }); - //master } finally { executeSpy.restore(); } From 0a2dfe8e04abb1bc74e6625681ea9c2e79a4c310 Mon Sep 17 00:00:00 2001 From: zyc9012 Date: Mon, 22 May 2023 17:43:42 +0800 Subject: [PATCH 35/61] Fix spread operator --- smoke_tests/commonjs/commonjs.js | 8 ++++---- smoke_tests/esm/esm.js | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/smoke_tests/commonjs/commonjs.js b/smoke_tests/commonjs/commonjs.js index 7dc7bde..b5d286f 100644 --- a/smoke_tests/commonjs/commonjs.js +++ b/smoke_tests/commonjs/commonjs.js @@ -28,7 +28,7 @@ const run = async () => { { console.log("getJson async await"); - const page1 = await getJson({ engine: "google", ...params }); + const page1 = await getJson(Object.assign({ engine: "google" }, params)); searchId = page1["search_metadata"]["id"]; if (!page1["organic_results"]) throw new Error("No organic results"); if (page1.next) { @@ -39,7 +39,7 @@ const run = async () => { { console.log("getJson callback"); - getJson({ engine: "google", ...params }, (page1) => { + getJson(Object.assign({ engine: "google" }, params), (page1) => { if (!page1["organic_results"]) throw new Error("No organic results"); if (page1.next) { page1.next((page2) => { @@ -121,10 +121,10 @@ const run = async () => { { console.log("getHtml"); - const html = await getHtml({ engine: "google", ...params }); + const html = await getHtml(Object.assign({ engine: "google" }, params)); if (html.length < 1000) throw new Error("Incorrect HTML"); - getHtml({ engine: "google", ...params }, (html) => { + getHtml(Object.assign({ engine: "google" }, params), (html) => { if (html.length < 1000) throw new Error("Incorrect HTML"); }); } diff --git a/smoke_tests/esm/esm.js b/smoke_tests/esm/esm.js index d8503b7..d6bc0d6 100644 --- a/smoke_tests/esm/esm.js +++ b/smoke_tests/esm/esm.js @@ -31,7 +31,7 @@ let searchId; { console.log("getJson async await"); - const page1 = await getJson({ engine: "google", ...params }); + const page1 = await getJson(Object.assign({ engine: "google" }, params)); searchId = page1["search_metadata"]["id"]; if (!page1["organic_results"]) throw new Error("No organic results"); const page2 = await page1.next?.(); @@ -40,7 +40,7 @@ let searchId; { console.log("getJson callback"); - getJson({ engine: "google", ...params }, (page1) => { + getJson(Object.assign({ engine: "google" }, params), (page1) => { if (!page1["organic_results"]) throw new Error("No organic results"); page1.next?.((page2) => { if (!page2["organic_results"]) throw new Error("No organic results"); @@ -112,10 +112,10 @@ let searchId; { console.log("getHtml"); - const html = await getHtml({ engine: "google", ...params }); + const html = await getHtml(Object.assign({ engine: "google" }, params)); if (html.length < 1000) throw new Error("Incorrect HTML"); - getHtml({ engine: "google", ...params }, (html) => { + getHtml(Object.assign({ engine: "google" }, params), (html) => { if (html.length < 1000) throw new Error("Incorrect HTML"); }); } From 99fc9215fb01e05562a174122dd179d2ab0ad131 Mon Sep 17 00:00:00 2001 From: zyc9012 Date: Mon, 22 May 2023 17:48:23 +0800 Subject: [PATCH 36/61] Fix url compatibility --- src/utils.ts | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/src/utils.ts b/src/utils.ts index 628eb1e..fbaaacb 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -2,6 +2,7 @@ import type { EngineParameters } from "./types.ts"; import { version } from "../version.ts"; import https from "node:https"; import qs from "node:querystring"; +import url from "node:url"; import { RequestTimeoutError } from "./errors.ts"; import { EngineMap } from "./engines/engine_map.ts"; @@ -30,16 +31,21 @@ type NextParameters = { > ]: string; }; -export function extractNextParameters(json: { - serpapi_pagination?: { next: string }; - pagination?: { next: string }; -}) { +export function extractNextParameters( + json: { + serpapi_pagination?: { next: string }; + pagination?: { next: string }; + }, +) { const nextUrlString = json["serpapi_pagination"]?.["next"] || json["pagination"]?.["next"]; if (nextUrlString) { - const nextUrl = new URL(nextUrlString); - const nextParameters = Object.fromEntries(nextUrl.searchParams.entries()); + const nextUrl = new url.URL(nextUrlString); + const nextParameters: Record = {}; + for (const [k, v] of nextUrl.searchParams.entries()) { + nextParameters[k] = v; + } return nextParameters as NextParameters; } } From 257a77abfa08b7b4ecfb7cd1c041e4d5c5a0dca5 Mon Sep 17 00:00:00 2001 From: zyc9012 Date: Wed, 24 May 2023 18:08:42 +0800 Subject: [PATCH 37/61] Update README --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 145a499..88c7889 100644 --- a/README.md +++ b/README.md @@ -26,7 +26,8 @@ npm install serpapi ```js const { getJson } = require("serpapi"); -getJson("google", { +getJson({ + engine: "google", api_key: API_KEY, // Get your API_KEY from https://serpapi.com/manage-api-key q: "coffee", location: "Austin, Texas", From 614485c0a5189e19603cc609abd583ed7d71a3d6 Mon Sep 17 00:00:00 2001 From: zyc9012 Date: Tue, 30 May 2023 20:03:40 +0800 Subject: [PATCH 38/61] Add contributor --- scripts/build_npm.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/build_npm.ts b/scripts/build_npm.ts index 82e1be0..b39746b 100644 --- a/scripts/build_npm.ts +++ b/scripts/build_npm.ts @@ -68,6 +68,7 @@ await build({ ], contributors: [ { name: "Sebastian Quek", email: "sebastian@serpapi.com" }, + { name: "Yicheng Zhou", email: "zyc9012@gmail.com" }, ], }, }); From 8fd0b0547bebca72adff6bca537c84f5df6f2af3 Mon Sep 17 00:00:00 2001 From: zyc9012 Date: Tue, 30 May 2023 21:11:22 +0800 Subject: [PATCH 39/61] Remove types for all engines --- examples/deno/basic_ts/example.ts | 2 +- examples/deno/pagination_ts/example.ts | 2 +- examples/node/basic_ts_node_14_up/example.ts | 2 +- .../node/pagination_ts_node_14_up/example.ts | 2 +- mod.ts | 56 ---- src/engines/apple_app_store.ts | 61 ---- src/engines/apple_product.ts | 26 -- src/engines/apple_reviews.ts | 43 --- src/engines/baidu.ts | 77 ----- src/engines/baidu_news.ts | 52 ---- src/engines/bing.ts | 94 ------ src/engines/bing_images.ts | 132 --------- src/engines/bing_news.ts | 74 ----- src/engines/duckduckgo.ts | 51 ---- src/engines/ebay.ts | 63 ---- src/engines/engine_map.ts | 115 -------- src/engines/google.ts | 274 ------------------ src/engines/google_about_this_result.ts | 25 -- src/engines/google_autocomplete.ts | 45 --- src/engines/google_events.ts | 53 ---- src/engines/google_finance.ts | 20 -- src/engines/google_finance_markets.ts | 37 --- src/engines/google_immersive_product.ts | 11 - src/engines/google_jobs.ts | 84 ------ src/engines/google_jobs_listing.ts | 10 - src/engines/google_lens.ts | 19 -- src/engines/google_local_services.ts | 38 --- src/engines/google_maps.ts | 85 ------ src/engines/google_maps_photo_meta.ts | 10 - src/engines/google_maps_photos.ts | 36 --- src/engines/google_maps_reviews.ts | 45 --- src/engines/google_play.ts | 125 -------- src/engines/google_play_product.ts | 69 ----- src/engines/google_product.ts | 116 -------- src/engines/google_related_questions.ts | 11 - src/engines/google_reverse_image.ts | 78 ----- src/engines/google_scholar.ts | 131 --------- src/engines/google_scholar_author.ts | 66 ----- src/engines/google_scholar_cite.ts | 11 - src/engines/google_scholar_profiles.ts | 34 --- src/engines/google_shopping.ts | 178 ------------ src/engines/google_trends.ts | 128 -------- src/engines/google_trends_autocomplete.ts | 21 -- src/engines/home_depot.ts | 76 ----- src/engines/home_depot_product.ts | 21 -- src/engines/linkedin.ts | 17 -- src/engines/linkedin_profile.ts | 10 - src/engines/naver.ts | 52 ---- src/engines/walmart.ts | 98 ------- src/engines/walmart_product.ts | 22 -- src/engines/walmart_product_reviews.ts | 45 --- src/engines/yahoo.ts | 75 ----- src/engines/yahoo_images.ts | 108 ------- src/engines/yahoo_shopping.ts | 77 ----- src/engines/yahoo_videos.ts | 70 ----- src/engines/yandex.ts | 44 --- src/engines/yandex_images.ts | 124 -------- src/engines/yandex_videos.ts | 46 --- src/engines/yelp.ts | 86 ------ src/engines/yelp_reviews.ts | 54 ---- src/engines/youtube.ts | 46 --- src/serpapi.ts | 59 ++-- src/types.ts | 20 +- src/utils.ts | 9 +- 64 files changed, 39 insertions(+), 3732 deletions(-) delete mode 100644 src/engines/apple_app_store.ts delete mode 100644 src/engines/apple_product.ts delete mode 100644 src/engines/apple_reviews.ts delete mode 100644 src/engines/baidu.ts delete mode 100644 src/engines/baidu_news.ts delete mode 100644 src/engines/bing.ts delete mode 100644 src/engines/bing_images.ts delete mode 100644 src/engines/bing_news.ts delete mode 100644 src/engines/duckduckgo.ts delete mode 100644 src/engines/ebay.ts delete mode 100644 src/engines/engine_map.ts delete mode 100644 src/engines/google.ts delete mode 100644 src/engines/google_about_this_result.ts delete mode 100644 src/engines/google_autocomplete.ts delete mode 100644 src/engines/google_events.ts delete mode 100644 src/engines/google_finance.ts delete mode 100644 src/engines/google_finance_markets.ts delete mode 100644 src/engines/google_immersive_product.ts delete mode 100644 src/engines/google_jobs.ts delete mode 100644 src/engines/google_jobs_listing.ts delete mode 100644 src/engines/google_lens.ts delete mode 100644 src/engines/google_local_services.ts delete mode 100644 src/engines/google_maps.ts delete mode 100644 src/engines/google_maps_photo_meta.ts delete mode 100644 src/engines/google_maps_photos.ts delete mode 100644 src/engines/google_maps_reviews.ts delete mode 100644 src/engines/google_play.ts delete mode 100644 src/engines/google_play_product.ts delete mode 100644 src/engines/google_product.ts delete mode 100644 src/engines/google_related_questions.ts delete mode 100644 src/engines/google_reverse_image.ts delete mode 100644 src/engines/google_scholar.ts delete mode 100644 src/engines/google_scholar_author.ts delete mode 100644 src/engines/google_scholar_cite.ts delete mode 100644 src/engines/google_scholar_profiles.ts delete mode 100644 src/engines/google_shopping.ts delete mode 100644 src/engines/google_trends.ts delete mode 100644 src/engines/google_trends_autocomplete.ts delete mode 100644 src/engines/home_depot.ts delete mode 100644 src/engines/home_depot_product.ts delete mode 100644 src/engines/linkedin.ts delete mode 100644 src/engines/linkedin_profile.ts delete mode 100644 src/engines/naver.ts delete mode 100644 src/engines/walmart.ts delete mode 100644 src/engines/walmart_product.ts delete mode 100644 src/engines/walmart_product_reviews.ts delete mode 100644 src/engines/yahoo.ts delete mode 100644 src/engines/yahoo_images.ts delete mode 100644 src/engines/yahoo_shopping.ts delete mode 100644 src/engines/yahoo_videos.ts delete mode 100644 src/engines/yandex.ts delete mode 100644 src/engines/yandex_images.ts delete mode 100644 src/engines/yandex_videos.ts delete mode 100644 src/engines/yelp.ts delete mode 100644 src/engines/yelp_reviews.ts delete mode 100644 src/engines/youtube.ts diff --git a/examples/deno/basic_ts/example.ts b/examples/deno/basic_ts/example.ts index df6c117..5959c4c 100644 --- a/examples/deno/basic_ts/example.ts +++ b/examples/deno/basic_ts/example.ts @@ -6,7 +6,7 @@ const params = { engine: "google", q: "Coffee", api_key: apiKey, -} satisfies EngineParameters<"google">; +} as EngineParameters; // Show result as JSON (async/await) const response1 = await getJson(params); diff --git a/examples/deno/pagination_ts/example.ts b/examples/deno/pagination_ts/example.ts index 6eabcb7..5444431 100644 --- a/examples/deno/pagination_ts/example.ts +++ b/examples/deno/pagination_ts/example.ts @@ -10,7 +10,7 @@ const params = { engine: "google", q: "Coffee", api_key: apiKey, -} satisfies EngineParameters<"google">; +} satisfies EngineParameters; // Pagination (async/await) let page1 = await getJson(params); diff --git a/examples/node/basic_ts_node_14_up/example.ts b/examples/node/basic_ts_node_14_up/example.ts index 0f4c7fe..63285c6 100644 --- a/examples/node/basic_ts_node_14_up/example.ts +++ b/examples/node/basic_ts_node_14_up/example.ts @@ -16,7 +16,7 @@ const params = { engine: "google", q: "Coffee", api_key: apiKey, -} satisfies EngineParameters<"google">; +} satisfies EngineParameters; // Show result as JSON (async/await) const response1 = await getJson(params); diff --git a/examples/node/pagination_ts_node_14_up/example.ts b/examples/node/pagination_ts_node_14_up/example.ts index 3262529..8975558 100644 --- a/examples/node/pagination_ts_node_14_up/example.ts +++ b/examples/node/pagination_ts_node_14_up/example.ts @@ -21,7 +21,7 @@ const params = { engine: "google", q: "Coffee", api_key: apiKey, -} satisfies EngineParameters<"google">; +} satisfies EngineParameters; // Pagination (async/await) let page1 = await getJson(params); diff --git a/mod.ts b/mod.ts index 81018bb..5d03d1d 100644 --- a/mod.ts +++ b/mod.ts @@ -26,59 +26,3 @@ export { getJsonBySearchId, getLocations, } from "./src/serpapi.ts"; - -export type { GoogleParameters } from "./src/engines/google.ts"; -export type { GoogleShoppingParameters } from "./src/engines/google_shopping.ts"; -export type { GoogleJobsParameters } from "./src/engines/google_jobs.ts"; -export type { GoogleJobsListingParameters } from "./src/engines/google_jobs_listing.ts"; -export type { GoogleReverseImageParameters } from "./src/engines/google_reverse_image.ts"; -export type { GoogleScholarProfilesParameters } from "./src/engines/google_scholar_profiles.ts"; -export type { GoogleScholarParameters } from "./src/engines/google_scholar.ts"; -export type { GoogleScholarCiteParameters } from "./src/engines/google_scholar_cite.ts"; -export type { GoogleScholarAuthorParameters } from "./src/engines/google_scholar_author.ts"; -export type { GoogleProductParameters } from "./src/engines/google_product.ts"; -export type { GoogleMapsParameters } from "./src/engines/google_maps.ts"; -export type { GoogleMapsPhotosParameters } from "./src/engines/google_maps_photos.ts"; -export type { GoogleMapsPhotoMetaParameters } from "./src/engines/google_maps_photo_meta.ts"; -export type { GoogleMapsReviewsParameters } from "./src/engines/google_maps_reviews.ts"; -export type { GoogleEventsParameters } from "./src/engines/google_events.ts"; -export type { GoogleAutocompleteParameters } from "./src/engines/google_autocomplete.ts"; -export type { GoogleRelatedQuestionsParameters } from "./src/engines/google_related_questions.ts"; -export type { GoogleTrendsParameters } from "./src/engines/google_trends.ts"; -export type { GoogleTrendsAutocompleteParameters } from "./src/engines/google_trends_autocomplete.ts"; -export type { GoogleFinanceParameters } from "./src/engines/google_finance.ts"; -export type { GoogleFinanceMarketsParameters } from "./src/engines/google_finance_markets.ts"; -export type { GoogleImmersiveProductParameters } from "./src/engines/google_immersive_product.ts"; -export type { BingParameters } from "./src/engines/bing.ts"; -export type { BingNewsParameters } from "./src/engines/bing_news.ts"; -export type { BingImagesParameters } from "./src/engines/bing_images.ts"; -export type { BaiduParameters } from "./src/engines/baidu.ts"; -export type { BaiduNewsParameters } from "./src/engines/baidu_news.ts"; -export type { YahooParameters } from "./src/engines/yahoo.ts"; -export type { YahooImagesParameters } from "./src/engines/yahoo_images.ts"; -export type { YahooVideosParameters } from "./src/engines/yahoo_videos.ts"; -export type { YahooShoppingParameters } from "./src/engines/yahoo_shopping.ts"; -export type { EbayParameters } from "./src/engines/ebay.ts"; -export type { YandexParameters } from "./src/engines/yandex.ts"; -export type { YandexImagesParameters } from "./src/engines/yandex_images.ts"; -export type { YandexVideosParameters } from "./src/engines/yandex_videos.ts"; -export type { YoutubeParameters } from "./src/engines/youtube.ts"; -export type { WalmartParameters } from "./src/engines/walmart.ts"; -export type { WalmartProductParameters } from "./src/engines/walmart_product.ts"; -export type { WalmartProductReviewsParameters } from "./src/engines/walmart_product_reviews.ts"; -export type { HomeDepotParameters } from "./src/engines/home_depot.ts"; -export type { HomeDepotProductParameters } from "./src/engines/home_depot_product.ts"; -export type { LinkedinParameters } from "./src/engines/linkedin.ts"; -export type { LinkedinProfileParameters } from "./src/engines/linkedin_profile.ts"; -export type { DuckduckgoParameters } from "./src/engines/duckduckgo.ts"; -export type { GooglePlayProductParameters } from "./src/engines/google_play_product.ts"; -export type { GooglePlayParameters } from "./src/engines/google_play.ts"; -export type { AppleAppStoreParameters } from "./src/engines/apple_app_store.ts"; -export type { AppleReviewsParameters } from "./src/engines/apple_reviews.ts"; -export type { AppleProductParameters } from "./src/engines/apple_product.ts"; -export type { NaverParameters } from "./src/engines/naver.ts"; -export type { GoogleLensParameters } from "./src/engines/google_lens.ts"; -export type { GoogleLocalServicesParameters } from "./src/engines/google_local_services.ts"; -export type { GoogleAboutThisResultParameters } from "./src/engines/google_about_this_result.ts"; -export type { YelpParameters } from "./src/engines/yelp.ts"; -export type { YelpReviewsParameters } from "./src/engines/yelp_reviews.ts"; diff --git a/src/engines/apple_app_store.ts b/src/engines/apple_app_store.ts deleted file mode 100644 index 6c0fd41..0000000 --- a/src/engines/apple_app_store.ts +++ /dev/null @@ -1,61 +0,0 @@ -import type { BaseParameters } from "../types.ts"; - -export type AppleAppStoreParameters = BaseParameters & { - /** - * Search Term - * Parameter defines the query you want to search. You can use any search term that - * you would use in a regular App Store search. (e.g. `Coffee`) - */ - term: string; - - /** - * Store Region - * Parameter defines the country to use for the search. It's a two-letter country - * code. (e.g., `us` (default) for the United States, `uk` for United Kingdom, or - * `fr` for France). Head to the [Apple Regions](https://serpapi.com/apple-regions) - * for a full list of supported Apple Regions - */ - country?: string; - - /** - * Store Language - * Parameter defines the language to use for the search. It's a four-letter country - * code. (e.g., `en-us` (default) for the English, `fr-fr` for French, or `uk-ua` - * for Ukranian). Head to the [Apple - * Languages](https://serpapi.com/apple-languages) for a full list of supported - * Apple Languages - */ - lang?: string; - - /** - * Result Count - * Parameter defines the number of results you want to get per each page. It - * defaults to `10`. Maximum number of results you can get per page is `200`. Any - * number greater than maximum number will default to `200`. - */ - num?: string; - - /** - * Page Number - * Parameter is used to get the items on a specific page. (e.g., 0 (default) is the - * first page of results, 1 is the 2nd page of results, 2 is the 3rd page of - * results, etc.). - */ - page?: string; - - /** - * Disallow Explicit Apps - * Parameter defines the filter for disallowing explicit apps. It defaults to - * `false`. - */ - disallow_explicit?: boolean; - - /** - * App Property - * Parameter allows to search the property of an app. - * `developer` allows searching the developer title of an app ( e.g., - * property=`developer` and term=`Coffee` gives apps with "Coffee" in their - * developer's name. (Ex: `Coffee Inc.`) - */ - property?: string; -}; diff --git a/src/engines/apple_product.ts b/src/engines/apple_product.ts deleted file mode 100644 index 4272e64..0000000 --- a/src/engines/apple_product.ts +++ /dev/null @@ -1,26 +0,0 @@ -import type { BaseParameters } from "../types.ts"; - -export type AppleProductParameters = BaseParameters & { - /** - * Product Id - * Parameter defines the product id you want to search. You can use the specific id - * of a product that you would like to get the product page of. - */ - product_id: string; - - /** - * Product Type - * Parameter defines the type of Apple Product to get the product page of. It - * defaults to `app`. - */ - type?: string; - - /** - * Store Region - * Parameter defines the country to use for the search. It's a two-letter country - * code. (e.g., `us` (default) for the United States, `uk` for United Kingdom, or - * `fr` for France). Head to the [Apple Regions](https://serpapi.com/apple-regions) - * for a full list of supported Apple Regions. - */ - country?: string; -}; diff --git a/src/engines/apple_reviews.ts b/src/engines/apple_reviews.ts deleted file mode 100644 index 1d22f46..0000000 --- a/src/engines/apple_reviews.ts +++ /dev/null @@ -1,43 +0,0 @@ -import type { BaseParameters } from "../types.ts"; - -export type AppleReviewsParameters = BaseParameters & { - /** - * Product Id - * Parameter defines the ID of a product you want to get the reviews for. - * You can find the ID of a product from [App Store - * API](https://serpapi.com/apple-app-store) json results, - * You can also get it from the URL of the app. For example - * `product_id` of - * "https://apps.apple.com/us/app/the-great-coffee-app/id534220544", is the long - * numerical value that comes after "id", 534220544 (default). - */ - product_id: string; - - /** - * Store Region - * Parameter defines the country to use for the search. It's a two-letter country - * code. (e.g., `us` (default) for the United States, `uk` for United Kingdom, or - * `fr` for France). Head to the [Apple Regions](https://serpapi.com/apple-regions) - * for a full list of supported Apple Regions. - */ - country?: string; - - /** - * Page Number - * Parameter is used to get the items on a specific page. (e.g., 1 (default) is the - * first page of results, 2 is the 2nd page of results, 3 is the 3rd page of - * results, etc.). - */ - page?: string; - - /** - * Sort by - * Parameter is used for sorting reviews. - * It can be set to: - * `mostrecent`: Most recent (default), - * `mosthelpful`: Most helpful, - * `mostfavorable`: Most favorable, - * `mostcritical`: Most critical - */ - sort?: string; -}; diff --git a/src/engines/baidu.ts b/src/engines/baidu.ts deleted file mode 100644 index 7143052..0000000 --- a/src/engines/baidu.ts +++ /dev/null @@ -1,77 +0,0 @@ -import type { BaseParameters } from "../types.ts"; - -export type BaiduParameters = BaseParameters & { - /** - * Search Query - * Parameter defines the the search query, including all Baidu search operators. - * (e.g., `inurl:`, `site:`, `intitle:`, etc.). - */ - q: string; - - /** - * Choose Language - * Parameter defines which language to restrict results. Available options: - * `1` - All languages - * `2` - Simplified Chinese - * `3` - Traditional Chinese - */ - ct?: string; - - /** - * Result Offset - * Parameter defines the result offset. It skips the given number of results. It's - * used for pagination. (e.g., `0` (default) is the first page of results, `10` is - * the 2nd page of results, `20` is the 3rd page of results, etc.). - */ - pn?: string; - - /** - * Number of Results - * Parameter defines the maximum number of results to return, limited to 50. (e.g., - * `10` (default) returns 10 results, `30` returns 30 results, and `50` returns 50 - * results). This parameter is only available for **desktop and tablet** searches. - */ - rn?: string; - - /** - * Time Period for Results - * Parameter defines the time period for results. (e.g., - * `stf=1672999365,1673604165|stftype=1` only returns results from the past 7 days. - * First integer within the parameter,`1672999365` is Unix Timestamp for 7 days - * ago. Second integer,`1673604165` is Unix Timestamp for now.). - */ - gpc?: string; - - /** - * Search Type - * Similar to using `inurl:` or `intitle:`. (e.g., `1` to search by page title, `2` - * to search by web address.). - */ - q5?: string; - - /** - * Search Type - * Similar to using `site:`. (e.g., `q6=serpapi.com` to search for results only - * from the domain `serpapi.com`). - */ - q6?: string; - - /** - * Previous Search Query - * Defines the previous search query. - */ - bs?: string; - - /** - * Original Search Query - * Defines the original search query when navigated from a related search. - */ - oq?: string; - - /** - * Originating Search Type - * Defines the originating search type. (e.g., `8` is a normal search, `3` is from - * the suggestion list, and `1` is a related search. - */ - f?: string; -}; diff --git a/src/engines/baidu_news.ts b/src/engines/baidu_news.ts deleted file mode 100644 index 5ab6f6b..0000000 --- a/src/engines/baidu_news.ts +++ /dev/null @@ -1,52 +0,0 @@ -import type { BaseParameters } from "../types.ts"; - -export type BaiduNewsParameters = BaseParameters & { - /** - * Search Query - * Parameter defines the the search query, including all Baidu search operators. - * (e.g., `inurl:`, `site:`, `intitle:`, etc.). - */ - q: string; - - /** - * Choose Language - * Parameter defines which language to restrict results. Available options: - * `1` - All Languages. - * `2` - Simplified Chinese - * `3` - Traditional Chinese. - */ - ct?: string; - - /** - * Result Offset - * Parameter defines the result offset. It skips the given number of results. It's - * used for pagination. (e.g., `0` (default) is the first page of results, `10` is - * the 2nd page of results, `20` is the 3rd page of results, etc.). - */ - pn?: string; - - /** - * Number of Results - * Parameter defines the maximum number of results to return, limited to 50. (e.g., - * `10` (default) returns 10 results, `30` returns 30 results, and `50` returns 50 - * results). - */ - rn?: string; - - /** - * Sort Type - * Parameter defines the sort type for results. Available options: - * `1` - Sort by attraction (default) - * `4` - Sort by time - */ - rtt?: string; - - /** - * Medium Filtering - * Parameter defines medium filtering for results. Available options: - * `0` - No filtering - * `1` - Show results from medium sites - * `2` - Show results from Baijiahao (baijiahao.baidu.com) - */ - medium?: string; -}; diff --git a/src/engines/bing.ts b/src/engines/bing.ts deleted file mode 100644 index a094c4d..0000000 --- a/src/engines/bing.ts +++ /dev/null @@ -1,94 +0,0 @@ -import type { BaseParameters } from "../types.ts"; - -export type BingParameters = BaseParameters & { - /** - * Search Query - * Parameter defines the search query. You can use anything that you would use in a - * regular Bing search. (e.g., `'query'`, `NOT`, `OR`, `site:`, `filetype:`, - * `near:`, `ip:`, `loc:`, `feed:` etc.). - */ - q: string; - - /** - * Location - * Parameter defines from where you want the search to originate. If several - * locations match the location requested, we'll pick the most popular one. Head to - * the [/locations.json API](https://serpapi.com/locations-api) if you need more - * precise control. - */ - location?: string; - - /** - * Latitude - * Defines a GPS latitude for the search origin - */ - lat?: string; - - /** - * Longitude - * Defines a GPS longitude for the search origin - */ - lon?: string; - - /** - * Market codes - * The market where the results come from (e.g. `en-US`). Typically, mkt is the - * country where the user is making the request from. However, it could be a - * different country if the user is not located in a country where Bing delivers - * results. The market must be in the form -. For - * example, en-US. The string is case insensitive. For a list of possible market - * values, see [Market - * Codes](https://docs.microsoft.com/en-us/rest/api/cognitiveservices/bing-custom-search-api-v7-reference#market-codes). - * NOTE: If known, you are encouraged to always specify the market. Specifying the - * market helps Bing route the request and return an appropriate and optimal - * response. If you specify a market that is not listed in [Market - * Codes](https://docs.microsoft.com/en-us/rest/api/cognitiveservices/bing-custom-search-api-v7-reference#market-codes), - * Bing uses a best fit market code based on an internal mapping that is subject to - * change. - * This parameter and the cc query parameter are mutually exclusive—do not specify - * both. - */ - mkt?: string; - - /** - * Country - * Parameter defines the country to search from. It follows the 2-character - * [ISO_3166-1](https://en.wikipedia.org/wiki/ISO_3166-1) format. (e.g., `us` for - * United States, `de` for Germany, `gb` for United Kingdom, etc.). - */ - cc?: string; - - /** - * Result Offset - * Parameter controls the offset of the organic results. This parameter defaults to - * `1`. (e.g., `first=10` will move the 10th organic result to the first position). - */ - first?: string; - - /** - * Number of Results - * Parameter controls the number of results per page. Minimum: `1`, Maximum: `50`. - * This parameter is only a suggestion and might not reflect actual results - * returned. - */ - count?: string; - - /** - * Adult Content Filtering - * Parameter defines the level of filtering for adult content. It can be set to: - * `Off` to return webpages with adult text, images, or videos. - * `Moderate` to return webpages with adult text, but not adult images or videos. - * `Strict` to not return webpages with adult text, images, or videos. - */ - safeSearch?: string; - - /** - * Additional Filtering - * Parameter allows usage of a more complex filtering options such as filtering by - * date range `ex1:"ez5_18169_18230"` or using a specific display filters such as - * `ufn:"Wunderman+Thompson"+sid:"5bede9a2-1bda-9887-e6eb-30b1b8b6b513"+catguid:"5bede9a2-1bda-9887-e6eb-30b1b8b6b513_cfb02057"+segment:"generic.carousel"+entitysegment:"Organization"`. - * Exact values can be constructed by using Bing search and copying `filters` query - * parameter. - */ - filters?: string; -}; diff --git a/src/engines/bing_images.ts b/src/engines/bing_images.ts deleted file mode 100644 index 3b0936c..0000000 --- a/src/engines/bing_images.ts +++ /dev/null @@ -1,132 +0,0 @@ -import type { BaseParameters } from "../types.ts"; - -export type BingImagesParameters = BaseParameters & { - /** - * Search Query - * Parameter defines the search query. You can use anything that you would use in a - * regular Bing Images search. - */ - q: string; - - /** - * Market codes - * The market where the results come from (e.g. `en-US`). Typically, mkt is the - * country where the user is making the request from. However, it could be a - * different country if the user is not located in a country where Bing Images API - * delivers results. The market must be in the form -. - * For example, en-US. The string is case insensitive. For a list of possible - * market values, see [Market - * Codes](https://docs.microsoft.com/en-us/rest/api/cognitiveservices/bing-custom-search-api-v7-reference#market-codes). - * NOTE: If known, you are encouraged to always specify the market. Specifying the - * market helps Bing route the request and return an appropriate and optimal - * response. If you specify a market that is not listed in [Market - * Codes](https://docs.microsoft.com/en-us/rest/api/cognitiveservices/bing-custom-search-api-v7-reference#market-codes), - * Bing Images API uses a best fit market code based on an internal mapping that is - * subject to change. - * This parameter and the cc query parameter are mutually exclusive—do not specify - * both. - */ - mkt?: string; - - /** - * Country - * Parameter defines the country to search from. It follows the 2-character - * [ISO_3166-1](https://en.wikipedia.org/wiki/ISO_3166-1) format. (e.g., `us` for - * United States, `de` for Germany, `gb` for United Kingdom, etc.). - */ - cc?: string; - - /** - * Result Offset - * Parameter controls the offset of the organic results. This parameter defaults to - * `1`. (e.g., `first=10` will move the 10th organic result to the first position). - */ - first?: string; - - /** - * Number of Results - * Parameter controls the number of results per page. This parameter is only a - * suggestion and might not reflect the returned results. - */ - count?: string; - - /** - * Image Size - * Parameter is used for filtering images by size. It can be set to: - * `small` - Small - * `medium` - Medium - * `large` - Large - * `wallpaper` - Extra Large - */ - imagesize?: string; - - /** - * Color - * Parameter is used for filtering images by color. It can be set to: - * `color` - Color Only - * `bw` - Black & white - * `FGcls_RED` - Red - * `FGcls_ORGANGE` - Orange - * `FGcls_YELLOW` - Yellow - * `FGcls_GREEN` - Green - * `FGcls_TEAL` - Teal - * `FGcls_BLUE` - Blue - * `FGcls_PURPLE` - Purple - * `FGcls_PINK` - Pink - * `FGcls_BROWN` - Brown - * `FGcls_BLACK` - Black - * `FGcls_GRAY` - Gray - * `FGcls_WHITE` - White - */ - color2?: string; - - /** - * Type - * Parameter is used for filtering images by image type. It can be set to: - * `photo` - Photo - * `clipart` - Clipart - * `linedrawing` - Line Drawing - * `animatedgif` - Animated GIF - * `transparent` - Transparent - */ - photo?: string; - - /** - * Layout - * Parameter is used for filtering images by layout. It can be set to: - * `square` - Square - * `wide` - Wide - * `tall` - Tall - */ - aspect?: string; - - /** - * People - * Parameter is used for filtering images by people. It can be set to: - * `face` - Faces Only - * `portrait` - Head & Shoulders - */ - face?: string; - - /** - * Date - * Parameter is used for filtering images by date. It can be set to: - * `lt1440` - Past 24 hours - * `lt10080` - Past week - * `lt43200` - Past month - * `lt525600` - Past year - */ - age?: string; - - /** - * License - * Parameter is used for filtering images by license. It can be set to: - * `Type-Any` - All Creative Commons - * `L1` - Public Domain - * `L2_L3_L4_L5_L6_L7` - Free to share and use - * `L2_L3_L4` - Free to share and use commercially - * `L2_L3_L5_L6` - Free to modify, share and use - * `L2_L3` - Free to modify, share, and use commercially - */ - license?: string; -}; diff --git a/src/engines/bing_news.ts b/src/engines/bing_news.ts deleted file mode 100644 index e60c611..0000000 --- a/src/engines/bing_news.ts +++ /dev/null @@ -1,74 +0,0 @@ -import type { BaseParameters } from "../types.ts"; - -export type BingNewsParameters = BaseParameters & { - /** - * Search Query - * Parameter defines the search query. You can use anything that you would use in a - * regular Bing search. (e.g., `'query'`, `NOT`, `OR`, `site:`, `filetype:`, - * `near:`, `ip:`, `loc:`, `feed:` etc.). - */ - q: string; - - /** - * Market codes - * The market where the results come from (e.g. `en-US`). Typically, mkt is the - * country where the user is making the request from. However, it could be a - * different country if the user is not located in a country where Bing delivers - * results. The market must be in the form -. For - * example, en-US. The string is case insensitive. For a list of possible market - * values, see [Market - * Codes](https://docs.microsoft.com/en-us/rest/api/cognitiveservices/bing-custom-search-api-v7-reference#market-codes). - * NOTE: If known, you are encouraged to always specify the market. Specifying the - * market helps Bing route the request and return an appropriate and optimal - * response. If you specify a market that is not listed in [Market - * Codes](https://docs.microsoft.com/en-us/rest/api/cognitiveservices/bing-custom-search-api-v7-reference#market-codes), - * Bing uses a best fit market code based on an internal mapping that is subject to - * change. - * This parameter and the cc query parameter are mutually exclusive—do not specify - * both. - */ - mkt?: string; - - /** - * Country - * Parameter defines the country to search from. It follows the 2-character - * [ISO_3166-1](https://en.wikipedia.org/wiki/ISO_3166-1) format. (e.g., `us` for - * United States, `de` for Germany, `gb` for United Kingdom, etc.). - */ - cc?: string; - - /** - * Result Offset - * Parameter controls the offset of the organic results. This parameter defaults to - * `1`. (e.g., `first=10` will move the 10th organic result to the first position). - */ - first?: string; - - /** - * Number of Results - * Parameter controls the number of results per page. This parameter is only a - * suggestion and might not reflect actual results returned. - */ - count?: string; - - /** - * Sort by Date - * Parameter defines results sorted by date. If the parameter is not set, it will - * default to the "Best match" sorting, otherwise, if set to: e.g. `interval="4"`, - * it will show results from the past hour. Other options are: - * `interval="7"`: Past 24 hours, - * `interval="8"`: Past 7 days, - * `interval="9"`: Past 30 days, - * `sortbydate="1"`: Most Recent - */ - qft?: string; - - /** - * Adult Content Filtering - * Parameter defines the level of filtering for adult content. It can be set to: - * `Off` to return webpages with adult text, images, or videos. - * `Moderate` to return webpages with adult text, but not adult images or videos. - * `Strict` to not return webpages with adult text, images, or videos. - */ - safeSearch?: string; -}; diff --git a/src/engines/duckduckgo.ts b/src/engines/duckduckgo.ts deleted file mode 100644 index 3b8d36d..0000000 --- a/src/engines/duckduckgo.ts +++ /dev/null @@ -1,51 +0,0 @@ -import type { BaseParameters } from "../types.ts"; - -export type DuckduckgoParameters = BaseParameters & { - /** - * Search Query - * Parameter defines the query you want to search. You can use anything that you - * would use in a regular DuckDuckGo search. (e.g., `inurl:`, `site:`, `intitle:`, - * etc.) - */ - q: string; - - /** - * Region - * Parameter defines the region to use for the DuckDuckGo search. Region code - * examples: `us-en` for the United States, `uk-en` for United Kingdom, or `fr-fr` - * for France. Head to the [DuckDuckGo - * regions](https://serpapi.com/duckduckgo-regions) for a full list of supported - * regions. - */ - kl?: string; - - /** - * Adult Content Filtering - * Parameter defines the level of filtering for adult content. It can be set to `1` - * (Strict), `-1` (Moderate - default), or `-2` (Off). - */ - safe?: string; - - /** - * Filter By Date - * Parameter defines results filtered by date. - * It can be set to: - * `d`: Past day, - * `w`: Past week, - * `m`: Past month, - * `y`: Past year, - * or you can pass a custom date following the next format: `from_date` + `..` + - * `to_date` (e.g. `2021-06-15..2021-06-16`). - */ - df?: string; - - /** - * Result Offset - * Parameter defines the result offset. It skips the given number of results. It's - * used for pagination. When pagination is not being used (initial search request), - * number of `organic_results` can vary between 26 and 30. When pagination is being - * used (value of start parameter is bigger then 0), `organic_results` return 50 - * results. - */ - start?: number; -}; diff --git a/src/engines/ebay.ts b/src/engines/ebay.ts deleted file mode 100644 index 06f1a56..0000000 --- a/src/engines/ebay.ts +++ /dev/null @@ -1,63 +0,0 @@ -import type { BaseParameters } from "../types.ts"; - -export type EbayParameters = BaseParameters & { - /** - * Search Query - * Parameter defines the search query. You can use anything that you would use in a - * regular Ebay search. - */ - _nkw: string; - - /** - * Domain - * Parameter defines the Ebay domain to use. It defaults to `ebay.com`. Head to the - * [Ebay domains](https://serpapi.com/ebay-domains) for a full list of supported - * Ebay domains. - */ - ebay_domain?: string; - - /** - * Page Number - * Parameter defines the page number. It's used for pagination. (e.g., `1` - * (default) is the first page of results, `2` is the 2nd page of results, `3` is - * the 3rd page of results, etc.). - */ - _pgn?: string; - - /** - * Number of Results - * Parameter defines the maximum number of results to return. There are total of - * four options: `25`, `50` (default), `100` and `200` results. - */ - _ipg?: string; - - /** - * Exclude Auto-corrected Results - * Parameter defines the exclusion of results from an auto-corrected query that is - * spelled wrong. It should be set to `spell_auto_correct` to exclude these - * results. - */ - _blrs?: string; - - /** - * Price Low - * Parameter defines the lowest price of items that should be included in the - * results (e.g. `10` will only return items that have higher price then `10`). - */ - _udlo?: string; - - /** - * Price High - * Parameter defines the highest price of items that should be included in the - * results (e.g. `20` will only return items that have lower price then `20`). - */ - _udhi?: string; - - /** - * Category ID - * Parameter defines the ID of a category where you want your search to be - * concentrated. ID values are accessible inside `categories` array, located in our - * JSON output (e.g. `categories[1].id`). - */ - category_id?: string; -}; diff --git a/src/engines/engine_map.ts b/src/engines/engine_map.ts deleted file mode 100644 index ba71040..0000000 --- a/src/engines/engine_map.ts +++ /dev/null @@ -1,115 +0,0 @@ -import type { GoogleParameters } from "./google.ts"; -import type { GoogleShoppingParameters } from "./google_shopping.ts"; -import type { GoogleJobsParameters } from "./google_jobs.ts"; -import type { GoogleJobsListingParameters } from "./google_jobs_listing.ts"; -import type { GoogleReverseImageParameters } from "./google_reverse_image.ts"; -import type { GoogleScholarProfilesParameters } from "./google_scholar_profiles.ts"; -import type { GoogleScholarParameters } from "./google_scholar.ts"; -import type { GoogleScholarCiteParameters } from "./google_scholar_cite.ts"; -import type { GoogleScholarAuthorParameters } from "./google_scholar_author.ts"; -import type { GoogleProductParameters } from "./google_product.ts"; -import type { GoogleMapsParameters } from "./google_maps.ts"; -import type { GoogleMapsPhotosParameters } from "./google_maps_photos.ts"; -import type { GoogleMapsPhotoMetaParameters } from "./google_maps_photo_meta.ts"; -import type { GoogleMapsReviewsParameters } from "./google_maps_reviews.ts"; -import type { GoogleEventsParameters } from "./google_events.ts"; -import type { GoogleAutocompleteParameters } from "./google_autocomplete.ts"; -import type { GoogleRelatedQuestionsParameters } from "./google_related_questions.ts"; -import type { GoogleTrendsParameters } from "./google_trends.ts"; -import type { GoogleTrendsAutocompleteParameters } from "./google_trends_autocomplete.ts"; -import type { GoogleFinanceParameters } from "./google_finance.ts"; -import type { GoogleFinanceMarketsParameters } from "./google_finance_markets.ts"; -import type { GoogleImmersiveProductParameters } from "./google_immersive_product.ts"; -import type { BingParameters } from "./bing.ts"; -import type { BingNewsParameters } from "./bing_news.ts"; -import type { BingImagesParameters } from "./bing_images.ts"; -import type { BaiduParameters } from "./baidu.ts"; -import type { BaiduNewsParameters } from "./baidu_news.ts"; -import type { YahooParameters } from "./yahoo.ts"; -import type { YahooImagesParameters } from "./yahoo_images.ts"; -import type { YahooVideosParameters } from "./yahoo_videos.ts"; -import type { YahooShoppingParameters } from "./yahoo_shopping.ts"; -import type { EbayParameters } from "./ebay.ts"; -import type { YandexParameters } from "./yandex.ts"; -import type { YandexImagesParameters } from "./yandex_images.ts"; -import type { YandexVideosParameters } from "./yandex_videos.ts"; -import type { YoutubeParameters } from "./youtube.ts"; -import type { WalmartParameters } from "./walmart.ts"; -import type { WalmartProductParameters } from "./walmart_product.ts"; -import type { WalmartProductReviewsParameters } from "./walmart_product_reviews.ts"; -import type { HomeDepotParameters } from "./home_depot.ts"; -import type { HomeDepotProductParameters } from "./home_depot_product.ts"; -import type { LinkedinParameters } from "./linkedin.ts"; -import type { LinkedinProfileParameters } from "./linkedin_profile.ts"; -import type { DuckduckgoParameters } from "./duckduckgo.ts"; -import type { GooglePlayProductParameters } from "./google_play_product.ts"; -import type { GooglePlayParameters } from "./google_play.ts"; -import type { AppleAppStoreParameters } from "./apple_app_store.ts"; -import type { AppleReviewsParameters } from "./apple_reviews.ts"; -import type { AppleProductParameters } from "./apple_product.ts"; -import type { NaverParameters } from "./naver.ts"; -import type { GoogleLensParameters } from "./google_lens.ts"; -import type { GoogleLocalServicesParameters } from "./google_local_services.ts"; -import type { GoogleAboutThisResultParameters } from "./google_about_this_result.ts"; -import type { YelpParameters } from "./yelp.ts"; -import type { YelpReviewsParameters } from "./yelp_reviews.ts"; - -export type EngineMap = { - google: { parameters: GoogleParameters }; - google_shopping: { parameters: GoogleShoppingParameters }; - google_jobs: { parameters: GoogleJobsParameters }; - google_jobs_listing: { parameters: GoogleJobsListingParameters }; - google_reverse_image: { parameters: GoogleReverseImageParameters }; - google_scholar_profiles: { parameters: GoogleScholarProfilesParameters }; - google_scholar: { parameters: GoogleScholarParameters }; - google_scholar_cite: { parameters: GoogleScholarCiteParameters }; - google_scholar_author: { parameters: GoogleScholarAuthorParameters }; - google_product: { parameters: GoogleProductParameters }; - google_maps: { parameters: GoogleMapsParameters }; - google_maps_photos: { parameters: GoogleMapsPhotosParameters }; - google_maps_photo_meta: { parameters: GoogleMapsPhotoMetaParameters }; - google_maps_reviews: { parameters: GoogleMapsReviewsParameters }; - google_events: { parameters: GoogleEventsParameters }; - google_autocomplete: { parameters: GoogleAutocompleteParameters }; - google_related_questions: { parameters: GoogleRelatedQuestionsParameters }; - google_trends: { parameters: GoogleTrendsParameters }; - google_trends_autocomplete: { - parameters: GoogleTrendsAutocompleteParameters; - }; - google_finance: { parameters: GoogleFinanceParameters }; - google_finance_markets: { parameters: GoogleFinanceMarketsParameters }; - google_immersive_product: { parameters: GoogleImmersiveProductParameters }; - bing: { parameters: BingParameters }; - bing_news: { parameters: BingNewsParameters }; - bing_images: { parameters: BingImagesParameters }; - baidu: { parameters: BaiduParameters }; - baidu_news: { parameters: BaiduNewsParameters }; - yahoo: { parameters: YahooParameters }; - yahoo_images: { parameters: YahooImagesParameters }; - yahoo_videos: { parameters: YahooVideosParameters }; - yahoo_shopping: { parameters: YahooShoppingParameters }; - ebay: { parameters: EbayParameters }; - yandex: { parameters: YandexParameters }; - yandex_images: { parameters: YandexImagesParameters }; - yandex_videos: { parameters: YandexVideosParameters }; - youtube: { parameters: YoutubeParameters }; - walmart: { parameters: WalmartParameters }; - walmart_product: { parameters: WalmartProductParameters }; - walmart_product_reviews: { parameters: WalmartProductReviewsParameters }; - home_depot: { parameters: HomeDepotParameters }; - home_depot_product: { parameters: HomeDepotProductParameters }; - linkedin: { parameters: LinkedinParameters }; - linkedin_profile: { parameters: LinkedinProfileParameters }; - duckduckgo: { parameters: DuckduckgoParameters }; - google_play_product: { parameters: GooglePlayProductParameters }; - google_play: { parameters: GooglePlayParameters }; - apple_app_store: { parameters: AppleAppStoreParameters }; - apple_reviews: { parameters: AppleReviewsParameters }; - apple_product: { parameters: AppleProductParameters }; - naver: { parameters: NaverParameters }; - google_lens: { parameters: GoogleLensParameters }; - google_local_services: { parameters: GoogleLocalServicesParameters }; - google_about_this_result: { parameters: GoogleAboutThisResultParameters }; - yelp: { parameters: YelpParameters }; - yelp_reviews: { parameters: YelpReviewsParameters }; -}; diff --git a/src/engines/google.ts b/src/engines/google.ts deleted file mode 100644 index 09fc203..0000000 --- a/src/engines/google.ts +++ /dev/null @@ -1,274 +0,0 @@ -import type { BaseParameters } from "../types.ts"; - -export type GoogleParameters = BaseParameters & { - /** - * Search Query - * Parameter defines the query you want to search. You can use anything that you - * would use in a regular Google search. e.g. `inurl:`, `site:`, `intitle:`. We - * also support advanced search query parameters such as as_dt and as_eq. See the - * [full list](https://serpapi.com/advanced-google-query-parameters) of supported - * advanced search query parameters. - */ - q: string; - - /** - * Location - * Parameter defines from where you want the search to originate. If several - * locations match the location requested, we'll pick the most popular one. Head to - * the [/locations.json API](https://serpapi.com/locations-api) if you need more - * precise control. location and uule parameters can't be used together. Avoid - * utilizing location when setting the location outside the U.S. when using Google - * Shopping and/or Google Product API. - */ - location?: string; - - /** - * Encoded Location - * Parameter is the Google encoded location you want to use for the search. uule - * and location parameters can't be used together. - */ - uule?: string; - - /** - * Google Place ID - * Parameter defines the id (`CID`) of the Google My Business listing you want to - * scrape. Also known as Google Place ID. - */ - ludocid?: string; - - /** - * Additional Google Place ID - * Parameter that you might have to use to force the knowledge graph map view to - * show up. You can find the lsig ID by using our [Local Pack - * API](https://serpapi.com/local-pack) or [Places Results - * API](https://serpapi.com/places-results). - * lsig ID is also available via a redirect Google uses within [Google My - * Business](https://www.google.com/business/). - */ - lsig?: string; - - /** - * Google Knowledge Graph ID - * Parameter defines the id (`KGMID`) of the Google Knowledge Graph listing you - * want to scrape. Also known as Google Knowledge Graph ID. Searches with kgmid - * parameter will return results for the originally encrypted search parameters. - * For some searches, kgmid may override all other parameters except start, and num - * parameters. - */ - kgmid?: string; - - /** - * Google Cached Search Parameters ID - * Parameter defines the cached search parameters of the Google Search you want to - * scrape. Searches with si parameter will return results for the originally - * encrypted search parameters. For some searches, si may override all other - * parameters except start, and num parameters. si can be used to scrape Google - * Knowledge Graph Tabs. - */ - si?: string; - - /** - * Domain - * Parameter defines the Google domain to use. It defaults to `google.com`. Head to - * the [Google domains page](https://serpapi.com/google-domains) for a full list of - * supported Google domains. - */ - google_domain?: string; - - /** - * Country - * Parameter defines the country to use for the Google search. It's a two-letter - * country code. (e.g., `us` for the United States, `uk` for United Kingdom, or - * `fr` for France). Head to the [Google countries - * page](https://serpapi.com/google-countries) for a full list of supported Google - * countries. - */ - gl?: string; - - /** - * Language - * Parameter defines the language to use for the Google search. It's a two-letter - * language code. (e.g., `en` for English, `es` for Spanish, or `fr` for French). - * Head to the [Google languages page](https://serpapi.com/google-languages) for a - * full list of supported Google languages. - */ - hl?: string; - - /** - * Set Multiple Languages - * Parameter defines one or multiple languages to limit the search to. It uses - * `lang_{two-letter language code}` to specify languages and `|` as a delimiter. - * (e.g., `lang_fr|lang_de` will only search French and German pages). Head to the - * [Google lr languages page](https://serpapi.com/google-lr-languages) for a full - * list of supported languages. - */ - lr?: string; - - /** - * as_dt - * Parameter controls whether to include or exclude results from the site named in - * the as_sitesearch parameter. - */ - as_dt?: string; - - /** - * as_epq - * Parameter identifies a phrase that all documents in the search results must - * contain. You can also use the [phrase - * search](https://developers.google.com/custom-search/docs/xml_results#PhraseSearchqt) - * query term to search for a phrase. - */ - as_epq?: string; - - /** - * as_eq - * Parameter identifies a word or phrase that should not appear in any documents in - * the search results. You can also use the [exclude - * query](https://developers.google.com/custom-search/docs/xml_results#Excludeqt) - * term to ensure that a particular word or phrase will not appear in the documents - * in a set of search results. - */ - as_eq?: string; - - /** - * as_lq - * Parameter specifies that all search results should contain a link to a - * particular URL. You can also use the - * [link:](https://developers.google.com/custom-search/docs/xml_results#BackLinksqt) - * query term for this type of query. - */ - as_lq?: string; - - /** - * as_nlo - * Parameter specifies the starting value for a search range. Use as_nlo and as_nhi - * to append an inclusive search range. - */ - as_nlo?: string; - - /** - * as_nhi - * Parameter specifies the ending value for a search range. Use as_nlo and as_nhi - * to append an inclusive search range. - */ - as_nhi?: string; - - /** - * as_oq - * Parameter provides additional search terms to check for in a document, where - * each document in the search results must contain at least one of the additional - * search terms. You can also use the [Boolean - * OR](https://developers.google.com/custom-search/docs/xml_results#BooleanOrqt) - * query term for this type of query. - */ - as_oq?: string; - - /** - * as_q - * Parameter provides search terms to check for in a document. This parameter is - * also commonly used to allow users to specify additional terms to search for - * within a set of search results. - */ - as_q?: string; - - /** - * as_qdr - * Parameter requests search results from a specified time period (quick date - * range). The following values are supported: - * `d[number]`: requests results from the specified number of past days. Example - * for the past 10 days: `as_qdr=d10` - * `w[number]`: requests results from the specified number of past weeks. - * `m[number]`: requests results from the specified number of past months. - * `y[number]`: requests results from the specified number of past years. Example - * for the past year: `as_qdr=y` - */ - as_qdr?: string; - - /** - * as_rq - * Parameter specifies that all search results should be pages that are related to - * the specified URL. The parameter value should be a URL. You can also use the - * [related:](https://developers.google.com/custom-search/docs/xml_results#RelatedLinksqt) - * query term for this type of query. - */ - as_rq?: string; - - /** - * as_sitesearch - * Parameter allows you to specify that all search results should be pages from a - * given site. By setting the as_dt parameter, you can also use it to exclude pages - * from a given site from your search resutls. - */ - as_sitesearch?: string; - - /** - * Advanced Search Parameters - * (to be searched) parameter defines advanced search parameters that aren't - * possible in the regular query field. (e.g., advanced search for patents, dates, - * news, videos, images, apps, or text contents). - */ - tbs?: string; - - /** - * Adult Content Filtering - * Parameter defines the level of filtering for adult content. It can be set to - * `active`, or `off` (default). - */ - safe?: string; - - /** - * Exclude Auto-corrected Results - * Parameter defines the exclusion of results from an auto-corrected query that is - * spelled wrong. It can be set to `1` to exclude these results, or `0` to include - * them (default). - */ - nfpr?: string; - - /** - * Results Filtering - * Parameter defines if the filters for 'Similar Results' and 'Omitted Results' are - * on or off. It can be set to `1` (default) to enable these filters, or `0` to - * disable these filters. - */ - filter?: string; - - /** - * Search Type - * (to be matched) parameter defines the type of search you want to do. - * It can be set to: - * `(no tbm parameter)`: regular Google Search, - * `isch`: [Google Images API](https://serpapi.com/images-results), - * `lcl` - [Google Local API](https://serpapi.com/local-results) - * `vid`: [Google Videos API](https://serpapi.com/videos-results), - * `nws`: [Google News API](https://serpapi.com/news-results), - * `shop`: [Google Shopping API](https://serpapi.com/shopping-results), - * or any other Google service. - */ - tbm?: string; - - /** - * Result Offset - * Parameter defines the result offset. It skips the given number of results. It's - * used for pagination. (e.g., `0` (default) is the first page of results, `10` is - * the 2nd page of results, `20` is the 3rd page of results, etc.). - * Google Local Results only accepts multiples of `20`(e.g. `20` for the second - * page results, `40` for the third page results, etc.) as the start value. - */ - start?: number; - - /** - * Number of Results - * Parameter defines the maximum number of results to return. (e.g., `10` (default) - * returns 10 results, `40` returns 40 results, and `100` returns 100 results). - */ - num?: string; - - /** - * Page Number (images) - * Parameter defines the page number for [Google - * Images](https://serpapi.com/images-results). There are 100 images per page. This - * parameter is equivalent to start (offset) = ijn * 100. This parameter works only - * for [Google Images](https://serpapi.com/images-results) (set tbm to `isch`). - */ - ijn?: string; -}; diff --git a/src/engines/google_about_this_result.ts b/src/engines/google_about_this_result.ts deleted file mode 100644 index 58d308a..0000000 --- a/src/engines/google_about_this_result.ts +++ /dev/null @@ -1,25 +0,0 @@ -import type { BaseParameters } from "../types.ts"; - -export type GoogleAboutThisResultParameters = BaseParameters & { - /** - * Search Query - * Parameter defines the URL of a website which results you what to show. Value - * should be formated in the next order: `About URL` (e.g. `About - * https://www.starbucks.com/`) - */ - q: string; - - /** - * Domain - * Parameter defines the Google domain to use. It defaults to `google.com`. Head to - * the [Google domains page](https://serpapi.com/google-domains) for a full list of - * supported Google domains. - */ - google_domain?: string; - - /** - * Result ID - * Parameter defines unique ID of a website which results you what to show. - */ - ilps: string; -}; diff --git a/src/engines/google_autocomplete.ts b/src/engines/google_autocomplete.ts deleted file mode 100644 index dbeadda..0000000 --- a/src/engines/google_autocomplete.ts +++ /dev/null @@ -1,45 +0,0 @@ -import type { BaseParameters } from "../types.ts"; - -export type GoogleAutocompleteParameters = BaseParameters & { - /** - * Search Query - * Parameter defines the search query. A query that would be used to provide - * completion options. - */ - q: string; - - /** - * Country - * Parameter defines the country to use for the Google search. It's a two-letter - * country code. (e.g., `us` for the United States, `uk` for United Kingdom, or - * `fr` for France) Head to the [Google countries - * page](https://serpapi.com/google-countries) for a full list of supported Google - * countries. - */ - gl?: string; - - /** - * Language - * Parameter defines the language to use for the Google Autocomplete search. It's a - * two-letter language code. (e.g., `en` for English, `es` for Spanish, or `fr` for - * French). Head to the [Google languages - * page](https://serpapi.com/google-languages) for a full list of supported Google - * languages. - */ - hl?: string; - - /** - * Cursor pointer - * Cursor pointer defines the position of cursor for the query provided, position - * starts from 0 which is a case where cursor is placed before the query `|query`. - * If not provided acts as cursor is placed in the end of query `query|`. - */ - cp?: string; - - /** - * Client - * Parameter used to define client for autocomplete. List of supported - * [clients](https://serpapi.com/google-autocomplete-clients). - */ - client?: string; -}; diff --git a/src/engines/google_events.ts b/src/engines/google_events.ts deleted file mode 100644 index 81a72fd..0000000 --- a/src/engines/google_events.ts +++ /dev/null @@ -1,53 +0,0 @@ -import type { BaseParameters } from "../types.ts"; - -export type GoogleEventsParameters = BaseParameters & { - /** - * Search Query - * Parameter defines the query you want to search. To search for events in a - * specific location, just include the location inside your search query (e.g. - * `Events in Austin, TX`). - */ - q: string; - - /** - * Location - * Parameter defines from where you want the search to originate. If several - * locations match the location requested, we'll pick the most popular one. Head to - * the [/locations.json API](https://serpapi.com/locations-api) if you need more - * precise control. location and uule parameters can't be used together. Avoid - * utilizing location when setting the location outside the U.S. when using Google - * Shopping and/or Google Product API. - */ - location?: string; - - /** - * Encoded Location - * Parameter is the Google encoded location you want to use for the search. uule - * and location parameters can't be used together. - */ - uule?: string; - - /** - * Result Offset - * Parameter defines the result offset. It skips the given number of results. It's - * used for pagination. (e.g., `0` (default) is the first page of results, `10` is - * the 2nd page of results, `20` is the 3rd page of results, etc.). - */ - start?: number; - - /** - * Filtering - * Parameter allows the use of different filters. - * `date:today` - Today's Events - * `date:tomorrow` - Tomorrow's Events - * `date:week` - This Week's Events - * `date:today` - Today's Weekend's Events - * `date:next_week` - Next Week's Events - * `date:month` - This Month's Events - * `date:next_month` - Next Month's Events - * `event_type:Virtual-Event` - Online Events - * You can also mix different kinds of filters by separating them with a comma. - * `event_type:Virtual-Event,date:today` Today's Online Events - */ - htichips?: string; -}; diff --git a/src/engines/google_finance.ts b/src/engines/google_finance.ts deleted file mode 100644 index a32ac7d..0000000 --- a/src/engines/google_finance.ts +++ /dev/null @@ -1,20 +0,0 @@ -import type { BaseParameters } from "../types.ts"; - -export type GoogleFinanceParameters = BaseParameters & { - /** - * Search Query - * Parameter defines the query you want to search. It can be a stock, index, mutual - * fund, currency or futures. - */ - q: string; - - /** - * Language - * Parameter defines the language to use for the Google Finance search. It's a - * two-letter language code. (e.g., `en` for English, `es` for Spanish, or `fr` for - * French). Head to the [Google languages - * page](https://serpapi.com/google-languages) for a full list of supported Google - * languages. - */ - hl?: string; -}; diff --git a/src/engines/google_finance_markets.ts b/src/engines/google_finance_markets.ts deleted file mode 100644 index 8954852..0000000 --- a/src/engines/google_finance_markets.ts +++ /dev/null @@ -1,37 +0,0 @@ -import type { BaseParameters } from "../types.ts"; - -export type GoogleFinanceMarketsParameters = BaseParameters & { - /** - * Search Query - * Parameter is used for retrieving different market trends. Available options: - * `indexes` - Market indexes - * `most-active` - Most active - * `gainers` - Gainers - * `losers` - Losers - * `climate-leaders` - Climate leaders - * `cryptocurrencies` - Crypto - * `currencies` - Currencies - */ - trend: string; - - /** - * Language - * Parameter defines the language to use for the Google Finance Markets search. - * It's a two-letter language code. (e.g., `en` for English, `es` for Spanish, or - * `fr` for French). Head to the [Google languages - * page](https://serpapi.com/google-languages) for a full list of supported Google - * languages. - */ - hl?: string; - - /** - * Time Zone - * Parameter is used for expanding market indexes by region and retrieving more - * results. Available options: - * `americas` - Americas - * `europe-middle-east-africa` - Europe, Middle East, and Africa - * `asia-pacific` - Asia Pacific - * Parameter can be used only when trend parameter is set to: `indexes`. - */ - index_market?: string; -}; diff --git a/src/engines/google_immersive_product.ts b/src/engines/google_immersive_product.ts deleted file mode 100644 index 2986e45..0000000 --- a/src/engines/google_immersive_product.ts +++ /dev/null @@ -1,11 +0,0 @@ -import type { BaseParameters } from "../types.ts"; - -export type GoogleImmersiveProductParameters = BaseParameters & { - /** - * Page Token - * Parameter defines the token needed to show more product info in Google immersive - * popup. Token is generated by SerpApi using our [Google Immersive Product - * API](https://serpapi.com/related-questions). - */ - page_token: string; -}; diff --git a/src/engines/google_jobs.ts b/src/engines/google_jobs.ts deleted file mode 100644 index 11021b4..0000000 --- a/src/engines/google_jobs.ts +++ /dev/null @@ -1,84 +0,0 @@ -import type { BaseParameters } from "../types.ts"; - -export type GoogleJobsParameters = BaseParameters & { - /** - * Search Query - * Parameter defines the query you want to search. - */ - q: string; - - /** - * Location - * Parameter defines from where you want the search to originate. If several - * locations match the location requested, we'll pick the most popular one. Head to - * the [/locations.json API](https://serpapi.com/locations-api) if you need more - * precise control. location and uule parameters can't be used together. Avoid - * utilizing location when setting the location outside the U.S. when using Google - * Shopping and/or Google Product API. - */ - location?: string; - - /** - * Encoded Location - * Parameter is the Google encoded location you want to use for the search. uule - * and location parameters can't be used together. - */ - uule?: string; - - /** - * Domain - * Parameter defines the Google domain to use. It defaults to `google.com`. Head to - * the [Google domains page](https://serpapi.com/google-domains) for a full list of - * supported Google domains. - */ - google_domain?: string; - - /** - * Country - * Parameter defines the country to use for the Google search. It's a two-letter - * country code. (e.g., `us` for the United States, `uk` for United Kingdom, or - * `fr` for France) Head to the [Google countries - * page](https://serpapi.com/google-countries) for a full list of supported Google - * countries. - */ - gl?: string; - - /** - * Language - * Parameter defines the language to use for the Google Jobs search. It's a - * two-letter language code. (e.g., `en` for English, `es` for Spanish, or `fr` for - * French). Head to the [Google languages - * page](https://serpapi.com/google-languages) for a full list of supported Google - * languages. - */ - hl?: string; - - /** - * Result Offset - * Parameter defines the result offset. It skips the given number of results. It's - * used for pagination. (e.g., `0` (default) is the first page of results, `10` is - * the 2nd page of results, `20` is the 3rd page of results, etc.). - */ - start?: number; - - /** - * Chips - * Parameter defines additional query conditions. Top of a job search page contains - * elements called chips, its values are extracted in order to be passed to chips - * parameter. E.g. `city:Owg_06VPwoli_nfhBo8LyA==` will return results for New - * York. - */ - chips?: string; - - /** - * Search Radius - * Defines search radius in kilometers. Does not strictly limit the radius. - */ - lrad?: string; - - /** - * Work From Home - * Parameter will filter the results by work from home. - */ - ltype?: string; -}; diff --git a/src/engines/google_jobs_listing.ts b/src/engines/google_jobs_listing.ts deleted file mode 100644 index 538754d..0000000 --- a/src/engines/google_jobs_listing.ts +++ /dev/null @@ -1,10 +0,0 @@ -import type { BaseParameters } from "../types.ts"; - -export type GoogleJobsListingParameters = BaseParameters & { - /** - * Search Query - * Parameter defines the `job_id` string which can be obtained from Google Jobs - * API. - */ - q: string; -}; diff --git a/src/engines/google_lens.ts b/src/engines/google_lens.ts deleted file mode 100644 index ca8a01e..0000000 --- a/src/engines/google_lens.ts +++ /dev/null @@ -1,19 +0,0 @@ -import type { BaseParameters } from "../types.ts"; - -export type GoogleLensParameters = BaseParameters & { - /** - * Image URL - * Parameter defines the URL of an image to perform the Google Lens search. - */ - url: string; - - /** - * Language - * Parameter defines the language to use for the Google Lens search. It's a - * two-letter language code. (e.g., `en` for English, `es` for Spanish, or `fr` for - * French). Head to the [Google languages - * page](https://serpapi.com/google-languages) for a full list of supported Google - * languages. - */ - hl?: string; -}; diff --git a/src/engines/google_local_services.ts b/src/engines/google_local_services.ts deleted file mode 100644 index 0707d67..0000000 --- a/src/engines/google_local_services.ts +++ /dev/null @@ -1,38 +0,0 @@ -import type { BaseParameters } from "../types.ts"; - -export type GoogleLocalServicesParameters = BaseParameters & { - /** - * Search Query - * Parameter defines the service you want to search for. - */ - q: string; - - /** - * Place ID - * Parameter defines the Google ID of a place. To aquire the ID you can eather use - * [Google's place ID - * finder](https://developers.google.com/maps/documentation/javascript/examples/places-placeid-finder), - * or SerpApi's [Google Maps API](https://serpapi.com/google-maps-api). Example ID - * for "New York, NY, USA": `ChIJOwg_06VPwokRYv534QaPC8g`. - */ - place_id: string; - - /** - * Language - * Parameter defines the language to use for the Google Local Services search. It's - * a two-letter language code. (e.g., `en` for English, `es` for Spanish, or `fr` - * for French). Head to the [Google languages - * page](https://serpapi.com/google-languages) for a full list of supported Google - * languages. - */ - hl?: string; - - /** - * Job Type - * Parameter defines the type of a job, or a subcategory of a service you have - * searched for. For example, if you search for "Electrician", you can be more - * specific by adding a job_type (e.g. `job_type=Restore power`). You can find more - * options under: `local_ads[index].services`. - */ - job_type?: string; -}; diff --git a/src/engines/google_maps.ts b/src/engines/google_maps.ts deleted file mode 100644 index 04fcf9b..0000000 --- a/src/engines/google_maps.ts +++ /dev/null @@ -1,85 +0,0 @@ -import type { BaseParameters } from "../types.ts"; - -export type GoogleMapsParameters = BaseParameters & { - /** - * Search Query - * Parameter defines the query you want to search. You can use anything that you - * would use in a regular Google Maps search. The parameter is only required if - * type is set to `search`. - */ - q?: string; - - /** - * GPS Coordinates - * Parameter defines GPS coordinates of location where you want your q (query) to - * be applied. It has to be constructed in the next sequence: - * `@` + `latitude` + `,` + `longitude` + `,` + `zoom` - * This will form a string that looks like this: - * e.g. `@40.7455096,-74.0083012,14z`. The `zoom` parameter is optional but - * recommended for higher precision (it ranges from `3z`, map completely zoomed out - * - to `21z`, map completely zoomed in). The parameter should only be used when - * type is set to `search`. - * Parameter is required when using pagination. - */ - ll?: string; - - /** - * Domain - * Parameter defines the Google domain to use. It defaults to `google.com`. Head to - * the [Google domains page](https://serpapi.com/google-domains) for a full list of - * supported Google domains. - */ - google_domain?: string; - - /** - * Language - * Parameter defines the language to use for the Google Maps search. It's a - * two-letter language code. (e.g., `en` for English, `es` for Spanish, or `fr` for - * French). Head to the [Google languages - * page](https://serpapi.com/google-languages) for a full list of supported Google - * languages. - */ - hl?: string; - - /** - * Data - * Parameter is only required if type is set to `place`. In that case, it defines a - * search for a specific place. It has to be constructed in the next sequence: - * `!4m5!3m4!1s` + `data_id` + `!8m2!3d` + `latitude` + `!4d` + `longitude` - * This will form a string that looks like this: e.g. - * `!4m5!3m4!1s0x89c259b7abdd4769:0xc385876db174521a!8m2!3d40.750231!4d-74.004019`. - * Parameter can also be used to filter the search results. You can visit [Google - * Maps](https://google.com/maps) website, set filters you want and simply copy the - * data value from their URL to SerpApi URL. - */ - data?: string; - - /** - * Place ID - * Parameter defines the unique reference to a place on a Google Map. Place IDs are - * available for most locations, including businesses, landmarks, parks, and - * intersections. You can find the place_id using our [Google Maps - * API](https://serpapi.com/maps-local-results). - * You can read more about Place IDs - * [here](https://developers.google.com/maps/documentation/places/web-service/place-id). - * place_id can be used without any other optional parameter. - */ - place_id?: string; - - /** - * Type of search - * Parameter defines the type of search you want to make. It can be set to: - * `search` - returns a list of results for the set q parameter, - * `place` - returns results for a specific place when data parameter is set - * Parameter is not required when using place_id. - */ - type: string; - - /** - * Result Offset - * Parameter defines the result offset. It skips the given number of results. It's - * used for pagination. (e.g., `0` (default) is the first page of results, `20` is - * the 2nd page of results, `40` is the 3rd page of results, etc.). - */ - start?: number; -}; diff --git a/src/engines/google_maps_photo_meta.ts b/src/engines/google_maps_photo_meta.ts deleted file mode 100644 index 410af56..0000000 --- a/src/engines/google_maps_photo_meta.ts +++ /dev/null @@ -1,10 +0,0 @@ -import type { BaseParameters } from "../types.ts"; - -export type GoogleMapsPhotoMetaParameters = BaseParameters & { - /** - * Data ID - * Parameter defines the Google Maps Photos' data ID. Find the data ID of a photo - * using our [Google Maps Photos API](https://serpapi.com/google-maps-photos-api). - */ - data_id: string; -}; diff --git a/src/engines/google_maps_photos.ts b/src/engines/google_maps_photos.ts deleted file mode 100644 index 1783a7c..0000000 --- a/src/engines/google_maps_photos.ts +++ /dev/null @@ -1,36 +0,0 @@ -import type { BaseParameters } from "../types.ts"; - -export type GoogleMapsPhotosParameters = BaseParameters & { - /** - * Data ID - * Parameter defines the Google Maps data ID. Find the data ID of a place using our - * [Google Maps API](https://serpapi.com/google-maps-api). - */ - data_id: string; - - /** - * Language - * Parameter defines the language to use for the Google Maps Photos search. It's a - * two-letter language code, for example, `en` for English (default), `es` for - * Spanish, or `fr` for French). Head to the [Google languages - * page](https://serpapi.com/google-languages) for a full list of supported Google - * languages. - */ - hl?: string; - - /** - * Category ID - * Parameter defines the ID of a category. You can find the value of an ID inside - * the `categories` array, using our [Google Maps Photos - * API](https://serpapi.com/google-maps-photos-api). The number, and the type of - * categories can vary between places. - */ - category_id?: string; - - /** - * Next Page Token - * Parameter defines the next page token. It is used for retrieving the next page - * results. `20` results are returned per page. - */ - next_page_token?: string; -}; diff --git a/src/engines/google_maps_reviews.ts b/src/engines/google_maps_reviews.ts deleted file mode 100644 index d3a5892..0000000 --- a/src/engines/google_maps_reviews.ts +++ /dev/null @@ -1,45 +0,0 @@ -import type { BaseParameters } from "../types.ts"; - -export type GoogleMapsReviewsParameters = BaseParameters & { - /** - * Data ID - * Parameter defines the Google Maps data ID. Find the data ID by using our [Google - * Maps API](https://serpapi.com/google-maps-api). - */ - data_id: string; - - /** - * Language - * Parameter defines the language to use for the Google Maps Reviews search. It's a - * two-letter language code, for example, `en` for English (default), `es` for - * Spanish, or `fr` for French). Head to the [Google languages - * page](https://serpapi.com/google-languages) for a full list of supported Google - * languages. - */ - hl?: string; - - /** - * Sort By - * Parameter is used for sorting and refining results. Available options: - * `qualityScore` - the most relevant reviews. - * `newestFirst` - the most recent reviews. - * `ratingHigh` - the highest rating reviews. - * `ratingLow` - the lowest rating reviews. - */ - sort_by?: string; - - /** - * Topic ID - * Parameter defines the ID of the topic you want to use for filtering reviews. You - * can access IDs inside our structured JSON response. - */ - topic_id?: string; - - /** - * Next Page Token - * Parameter defines the next page token. It is used for retrieving the next page - * results. - * Usage of start parameter (results offset) has been deprecated by Google. - */ - next_page_token?: string; -}; diff --git a/src/engines/google_play.ts b/src/engines/google_play.ts deleted file mode 100644 index 1b21830..0000000 --- a/src/engines/google_play.ts +++ /dev/null @@ -1,125 +0,0 @@ -import type { BaseParameters } from "../types.ts"; - -export type GooglePlayParameters = BaseParameters & { - /** - * Search Query - * Parameter defines the query you want to search. You can search for apps, games, - * movies or books. - */ - q?: string; - - /** - * Country - * Parameter defines the country to use for the Google Play search. It's a - * two-letter country code. (e.g., `us` (default) for the United States, `uk` for - * United Kingdom, or `fr` for France). You can find the full list of Google Play - * country availability here: [Google Play - * Countries](https://support.google.com/googleplay/answer/2843119?hl=en#zippy=%2Caudiobooks%2Ce-books%2Cpaid-android-apps%2Cmovies-tv%2Cbooks). - * Afterwards, head to the [Google countries - * page](https://serpapi.com/google-countries) page for a two-letter country code. - */ - gl?: string; - - /** - * Language - * Parameter defines the language to use for the Google Play search. It's a - * two-letter language code. (e.g., `en` (default) for English, `es` for Spanish, - * or `fr` for French). Head to the [Google languages - * page](https://serpapi.com/google-languages) for a full list of supported Google - * languages. - */ - hl?: string; - - /** - * Store - * Parameter defines the type of Google Play store. Available options: - * `apps` - Apps store (default) - * `games` - Games store - * `movies` - Movies store - * `books` - Books store - */ - store: string; - - /** - * Apps Category - * Parameter defines the apps and games store category. Head to the Google Play - * store [Apps Categories](https://serpapi.com/google-play-apps-categories) for a - * full list of supported Google Play Apps store categories. - */ - apps_category?: string; - - /** - * Movies Category - * Parameter defines the movies store category. Head to the Google Play store - * [Movies Categories](https://serpapi.com/google-play-movies-categories) for a - * full list of supported Google Play Movies store categories. - */ - movies_category?: string; - - /** - * Books Category - * Parameter defines the books store category. Head to the Google Play store [Books - * Categories](https://serpapi.com/google-play-books-categories) for a full list of - * supported Google Play Books store categories. - */ - books_category?: string; - - /** - * Age - * Parameter defines age subcategory. age works, and should be aplied only in - * combination with next parameter / value pairs: - * `store=apps`, `apps_category=FAMILY`; - * `store=movies`, `movies_category=FAMILY`; - * `store=books`, `books_category=coll_1689` - * It can be set to: - * `AGE_RANGE1` - Ages up to 5 - * `AGE_RANGE2` - Ages 6-8 - * `AGE_RANGE3` - Ages 9-12 - */ - age?: string; - - /** - * Price - * Parameter is used for sorting items by price. It is exclusive to the Google Play - * `Books` store, and it should be used only in combination with the q parameter. - * It can be set to: - * `1` - Free - * `2` - Paid - */ - price?: string; - - /** - * Device - * Parameter defines the device for sorting results. Available options: - * `phone` - Phone device (default) - * `tablet` - Tablet device - * `tv` - TV device - * `chromebook` - Chromebook device - * `watch` - Watch device - * `car` - Car device - */ - store_device?: string; - - /** - * Chart - * Parameter is used for showing top charts. It can return up to `50` results. Each - * store contains different charts which require different values for retrieving - * results. To get the value of a specific chart you can use our Google Play Store - * API JSON output: `chart_options[index].value` (e.g. `chart=topselling_free`). - */ - chart?: string; - - /** - * See more token - * Parameter defines the token used for retrieving all of the results from a - * specific sections. - */ - see_more_token?: string; - - /** - * Next Page Token - * Parameter defines the next page token. It is used for retrieving the next page - * results. - */ - next_page_token?: string; -}; diff --git a/src/engines/google_play_product.ts b/src/engines/google_play_product.ts deleted file mode 100644 index 41c247f..0000000 --- a/src/engines/google_play_product.ts +++ /dev/null @@ -1,69 +0,0 @@ -import type { BaseParameters } from "../types.ts"; - -export type GooglePlayProductParameters = BaseParameters & { - /** - * Product ID - * Parameter defines the ID of a product you want to get the results for. - */ - product_id: string; - - /** - * Country - * Parameter defines the country to use for the Google Play search. It's a - * two-letter country code. (e.g., `us` (default) for the United States, `uk` for - * United Kingdom, or `fr` for France). You can find the full list of Google Play - * country availability here: [Google Play - * Countries](https://support.google.com/googleplay/answer/2843119?hl=en#zippy=%2Caudiobooks%2Ce-books%2Cpaid-android-apps%2Cmovies-tv%2Cbooks). - * Afterwards, head to the [Google countries - * page](https://serpapi.com/google-countries) page for a two-letter country code. - */ - gl?: string; - - /** - * Language - * Parameter defines the language to use for the Google Play search. It's a - * two-letter language code. (e.g., `en` (default) for English, `es` for Spanish, - * or `fr` for French). Head to the [Google languages - * page](https://serpapi.com/google-languages) for a full list of supported Google - * languages. - */ - hl?: string; - - /** - * Store - * Parameter defines the type of Google Play store. There are five types in total: - * `apps` (default), `movies`, `tv`, `books` and `audiobooks` store. - */ - store: string; - - /** - * Season ID - * Parameter defines the ID of a season you want to get the results for. It should - * be used only when store parameter is set to `tv`. e.g. `store=tv`. - */ - season_id?: string; - - /** - * Show All Reviews - * Parameter is used for retriving all reviews of a product. It can be set to - * `true` or `false` (default). - */ - all_reviews?: string; - - /** - * Number of Results - * Parameter defines the maximum number of reviews to return. (e.g., `40` (default) - * returns 40 reviews, `80` returns 80 reviews, and `100` returns 100 reviews). - * Maximum number of reviews you can return per search is `199`. - * It should be used only when all_reviews parameter is set to `true`. - */ - num?: string; - - /** - * Next Page Token - * Parameter defines the next page token. It is used for retrieving the next page - * results. - * It should be used only when all_reviews parameter is set to `true`. - */ - next_page_token?: string; -}; diff --git a/src/engines/google_product.ts b/src/engines/google_product.ts deleted file mode 100644 index 33996c4..0000000 --- a/src/engines/google_product.ts +++ /dev/null @@ -1,116 +0,0 @@ -import type { BaseParameters } from "../types.ts"; - -export type GoogleProductParameters = BaseParameters & { - /** - * Product ID - * Parameter defines the product to get results for. Normally found from shopping - * results for supported products (e.g., - * `https://www.google.com/shopping/product/{product_id}`). - */ - product_id: string; - - /** - * Location - * Parameter defines from where you want the search to originate. If several - * locations match the location requested, we'll pick the most popular one. Head to - * the [/locations.json API](https://serpapi.com/locations-api) if you need more - * precise control. location and uule parameters can't be used together. - */ - location?: string; - - /** - * Encoded Location - * Parameter is the Google encoded location you want to use for the search. uule - * and location parameters can't be used together. - */ - uule?: string; - - /** - * Domain - * Parameter defines the Google domain to use. It defaults to `google.com`. Head to - * the [Google domains page](https://serpapi.com/google-domains) for a full list of - * supported Google domains. - */ - google_domain?: string; - - /** - * Country - * Parameter defines the country to use for the Google search. It's a two-letter - * country code. (e.g., `us` for the United States, `uk` for United Kingdom, or - * `fr` for France) Head to the [Google countries - * page](https://serpapi.com/google-countries) for a full list of supported Google - * countries. - */ - gl?: string; - - /** - * Language - * Parameter defines the language to use for the Google Product search. It's a - * two-letter language code. (e.g., `en` for English, `es` for Spanish, or `fr` for - * French). Head to the [Google languages - * page](https://serpapi.com/google-languages) for a full list of supported Google - * languages. - */ - hl?: string; - - /** - * Result Offset - * Parameter defines the result offset. It skips the given number of results. It's - * used for pagination. (e.g., `0` (default) is the first page of results, `10` is - * the 2nd page of results, `20` is the 3rd page of results, etc.) This parameter - * works only for Google [Online Sellers](https://serpapi.com/online-sellers) and - * [Reviews](https://serpapi.com/reviews-results). - */ - start?: number; - - /** - * Start From Page - * Parameter defines the page number for Google [Online - * Sellers](https://serpapi.com/online-sellers) and - * [Reviews](https://serpapi.com/reviews-results). There are 10 results per page. - * This parameter is equivalent to start (offset) = page * 10. This parameter works - * only for Google [Online Sellers](https://serpapi.com/online-sellers) and - * [Reviews](https://serpapi.com/reviews-results). - */ - page?: string; - - /** - * Offers Results - * Parameter for fetching offers results. Replaces former `sellers=online` results. - * It can be set to `1` or `true` - */ - offers?: string; - - /** - * Fetch Specs Results - * Parameter for fetching specs results. It can be set to `1` or `true` - */ - specs?: string; - - /** - * Fetch Reviews Results - * Parameter for fetching reviews results. It can be set to `1` or `true`. - */ - reviews?: string; - - /** - * Advanced Filter Parameter - * Parameter defines filters and sorting for reviews and offers results. - * Offers filters: - * `freeship:1` Show only products with free shipping - * `ucond:1` Show only used products - * `scoring:p` Sort by base price - * `scoring:tp` Sort by total price - * `scoring:cpd` Sort by current promotion deals (special offers) - * `scoring:mrd` Sort by sellers rating - * Reviews filters: - * `rsort:0` Sort by relevance - * `rsort:1` Sort by date - * `pub:{source}` {source} is where the review originated, e.g. Best Buy - * `rnum:{number}` Number of results (100 is max). `rnum` takes the precedence over - * the `start` and `page` parameters - * `rpt:{number}` Encoded pagination offset (check `serpapi_pagination.next` for - * the value) - */ - filter?: string; -}; diff --git a/src/engines/google_related_questions.ts b/src/engines/google_related_questions.ts deleted file mode 100644 index ea58112..0000000 --- a/src/engines/google_related_questions.ts +++ /dev/null @@ -1,11 +0,0 @@ -import type { BaseParameters } from "../types.ts"; - -export type GoogleRelatedQuestionsParameters = BaseParameters & { - /** - * Next Page Token - * Parameter defines the token needed to show more Q&A pairs that Google generates - * when a question gets clicked. Token is generated by SerpApi using our [Google - * Related Questions API](https://serpapi.com/related-questions). - */ - next_page_token: string; -}; diff --git a/src/engines/google_reverse_image.ts b/src/engines/google_reverse_image.ts deleted file mode 100644 index 7f59308..0000000 --- a/src/engines/google_reverse_image.ts +++ /dev/null @@ -1,78 +0,0 @@ -import type { BaseParameters } from "../types.ts"; - -export type GoogleReverseImageParameters = BaseParameters & { - /** - * Search Query - * Parameter defines the query you want to search. You can also include location in - * a query as location parameter doesn't work for this kind of search. - */ - q?: string; - - /** - * Location - * Parameter defines from where you want the search to originate. If several - * locations match the location requested, we'll pick the most popular one. Head to - * the [/locations.json API](https://serpapi.com/locations-api) if you need more - * precise control. location and uule parameters can't be used together. Avoid - * utilizing location when setting the location outside the U.S. when using Google - * Shopping and/or Google Product API. - */ - location?: string; - - /** - * Encoded Location - * Parameter is the Google encoded location you want to use for the search. uule - * and location parameters can't be used together. - */ - uule?: string; - - /** - * Domain - * Parameter defines the Google domain to use. It defaults to `google.com`. Head to - * the [Google domains page](https://serpapi.com/google-domains) for a full list of - * supported Google domains. - */ - google_domain?: string; - - /** - * Country - * Parameter defines the country to use for the Google search. It's a two-letter - * country code. (e.g., `us` for the United States, `uk` for United Kingdom, or - * `fr` for France) Head to the [Google countries - * page](https://serpapi.com/google-countries) for a full list of supported Google - * countries. - */ - gl?: string; - - /** - * Language - * Parameter defines the language to use for the Google Reverse Image search. It's - * a two-letter language code. (e.g., `en` for English, `es` for Spanish, or `fr` - * for French). Head to the [Google languages - * page](https://serpapi.com/google-languages) for a full list of supported Google - * languages. - */ - hl?: string; - - /** - * Set Multiple Languages - * Parameter defines one or multiple languages to limit the search to. It uses - * `lang_{two-letter language code}` to specify languages and `|` as a delimiter. - * (e.g., `lang_fr|lang_de` will only search French and German pages). - */ - lr?: string; - - /** - * Result Offset - * Parameter defines the result offset. It skips the given number of results. It's - * used for pagination. (e.g., `0` (default) is the first page of results, `10` is - * the 2nd page of results, `20` is the 3rd page of results, etc.). - */ - start?: number; - - /** - * Image url - * Parameter defines URL for an image to perform reverse search. - */ - image_url: string; -}; diff --git a/src/engines/google_scholar.ts b/src/engines/google_scholar.ts deleted file mode 100644 index 55140d0..0000000 --- a/src/engines/google_scholar.ts +++ /dev/null @@ -1,131 +0,0 @@ -import type { BaseParameters } from "../types.ts"; - -export type GoogleScholarParameters = BaseParameters & { - /** - * Search Query - * Parameter defines the query you want to search. You can also use helpers in your - * query such as: `author:`, or `source:`. - * Usage of `cites` parameter makes `q` optional. Usage of `cites` together with - * `q` triggers search within citing articles. - * Usage of `cluster` together with `q` and `cites` parameters is prohibited. Use - * `cluster` parameter only. - */ - q: string; - - /** - * Cited By - * Parameter defines unique ID for an article to trigger Cited By searches. Usage - * of `cites` will bring up a list of citing documents in Google Scholar. Example - * value: `cites=1275980731835430123`. Usage of `cites` and `q` parameters triggers - * search within citing articles. - */ - cites?: string; - - /** - * From Year - * Parameter defines the year from which you want the results to be included. (e.g. - * if you set as_ylo parameter to the year `2018`, the results before that year - * will be omitted.). This parameter can be combined with the as_yhi parameter. - */ - as_ylo?: string; - - /** - * To Year - * Parameter defines the year until which you want the results to be included. - * (e.g. if you set as_yhi parameter to the year `2018`, the results after that - * year will be omitted.). This parameter can be combined with the as_ylo - * parameter. - */ - as_yhi?: string; - - /** - * Sort By Date - * Parameter defines articles added in the last year, sorted by date. It can be set - * to `1` to include only abstracts, or `2` to include everything. The default - * value is `0` which means that the articles are sorted by relevance. - */ - scisbd?: string; - - /** - * Versions Of - * Parameter defines unique ID for an article to trigger All Versions searches. - * Example value: `cluster=1275980731835430123`. Usage of `cluster` together with - * `q` and `cites` parameters is prohibited. Use `cluster` parameter only. - */ - cluster?: string; - - /** - * Language - * Parameter defines the language to use for the Google Scholar search. It's a - * two-letter language code. (e.g., `en` for English, `es` for Spanish, or `fr` for - * French). Head to the [Google languages - * page](https://serpapi.com/google-languages) for a full list of supported Google - * languages. - */ - hl?: string; - - /** - * Set Multiple Languages - * Parameter defines one or multiple languages to limit the search to. It uses - * `lang_{two-letter language code}` to specify languages and `|` as a delimiter. - * (e.g., `lang_fr|lang_de` will only search French and German pages). Head to the - * [Google lr languages](https://serpapi.com/google-lr-languages) for a full list - * of supported languages. - */ - lr?: string; - - /** - * Result Offset - * Parameter defines the result offset. It skips the given number of results. It's - * used for pagination. (e.g., `0` (default) is the first page of results, `10` is - * the 2nd page of results, `20` is the 3rd page of results, etc.). - */ - start?: number; - - /** - * Number of Results - * Parameter defines the maximum number of results to return, limited to 20. (e.g., - * `10` (default) returns 10 results, `20` returns 20 results). - */ - num?: string; - - /** - * Search Type / Filter - * Parameter can be used either as a search type or a filter. - * **As a Filter** (only works when searching articles): - * `0` - exclude patents (default). - * `7` - include patents. - * **As a Search Type**: - * `4` - Select case law (US courts only). This will select all the State and - * Federal courts. - * e.g. `as_sdt=4` - Selects case law (all courts) - * To select specific courts, see the full list of supported [Google Scholar - * courts](https://serpapi.com/google-scholar-courts). - * e.g. `as_sdt=4,33,192` - `4` is the required value and should always be in the - * first position, `33` selects all New York courts and `192` selects Tax Court. - * Values have to be separated by comma (`,`) - */ - as_sdt?: string; - - /** - * Adult Content Filtering - * Parameter defines the level of filtering for adult content. It can be set to - * `active`, or `off` (default). - */ - safe?: string; - - /** - * Results Filtering - * Parameter defines if the filters for 'Similar Results' and 'Omitted Results' are - * on or off. It can be set to `1` (default) to enable these filters, or `0` to - * disable these filters. - */ - filter?: string; - - /** - * Show Citations - * Parameter defines whether you would like to include citations or not. It can be - * set to `1` to exclude these results, or `0` (default) to include them. - */ - as_vis?: string; -}; diff --git a/src/engines/google_scholar_author.ts b/src/engines/google_scholar_author.ts deleted file mode 100644 index f4e4d15..0000000 --- a/src/engines/google_scholar_author.ts +++ /dev/null @@ -1,66 +0,0 @@ -import type { BaseParameters } from "../types.ts"; - -export type GoogleScholarAuthorParameters = BaseParameters & { - /** - * Author ID - * Parameter defines the ID of an author. You can find the ID either by using our - * [Google Scholar Profiles API](https://serpapi.com/google-scholar-profiles-api) - * or by going to the Google Scholar user profile page and getting it from there - * (e.g., `https://scholar.google.com/citations?user={author_id}`). - */ - author_id: string; - - /** - * Language - * Parameter defines the language to use for the Google Scholar Author search. It's - * a two-letter language code. (e.g., `en` for English, `es` for Spanish, or `fr` - * for French). Head to the [Google languages - * page](https://serpapi.com/google-languages) for a full list of supported Google - * languages. - */ - hl?: string; - - /** - * View Section - * Parameter is used for viewing specific parts of a page. It has two options: - * `view_citation` - Select to view - * [citations](https://serpapi.com/google-scholar-author-citation). citation_id is - * required. - * `list_colleagues` - Select to view all - * [co-authors](https://serpapi.com/google-scholar-author-co-authors) - */ - view_op?: string; - - /** - * Sort By - * Parameter is used for sorting and refining articles. Available options: - * `title` - Sorts articles by "Title". - * `pubdate` - Sorts articles by publish "date". - * By default, articles are sorted by the number of citations. - */ - sort?: string; - - /** - * Citation ID - * Parameter is used for retrieving individual article citation. It is a required - * parameter when `view_op=view_citation` is selected. You can access IDs inside - * our structured JSON response. - */ - citation_id?: string; - - /** - * Result Offset - * Parameter defines the result offset. It skips the given number of results. It's - * used for pagination. (e.g., `0` (default) is the first page of results, `20` is - * the 2nd page of results, `40` is the 3rd page of results, etc.). - */ - start?: number; - - /** - * Number of Results - * Parameter defines the number of results to return. (e.g., `20` (default) returns - * 20 results, `40` returns 40 results, etc.). Maximum number of results to return - * is `100`. - */ - num?: string; -}; diff --git a/src/engines/google_scholar_cite.ts b/src/engines/google_scholar_cite.ts deleted file mode 100644 index f7f2dd6..0000000 --- a/src/engines/google_scholar_cite.ts +++ /dev/null @@ -1,11 +0,0 @@ -import type { BaseParameters } from "../types.ts"; - -export type GoogleScholarCiteParameters = BaseParameters & { - /** - * Search Result ID - * Parameter defines the ID of an individual Google Scholar organic search result. - * You can find the ID inside the `result_id` by using our [Google Scholar - * API](https://serpapi.com/google-scholar-api). - */ - q: string; -}; diff --git a/src/engines/google_scholar_profiles.ts b/src/engines/google_scholar_profiles.ts deleted file mode 100644 index f3a62e4..0000000 --- a/src/engines/google_scholar_profiles.ts +++ /dev/null @@ -1,34 +0,0 @@ -import type { BaseParameters } from "../types.ts"; - -export type GoogleScholarProfilesParameters = BaseParameters & { - /** - * Search Query - * Parameter defines the author you want to search for. You can also use helpers in - * your query such as: `label:`. - */ - mauthors: string; - - /** - * Language - * Parameter defines the language to use for the Google Scholar Profiles search. - * It's a two-letter language code. (e.g., `en` for English, `es` for Spanish, or - * `fr` for French). Head to the [Google languages - * page](https://serpapi.com/google-languages) for a full list of supported Google - * languages. - */ - hl?: string; - - /** - * Next Page Token - * Parameter defines the next page token. It is used for retrieving the next page - * results. The parameter has the precedence over before_author parameter. - */ - after_author?: string; - - /** - * Previous Page Token - * Parameter defines the previous page token. It is used for retrieving the - * previous page results. - */ - before_author?: string; -}; diff --git a/src/engines/google_shopping.ts b/src/engines/google_shopping.ts deleted file mode 100644 index f7bc9eb..0000000 --- a/src/engines/google_shopping.ts +++ /dev/null @@ -1,178 +0,0 @@ -import type { BaseParameters } from "../types.ts"; - -export type GoogleShoppingParameters = BaseParameters & { - /** - * Search Query - * Parameter defines the query you want to search. You can use anything that you - * would use in a regular Google Shopping search. e.g. `inurl:`, `site:`, - * `intitle:`. - */ - q: string; - - /** - * Location - * Parameter defines from where you want the search to originate. If several - * locations match the location requested, we'll pick the most popular one. Head to - * the [/locations.json API](https://serpapi.com/locations-api) if you need a more - * precise control. location and uule parameters can't be used together. Avoid - * utilizing location when setting the location outside the U.S. when using Google - * Shopping API. - */ - location?: string; - - /** - * Encoded Location - * Parameter is the Google encoded location you want to use for the search. uule - * and location parameters can't be used together. - */ - uule?: string; - - /** - * Domain - * Parameter defines the Google domain to use. It defaults to `google.com`. Head to - * the [Google domains](https://serpapi.com/google-domains) for a full list of - * supported Google domains. - */ - google_domain?: string; - - /** - * Country - * Parameter defines the country to use for the Google search. It's a two-letter - * country code. (e.g., `us` for the United States, `uk` for United Kingdom, or - * `fr` for France) Head to the [Google - * countries](https://serpapi.com/google-countries) for a full list of supported - * Google countries. - */ - gl?: string; - - /** - * Language - * Parameter defines the language to use for the Google Maps search. It's a - * two-letter language code. (e.g., `en` for English, `es` for Spanish, or `fr` for - * French) Head to the Google languages for a full list of supported [Google - * languages](https://serpapi.com/google-languages). - */ - hl?: string; - - /** - * as_dt - * Parameter controls whether to include or exclude results from the site named in - * the as_sitesearch parameter. - */ - as_dt?: string; - - /** - * as_epq - * Parameter identifies a phrase that all documents in the search results must - * contain. You can also use the [phrase - * search](https://developers.google.com/custom-search/docs/xml_results#PhraseSearchqt) - * query term to search for a phrase. - */ - as_epq?: string; - - /** - * as_eq - * Parameter identifies a word or phrase that should not appear in any documents in - * the search results. You can also use the [exclude - * query](https://developers.google.com/custom-search/docs/xml_results#Excludeqt) - * term to ensure that a particular word or phrase will not appear in the documents - * in a set of search results. - */ - as_eq?: string; - - /** - * as_lq - * Parameter specifies that all search results should contain a link to a - * particular URL. You can also use the - * [link:](https://developers.google.com/custom-search/docs/xml_results#BackLinksqt) - * query term for this type of query. - */ - as_lq?: string; - - /** - * as_nlo - * Parameter specifies the starting value for a search range. Use as_nlo and as_nhi - * to append an inclusive search range. - */ - as_nlo?: string; - - /** - * as_nhi - * Parameter specifies the ending value for a search range. Use as_nlo and as_nhi - * to append an inclusive search range. - */ - as_nhi?: string; - - /** - * as_oq - * Parameter provides additional search terms to check for in a document, where - * each document in the search results must contain at least one of the additional - * search terms. You can also use the [Boolean - * OR](https://developers.google.com/custom-search/docs/xml_results#BooleanOrqt) - * query term for this type of query. - */ - as_oq?: string; - - /** - * as_q - * Parameter provides search terms to check for in a document. This parameter is - * also commonly used to allow users to specify additional terms to search for - * within a set of search results. - */ - as_q?: string; - - /** - * as_qdr - * Parameter requests search results from a specified time period (quick date - * range). The following values are supported: - * `d[number]`: requests results from the specified number of past days. Example - * for the past 10 days: `as_qdr=d10` - * `w[number]`: requests results from the specified number of past weeks. - * `m[number]`: requests results from the specified number of past months. - * `y[number]`: requests results from the specified number of past years. Example - * for the past year: `as_qdr=y` - */ - as_qdr?: string; - - /** - * as_rq - * Parameter specifies that all search results should be pages that are related to - * the specified URL. The parameter value should be a URL. You can also use the - * [related:](https://developers.google.com/custom-search/docs/xml_results#RelatedLinksqt) - * query term for this type of query. - */ - as_rq?: string; - - /** - * as_sitesearch - * Parameter allows you to specify that all search results should be pages from a - * given site. By setting the as_dt parameter, you can also use it to exclude pages - * from a given site from your search resutls. - */ - as_sitesearch?: string; - - /** - * Advanced Search Parameters - * (to be searched) parameter defines advanced search parameters that aren't - * possible in the regular query field. - */ - tbs?: string; - - /** - * Result Offset - * Parameter defines the result offset. It skips the given number of results. It's - * used for pagination. (e.g., `0` (default) is the first page of results, `60` is - * the 2nd page of results, `120` is the 3rd page of results, etc.). - */ - start?: number; - - /** - * Number of Results - * Parameter defines the maximum number of results to return. (e.g., `60` (default) - * returns 60 results, `40` returns 40 results, and `100` (maximum) returns 100 - * results). - * Any number greater than maximum number (`100`) will default to `100`. - * Any number lesser than minimum number (`1`) will default to `60`. - */ - num?: string; -}; diff --git a/src/engines/google_trends.ts b/src/engines/google_trends.ts deleted file mode 100644 index 4a5963e..0000000 --- a/src/engines/google_trends.ts +++ /dev/null @@ -1,128 +0,0 @@ -import type { BaseParameters } from "../types.ts"; - -export type GoogleTrendsParameters = BaseParameters & { - /** - * Search Query - * Parameter defines the query or queries you want to search. You can use anything - * that you would use in a regular Google Trends search. The maximum number of - * queries per search is `5` (this only applies to "Interest over time" and - * "Compared breakdown by region" data_type, other types of data will only accept - * `1` query per search). - * When passing multiple queries you need to use a comma (`,`) to separate them - * (e.g. `coffee,pizza,dark chocolate,/m/027lnzs,bread`). - * Query can be a "Search term" (e.g. `World Cup`, `Eminem`, `iPhone`, etc.) or a - * "Topic" (e.g. `/m/0663v`, `/m/027lnzs`, `/g/11mw8j71m4`, etc.). Queries that are - * "Topics" are encoded. To retrieve these values you can use our [Google Trends - * Autocomplete API](https://serpapi.com/google-trends-autocomplete). - */ - q: string; - - /** - * Location - * Parameter defines the location from where you want the search to originate. It - * defaults to `Worldwide` (activated when the value of geo parameter is not set or - * empty). Head to the [Google Trends - * Locations](https://serpapi.com/google-trends-locations) for a full list of - * supported Google Trends locations. - */ - geo?: string; - - /** - * Region - * Parameter is used for getting more specific results when using "Compared - * breakdown by region" and "Interest by region" data_type charts. Other data_type - * charts do not accept region parameter. The default value depends on the geo - * location that is set. Available options: - * `COUNTRY` - Country - * `REGION` - Subregion - * `DMA` - Metro - * `CITY` - City - * Not all region options will return results for every geo location. - */ - region?: string; - - /** - * Data type - * Parameter defines the type of search you want to do. Available options: - * `TIMESERIES` - [Interest over - * time](https://serpapi.com/google-trends-interest-over-time) (default) - Accepts - * both single and multiple queries per search. - * `GEO_MAP` - [Compared breakdown by - * region](https://serpapi.com/google-trends-compared-breakdown) - Accepts only - * multiple queries per search. - * `GEO_MAP_0` - [Interest by - * region](https://serpapi.com/google-trends-interest-by-region) - Accepts only - * single query per search. - * `RELATED_TOPICS` - [Related - * topics](https://serpapi.com/google-trends-related-topics) - Accepts only single - * query per search. - * `RELATED_QUERIES` - [Related - * queries](https://serpapi.com/google-trends-related-queries) - Accepts only - * single query per search. - */ - data_type?: string; - - /** - * Time Zone - * Parameter is used to define a time zone. The default value is set to `420` - * (Pacific Day Time(PDT): -07:00). Value is shown in minutes and can span from - * `-1439` to `1439`. - * tz can be calculated using the time difference between UTC +0 and desired - * timezone. - * Example: `60 x ((Time in UTC+0 Now) - (Time in PDT Now)) = 420` will give - * results for PDT now. Because the reference point is UTC+0, the positive time - * zones will result in negative tz whereas negative time zones will result in - * positive tz. - * tz parameter also affects timestamps in response. - */ - tz?: string; - - /** - * Category - * Parameter is used to define a search category. The default value is set to `0` - * ("All categories"). Head to the [Google Trends - * Categories](https://serpapi.com/google-trends-categories) for a full list of - * supported Google Trends Categories. - */ - cat?: string; - - /** - * Property - * Parameter is used for sorting results by property. The default property is set - * to `Web Search` (activated when the value of gprop parameter is not set or - * empty). Other available options: - * `images` - Image Search - * `news` - News Search - * `froogle` - Google Shopping - * `youtube` - YouTube Search - */ - gprop?: string; - - /** - * Date - * Parameter is used to define a date. Available options: - * `now 1-H` - Past hour - * `now 4-H` - Past 4 hours - * `now 1-d` - Past day - * `now 7-d` - Past 7 days - * `today 1-m` - Past 30 days - * `today 3-m` - Past 90 days - * `today 12-m` - Past 12 months - * `today 5-y` - Past 5 years - * `all` - 2004 - present - * You can also pass custom values: - * Dates from 2004 to present: `yyyy-mm-dd yyyy-mm-dd` (e.g. `2021-10-15 - * 2022-05-25`) - * Dates with hours within a week range: `yyyy-mm-ddThh yyyy-mm-ddThh` (e.g. - * `2022-05-19T10 2022-05-24T22`). Hours will be calculated depending on the tz - * (time zone) parameter. - */ - date?: string; - - /** - * Show CSV - * Parameter is used for retrieving the CSV results. Set the parameter to `true` to - * retrieve CSV results as an array. - */ - csv?: string; -}; diff --git a/src/engines/google_trends_autocomplete.ts b/src/engines/google_trends_autocomplete.ts deleted file mode 100644 index e383ebb..0000000 --- a/src/engines/google_trends_autocomplete.ts +++ /dev/null @@ -1,21 +0,0 @@ -import type { BaseParameters } from "../types.ts"; - -export type GoogleTrendsAutocompleteParameters = BaseParameters & { - /** - * Search Query - * Parameter defines the query you want to search. You can use anything that you - * would use in a regular Google Trends search. The query is used to retrieve - * suggested searches. - */ - q: string; - - /** - * Language - * Parameter defines the language to use for the Google Trends Autocomplete search. - * It's a two-letter language code. (e.g., `en` for English, `es` for Spanish, or - * `fr` for French). Head to the [Google languages - * page](https://serpapi.com/google-languages) for a full list of supported Google - * languages. - */ - hl?: string; -}; diff --git a/src/engines/home_depot.ts b/src/engines/home_depot.ts deleted file mode 100644 index 8fee3a8..0000000 --- a/src/engines/home_depot.ts +++ /dev/null @@ -1,76 +0,0 @@ -import type { BaseParameters } from "../types.ts"; - -export type HomeDepotParameters = BaseParameters & { - /** - * Search Query - * Parameter defines the search query. You can use anything that you would use in a - * regular The Home Depot search. - */ - q: string; - - /** - * Sorting - * Parameter defines results sorted by diferent options. - * It can be set to: - * `top_sellers`: Top Sellers, - * `price_low_to_high`: Price Low to High, - * `price_high_to_low`: Price High to Low, - * `top_rated`: Top Rated, - * `best_match`: Best Match - */ - hd_sort?: string; - - /** - * Filter tokens - * Used to pass filter tokens divided by comma. [Filter - * tokens](https://serpapi.com/home-depot-filtering) can be obtained from API - * response - */ - hd_filter_tokens?: string; - - /** - * ZIP Code - * ZIP Postal code. To filter the shipping products by a selected area. - */ - delivery_zip?: string; - - /** - * Store ID - * Store Id to filter the products by the specific store only. - */ - store_id?: string; - - /** - * Minimum price - * Defines lower bound for price in USD. - */ - lowerbound?: string; - - /** - * Maximum price - * Defines upper bound for price in USD. - */ - upperbound?: string; - - /** - * Result offset - * Defines offset for products result. A single page contains 24 products. First - * page offset is 0, second -> 24, third -> 48 and so on. - */ - nao?: string; - - /** - * Page Number - * Value is used to get the items on a specific page. (e.g., `1` (default) is the - * first page of results, `2` is the 2nd page of results, `3` is the 3rd page of - * results, etc.). - */ - page?: string; - - /** - * Page Size - * Determines the number of items per page. There are scenarios where Home depot - * overrides the ps value. By default Home depot returns `24` results. - */ - ps?: number; -}; diff --git a/src/engines/home_depot_product.ts b/src/engines/home_depot_product.ts deleted file mode 100644 index ba15863..0000000 --- a/src/engines/home_depot_product.ts +++ /dev/null @@ -1,21 +0,0 @@ -import type { BaseParameters } from "../types.ts"; - -export type HomeDepotProductParameters = BaseParameters & { - /** - * Product ID - * HomeDepot identifier of a product - */ - product_id: string; - - /** - * ZIP Code - * ZIP Postal code. To filter the shipping products by a selected area. - */ - delivery_zip?: string; - - /** - * Store ID - * Store Id to filter the products by the specific store only. - */ - store_id?: string; -}; diff --git a/src/engines/linkedin.ts b/src/engines/linkedin.ts deleted file mode 100644 index b3b17e9..0000000 --- a/src/engines/linkedin.ts +++ /dev/null @@ -1,17 +0,0 @@ -import type { BaseParameters } from "../types.ts"; - -export type LinkedinParameters = BaseParameters & { - /** - * First Name - * LinkedIn profile first name. Used as a part of url eg: - * `https://www.linkedin.com/pub/dir/first_name/last_name` - */ - first_name: string; - - /** - * Last Name - * LinkedIn profile last name. Used as a part of url eg: - * `https://www.linkedin.com/pub/dir/first_name/last_name` - */ - last_name?: string; -}; diff --git a/src/engines/linkedin_profile.ts b/src/engines/linkedin_profile.ts deleted file mode 100644 index ab41226..0000000 --- a/src/engines/linkedin_profile.ts +++ /dev/null @@ -1,10 +0,0 @@ -import type { BaseParameters } from "../types.ts"; - -export type LinkedinProfileParameters = BaseParameters & { - /** - * Profile ID - * LinkedIn profile ID. Can be found in profile url, e.g. profile id for - * https://www.linkedin.com/in/john-doe is john-doe - */ - profile_id: string; -}; diff --git a/src/engines/naver.ts b/src/engines/naver.ts deleted file mode 100644 index cdeb85d..0000000 --- a/src/engines/naver.ts +++ /dev/null @@ -1,52 +0,0 @@ -import type { BaseParameters } from "../types.ts"; - -export type NaverParameters = BaseParameters & { - /** - * Search Query - * Parameter defines the search query. You can use anything that you would use in a - * regular Naver search. (e.g., `'query'`, `NOT`, `OR`, `site:`, `filetype:`, - * `near:`, `ip:`, `loc:`, `feed:` etc.). - */ - query: string; - - /** - * Result Offset - * Parameter controls the offset of the organic results. This parameter defaults to - * 1 (except for the `web`). - * (e.g. The formula for all searches except the web is `start = (page number * - * 10) - 9` - * e.g. Page number 3 ` (3 * 10) - 9 = 21`) - * The formula for the web will be `start = (page number * 15) - 29` - * e.g. Page number 3 ` (3 * 15) - 29 = 16`. - */ - start?: number; - - /** - * Page number - * The page parameter does the `start` parameter math for you! Just define the page - * number you want. Pagination starts from 1. - */ - page?: string; - - /** - * Number of Results - * Parameter defines the maximum number of results to return. `50` (default) - * returns 50 results. Maximum number of results to return is `100`. - * Parameter can only be used with [Naver Images - * API](https://serpapi.com/naver-images-api). - */ - num?: string; - - /** - * The search type - * Parameter defines the Search type. This parameter defaults to `nexearch`. - * Available options: - * `nexearch`: regular Naver Search, - * `web`: [Web organic results](https://serpapi.com/naver-web-organic-results), - * `video`: [Naver video result](https://serpapi.com/naver-video-results), - * `news`: [Naver news results](https://serpapi.com/naver-news-results), - * `image`: [Naver Images Api](https://serpapi.com/naver-images-api). - * . - */ - where?: string; -}; diff --git a/src/engines/walmart.ts b/src/engines/walmart.ts deleted file mode 100644 index 265e4f3..0000000 --- a/src/engines/walmart.ts +++ /dev/null @@ -1,98 +0,0 @@ -import type { BaseParameters } from "../types.ts"; - -export type WalmartParameters = BaseParameters & { - /** - * Search Query - * Parameter defines the search query. You can use anything that you would use in a - * regular Walmart search. Either a `query` or a `cat_id` parameter is required. - */ - query: string; - - /** - * Sort By - * Parameter defines sorting. (e.g. `price_low`, `price_high`, `best_seller`, - * `best_match`, `rating_high`, `new`) - */ - sort?: string; - - /** - * Sort by Relevance - * Parameter enables sort by relevance. Walmart is by default showing results - * sorted by relevance and using the `sort` option. Set to `false` to disable sort - * by Relevance. - */ - soft_sort?: boolean; - - /** - * Grid View - * Parameter defines search results presentation: grid or list. `true` enables Grid - * View (default), `false` enables List View. - */ - grid?: boolean; - - /** - * Category ID - * Category on Walmart Search. (e.g. `0` (default) is all departments, - * `976759_976787` is 'Cookies', etc.). Either a `query` or a `cat_id` parameter is - * required. - */ - cat_id?: string; - - /** - * Filters - * Parameter defines items filtering based on their attributes. The structure is a - * list of `key:value` pairs separated by `||`. The key and value are separated by - * `:` - */ - facet?: string; - - /** - * Store Filter - * Store ID to filter the products by the specific store only. Head to the [Walmart - * Stores Locations](https://serpapi.com/walmart-stores) for a full list of - * supported stores. - * It's possible for the product pricing to differ between stores. - */ - store_id?: string; - - /** - * Min Price - * Lower bound of price range query. - */ - min_price?: string; - - /** - * Max Price - * Upper bound of price range query. - */ - max_price?: string; - - /** - * Exclude Auto-corrected Results - * Activate spelling fix. `true` (default) includes spelling fix, `false` searches - * without spelling fix. - */ - spelling?: boolean; - - /** - * NextDay Delivery - * Show results with NextDay delivery only. Set to `true` to enable or `false` - * (default) to disable - */ - nd_en?: boolean; - - /** - * Page Number - * Value is used to get the items on a specific page. (e.g., `1` (default) is the - * first page of results, `2` is the 2nd page of results, `3` is the 3rd page of - * results, etc.). Maximum page value is `100`. - */ - page?: string; - - /** - * Page Size - * Determines the number of items per page. There are scenarios where Walmart - * overrides the `ps` value. By default Walmart returns `40` results. - */ - ps?: number; -}; diff --git a/src/engines/walmart_product.ts b/src/engines/walmart_product.ts deleted file mode 100644 index 1dbeaac..0000000 --- a/src/engines/walmart_product.ts +++ /dev/null @@ -1,22 +0,0 @@ -import type { BaseParameters } from "../types.ts"; - -export type WalmartProductParameters = BaseParameters & { - /** - * Product ID - * Parameter defines the product to get results for. Normally found from shopping - * results for supported products (e.g., - * `https://www.walmart.com/ip/{product_id}`). You can pass `upc`, `product_id` and - * `us_item_id`. Better to use `us_item_id`. Response for `product_id` returns - * faster because of lack of redirects on Walmart.com - */ - product_id: string; - - /** - * Store Filter - * Store ID to filter the products by the specific store only. Head to the [Walmart - * Stores Locations](https://serpapi.com/walmart-stores) for a full list of - * supported stores. - * It's possible for the product pricing to differ between stores. - */ - store_id?: string; -}; diff --git a/src/engines/walmart_product_reviews.ts b/src/engines/walmart_product_reviews.ts deleted file mode 100644 index 1ba7e63..0000000 --- a/src/engines/walmart_product_reviews.ts +++ /dev/null @@ -1,45 +0,0 @@ -import type { BaseParameters } from "../types.ts"; - -export type WalmartProductReviewsParameters = BaseParameters & { - /** - * Product ID - * Parameter defines the unique product identifier to get reviews of a specific - * product. You need to pass the `us_item_id`. It can be found from either [Organic - * Results API](https://serpapi.com/walmart-organic-results) or [Product API - * results](https://serpapi.com/walmart-product-api). - */ - product_id: string; - - /** - * Filter by Rating - * Parameter is used for filtering reviews by rating. - * It can be set to: - * `1`: 1-star, - * `2`: 2-star, - * `3`: 3-star, - * `4`: 4-star, - * `5`: 5-star. - */ - rating?: string; - - /** - * Sort By - * Parameter is used for sorting reviews. - * It can be set to: - * `relevancy`, - * `helpful`, - * `submission-desc`, - * `submission-asc`, - * `rating-desc`, - * `rating-asc` - */ - sort?: string; - - /** - * Page Number - * Value is used to get the reviews on a specific page. (e.g., `1` (default) is the - * first page of results, `2` is the 2nd page of results, `3` is the 3rd page of - * results, etc.). - */ - page?: string; -}; diff --git a/src/engines/yahoo.ts b/src/engines/yahoo.ts deleted file mode 100644 index 6bf505c..0000000 --- a/src/engines/yahoo.ts +++ /dev/null @@ -1,75 +0,0 @@ -import type { BaseParameters } from "../types.ts"; - -export type YahooParameters = BaseParameters & { - /** - * Search Query - * Parameter defines the search query. You can use anything that you would use in a - * regular Yahoo! search. - */ - p: string; - - /** - * Domain - * Parameter defines the Yahoo! domain to use. It defaults to `search.yahoo.com`. - * If specified domain is allowed, it will be prepended to the domain (e.g., - * `fr.search.yahoo.com`). You can check [a full list of supported Yahoo! - * domains](https://serpapi.com/yahoo-domains). - */ - yahoo_domain?: string; - - /** - * Country - * Parameter defines the country to use for the Yahoo! search. It's a two-letter - * country code. (e.g., `us` for the United States, `uk` for United Kingdom, or - * `fr` for France) Head to the [Yahoo! - * countries](https://serpapi.com/yahoo-vc-countries) for a full list of supported - * Yahoo! countries. - */ - vc?: string; - - /** - * Set Language - * Parameter defines language to limit the search to. It uses `lang_{two-letter - * language code}` to specify languages. (e.g., `vl=lang_fr` will only search - * French). `fl` will be set to `1` if this parameter is used. You can check [a - * full list of supported Yahoo! - * languages](https://serpapi.com/yahoo-vl-languages). - */ - vl?: string; - - /** - * Result Offset - * Parameter defines the result offset. It skips the given number of results. It's - * used for pagination. (e.g., `1` (default) is the first page of results, `11` is - * the 2nd page of results, `21` is the 3rd page of results, etc.). - */ - b?: string; - - /** - * Adult Content Filtering - * Parameter defines the level of filtering for adult content. Strict: `r`, - * Moderate: `i`, Off: `p` - */ - vm?: string; - - /** - * Allowed domains - * Filter results by top-level domains separated by ','. (e.g., `.com,.org`) - */ - vs?: string; - - /** - * File formats - * `all formats` or specific file format like `pdf` or `txt`. You can check [a full - * list of supported Yahoo! file formats](https://serpapi.com/yahoo-vf-formats). - */ - vf?: string; - - /** - * Element Positions - * Parameter is responsible for rendering positions and expansions for some - * elements (e.g., `p:s,v:w,m:trendingdomain_center` to expand Related Trending - * Searches). - */ - fr2?: string; -}; diff --git a/src/engines/yahoo_images.ts b/src/engines/yahoo_images.ts deleted file mode 100644 index 56eb0d3..0000000 --- a/src/engines/yahoo_images.ts +++ /dev/null @@ -1,108 +0,0 @@ -import type { BaseParameters } from "../types.ts"; - -export type YahooImagesParameters = BaseParameters & { - /** - * Search Query - * Parameter defines the search query. You can use anything that you would use in a - * regular Yahoo! Images search. - */ - p: string; - - /** - * Domain - * Parameter defines the Yahoo! domain to use. It defaults to `search.yahoo.com`. - * If specified domain is allowed, it will be prepended to the domain (e.g., - * `fr.search.yahoo.com`). You can check [a full list of supported Yahoo! - * domains](https://serpapi.com/yahoo-domains). - */ - yahoo_domain?: string; - - /** - * Size - * Parameter is used for filtering images by size. It can be set to: - * `small` - Small - * `medium` - Medium - * `large` - Large - * `wallpaper` - Extra Large - */ - imgsz?: string; - - /** - * Color - * Parameter is used for filtering images by color. It can be set to: - * `color` - Color Only - * `bw` - Black & white - * `red` - Red color - * `orange` - Orange color - * `yellow` - Yellow color - * `green` - Green color - * `teal` - Teal color - * `blue` - Blue color - * `purple` - Purple color - * `pink` - Pink color - * `brown` - Brown color - * `black` - Black color - * `gray` - Gray color - * `white` - White color - */ - imgc?: string; - - /** - * Type - * Parameter is used for filtering images by image type. It can be set to: - * `photo` - Photo - * `clipart` - Clipart - * `linedrawing` - Line Drawing - * `gif` - Animated GIF - * `transparent` - Transparent - */ - imgty?: string; - - /** - * Layout - * Parameter is used for filtering images by layout. It can be set to: - * `square` - Square - * `wide` - Wide - * `tall` - Tall - */ - imga?: string; - - /** - * People - * Parameter is used for filtering images by people. It can be set to: - * `face` - Faces Only - * `portrait` - Head & Shoulders - * `nonportrait` - No People - */ - imgf?: string; - - /** - * Time - * Parameter is used for filtering images by time. It can be set to: - * `day` - Past 24 hours - * `week` - Past week - * `month` - Past month - * `year` - Past year - */ - imgt?: string; - - /** - * Usage Rights - * Parameter is used for filtering images by usage rights. It can be set to: - * `cc` - All Creative Commons - * `pd` - Public Domain - * `fsu` - Free to share and use - * `fsuc` - Free to share and use commercially - * `fmsu` - Free to modify, share and use - * `fmsuc` - Free to modify, share, and use commercially - */ - imgl?: string; - - /** - * Result Offset - * Parameter defines the result offset. It skips the given number of results. It's - * used for pagination. (e.g., `1` (default) starts from the first result, `61` - * starts from the 61st result, `121` starts from the 121st result, etc.). - */ - b?: string; -}; diff --git a/src/engines/yahoo_shopping.ts b/src/engines/yahoo_shopping.ts deleted file mode 100644 index 342d296..0000000 --- a/src/engines/yahoo_shopping.ts +++ /dev/null @@ -1,77 +0,0 @@ -import type { BaseParameters } from "../types.ts"; - -export type YahooShoppingParameters = BaseParameters & { - /** - * Search Query - * Parameter defines the search query. You can use anything that you would use in a - * regular Yahoo! shopping search. - */ - p: string; - - /** - * Min Price - * Lower bound of price range query. - */ - min_price?: string; - - /** - * Max Price - * Upper bound of price range query. - */ - max_price?: string; - - /** - * Sort By - * Parameter is used for sorting and refining results. Available options: - * `price` - the costliest items first. - * `relevancy` - the most relevant items first. - * `popularity` - the most popular items first. - * `discountPercentage` - the highest discounted items (by percentage) first. - */ - sort_by?: string; - - /** - * Order By - * Parameter used to sort the query results in a top to bottom style or vice-versa. - * Available options: `ASC` and `DESC`. - */ - order_by?: string; - - /** - * Gender and Age Range - * Gender and Age Range filters on Yahoo! Shopping Search separated by comma (`,`). - * (e.g. `gender_female,age_adult` is 'female' and 'adult', etc.). Can be obtained - * from `filters.gender` and `filters.age-range` in API response. - */ - category_attr_values?: string; - - /** - * Merchants - * Merchants ID separated by comma (`,`). Merchant IDs can be obtained from - * `filters.stores` in API response. (e.g. `3719d8d4-5edd-4817-998a-91f3229e7323,` - * is 'Walmart', etc.) - */ - merchants?: string; - - /** - * Result Offset - * Parameter defines the result offset. It skips the given number of results. It's - * used for pagination. (e.g., `1` (default) is the first page of results, `60` is - * the 2nd page of results, `120` is the 3rd page of results, etc.). - */ - start?: number; - - /** - * Number of Results - * Parameter defines the maximum number of results to return. (e.g., `10` (default) - * returns 10 results, `40` returns 40 results, and `100` returns 100 results). - */ - limit?: number; - - /** - * Page number - * The page parameter does the `start` parameter math for you! Just define the page - * number you want. Pagination starts from 1. - */ - page?: string; -}; diff --git a/src/engines/yahoo_videos.ts b/src/engines/yahoo_videos.ts deleted file mode 100644 index 3d8262b..0000000 --- a/src/engines/yahoo_videos.ts +++ /dev/null @@ -1,70 +0,0 @@ -import type { BaseParameters } from "../types.ts"; - -export type YahooVideosParameters = BaseParameters & { - /** - * Search Query - * Parameter defines the search query. You can use anything that you would use in a - * regular Yahoo! Videos search. - */ - p: string; - - /** - * Domain - * Parameter defines the Yahoo! domain to use. It defaults to `search.yahoo.com`. - * If specified domain is allowed, it will be prepended to the domain (e.g., - * `fr.search.yahoo.com`). You can check [a full list of supported Yahoo! - * domains](https://serpapi.com/yahoo-domains). - */ - yahoo_domain?: string; - - /** - * Length - * Parameter is used for filtering videos by length. It can be set to: - * `short` - Short (less than 5 minutes) - * `medium` - Medium (5-20 minutes) - * `long` - Long (more than 20 minutes) - */ - durs?: string; - - /** - * Date - * Parameter is used for filtering videos by date. It can be set to: - * `day` - Past 24 hours - * `week` - Past week - * `month` - Past month - * `year` - Past year - */ - vage?: string; - - /** - * Resolution - * Parameter is used for filtering videos by resolution. It can be set to: - * `360p` - 360p or higher - * `480p` - 480p or higher - * `720p` - 720p or higher - * `1080p` - 1080p or higher - */ - vres?: string; - - /** - * Source - * Parameter is used for filtering videos by source. It can be set to: - * `youtube` - YouTube - * `dailymotion` - Dailymotion - * `vimeo` - Vimeo - * `mtv` - MTV - * `cbsnews` - CBS - * `foxnews` - Fox - * `cnn` - CNN - * `msn` - MSN - */ - vsite?: string; - - /** - * Result Offset - * Parameter defines the result offset. It skips the given number of results. It's - * used for pagination. (e.g., `1` (default) starts from the first result, `61` - * starts from the 61st result, `121` starts from the 121st result, etc.). - */ - b?: string; -}; diff --git a/src/engines/yandex.ts b/src/engines/yandex.ts deleted file mode 100644 index 87211d4..0000000 --- a/src/engines/yandex.ts +++ /dev/null @@ -1,44 +0,0 @@ -import type { BaseParameters } from "../types.ts"; - -export type YandexParameters = BaseParameters & { - /** - * Search Query - * Parameter defines the search query. You can use anything that you would use in a - * regular Yandex search. - */ - text: string; - - /** - * Domain - * Parameter defines the Yandex domain to use. It defaults to `yandex.com`. Head to - * the [Yandex domains](https://serpapi.com/yandex-domains) for a full list of - * supported Yandex domains. - */ - yandex_domain?: string; - - /** - * Language - * Parameter defines the language to use for the Yandex search. Head to the [Yandex - * languages](https://serpapi.com/yandex-languages) for a full list of supported - * Yandex languages. - */ - lang?: string; - - /** - * Location - * ID of the country or region to search. Determines the rules for ranking - * documents. For example, if we pass the value `11316` in this parameter - * (Novosibirsk region), when generating search results, a formula is used that is - * defined for the Novosibirsk region. Head to the [Yandex - * locations](https://serpapi.com/yandex-locations) for a full list of supported - * Yandex locations. - * Supported only for `yandex.ru` and `yandex.com.tr` domains. - */ - lr?: string; - - /** - * Page number - * Parameter defines page number. Pagination starts from 0. - */ - p?: string; -}; diff --git a/src/engines/yandex_images.ts b/src/engines/yandex_images.ts deleted file mode 100644 index 0e1b7e7..0000000 --- a/src/engines/yandex_images.ts +++ /dev/null @@ -1,124 +0,0 @@ -import type { BaseParameters } from "../types.ts"; - -export type YandexImagesParameters = BaseParameters & { - /** - * Search Query - * Parameter defines the search query. You can use anything that you would use in a - * regular Yandex Images search. - * It can be optional if the url parameter is being used. - */ - text: string; - - /** - * Domain - * Parameter defines the Yandex Images domain to use. It defaults to `yandex.com`. - * Head to the [Yandex domains](https://serpapi.com/yandex-domains) for a full list - * of supported Yandex domains. - */ - yandex_domain?: string; - - /** - * Image Width - * Parameter defines the width of an image. It can only be used in combination with - * height parameter. - */ - width?: string; - - /** - * Image Height - * Parameter defines the height of an image. It can only be used in combination - * with width parameter. - */ - height?: string; - - /** - * File Type - * Parameter is used for filtering images by file type. It can be set to: - * `jpg` - JPG file extension - * `png` - PNG file extension - * `gifan` - GIF file extension - */ - file_type?: string; - - /** - * Color - * Parameter is used for filtering images by color. It can be set to: - * `color` - Color images only - * `gray` - Black and white - * `red` - Red color - * `orange` - Orange color - * `yellow` - Yellow color - * `cyan` - Cyan color - * `green` - Green color - * `blue` - Blue color - * `violet` - Violet color - * `white` - White color - * `black` - Black color - */ - color?: string; - - /** - * Orientation - * Parameter is used for filtering images by orientation. It can be set to: - * `horizontal` - Horizontal - * `vertical` - Vertical - * `square` - Square - */ - orientation?: string; - - /** - * Image Type - * Parameter is used for filtering images by image type. It can be set to: - * `photo` - Photograph - * `clipart` - White background - * `lineart` - Drawings and sketches - * `demotivator` - Demotivator - * `face` - People - */ - image_type?: string; - - /** - * On This Site - * Parameter is used for filtering images by their source. Example value: - * `www.shutterstock.com`. - */ - site?: string; - - /** - * Last 7 Days - * Parameter is used for showing images that appeared in the last 7 days. - */ - recent?: string; - - /** - * Image URL - * Parameter defines the URL for an image to perform the reverse image search. - */ - url?: string; - - /** - * Crop Coordinates - * Parameter is used to crop the image and perform the reverse search on the - * cropped part of the image. E.g. `0.04;0.46;0.27;0.84`. - * These coordinates are formatted in the next order: `left edge;top edge;right - * edge;bottom edge`. The minimum and maximum for each coordinate are `0` and `1`, - * respectively. - */ - crop?: string; - - /** - * Crop ID - * Parameter is used to filter results by a specific section of an image. Parameter - * will only work with images uploaded by Yandex (e.g. - * `https://avatars.mds.yandex.net/rest-of-the-image-url`). - * crop and crop_id parameters should not be used together. - */ - crop_id?: string; - - /** - * Page number - * Parameter defines the page number. Pagination starts from `0`, and it can return - * up to 30 results. - */ - p?: string; -}; diff --git a/src/engines/yandex_videos.ts b/src/engines/yandex_videos.ts deleted file mode 100644 index fce42ac..0000000 --- a/src/engines/yandex_videos.ts +++ /dev/null @@ -1,46 +0,0 @@ -import type { BaseParameters } from "../types.ts"; - -export type YandexVideosParameters = BaseParameters & { - /** - * Search Query - * Parameter defines the search query. You can use anything that you would use in a - * regular Yandex Videos search. - */ - text: string; - - /** - * Domain - * Parameter defines the Yandex Videos domain to use. It defaults to `yandex.com`. - * Head to the [Yandex domains](https://serpapi.com/yandex-domains) for a full list - * of supported Yandex domains. - */ - yandex_domain?: string; - - /** - * Duration - * Parameter is used for filtering videos by duration. It can be set to: - * `short` - Less than 10 minutes - * `medium` - 10-65 minutes - * `long` - More than 65 minutes - */ - duration?: string; - - /** - * HD - * Parameter is used for filtering videos by quality. - */ - hd?: string; - - /** - * Recent - * Parameter is used for showing recent videos. - */ - within?: string; - - /** - * Page number - * Parameter defines the page number. Pagination starts from `0`, and it can return - * up to 30 results. - */ - p?: string; -}; diff --git a/src/engines/yelp.ts b/src/engines/yelp.ts deleted file mode 100644 index 62899a5..0000000 --- a/src/engines/yelp.ts +++ /dev/null @@ -1,86 +0,0 @@ -import type { BaseParameters } from "../types.ts"; - -export type YelpParameters = BaseParameters & { - /** - * Search Query - * Parameter defines the query you want to search. You can use anything that you - * would use in a regular Yelp search. - */ - find_desc?: string; - - /** - * Location - * Parameter defines from where you want the search to originate. You can use any - * location you would use in a regular Yelp search. Following location formats are - * acceptable: - * - 706 Mission St, San Francisco, CA - * - San Francisco, CA - * - San Francisco, CA 94103 - * - 94103 - */ - find_loc: string; - - /** - * Area - * Parameter defines the distance (map radius) or neighborhoods you want to search - * in. You can use our JSON endpoint to fetch values for eather of them. - * Values for distance are accessible through `filters.distance` (e.g. value for - * 'Bird's-eye View' is - * `g:-97.86003112792969,30.21635515266855,-97.65541076660156,30.394199462058317`). - * You can also specify neighborhoods to search in. Values for neighborhoods are - * accessible through `filters.neighborhoods`. - * The value for a single neighborhood is formed in the next order: - * `filters.neighborhoods.value` + `filters.neighborhoods.list[0].value` (e.g. - * `p:TX:Austin::Downtown`). - * You can also set value for multiple neighborhoods: - * `filters.neighborhoods.value` + - * `[filters.neighborhoods.list[0].value,filters.neighborhoods.list[1].value]` - * (e.g. `p:TX:Austin::[Downtown,East_Austin]`). - * Distance and neighborhoods values can't be used together. - */ - l?: string; - - /** - * Domain - * Parameter defines the Yelp domain to use. It defaults to `yelp.com`. Head to the - * [Yelp domains](https://serpapi.com/yelp-domains) for a full list of supported - * Yelp domains. - */ - yelp_domain?: string; - - /** - * Category - * Parameter is used to define a category. It can be used alongside find_desc - * parameter to refine the search. - */ - cflt?: string; - - /** - * Sort By - * Parameter is used for sorting results. Available options: - * `recommended` - Recommended (default) - * `rating` - Highest Rated - * `review_count` - Most Reviewed - */ - sortby?: string; - - /** - * Filters - * Parameter is used for refining results. You can add more filters like 'price', - * 'features', etc. to your search. You can use our JSON endpoint to fetch values. - * Values for filters are accessible through `filters` (e.g. value for filtering - * results by 'No Smoking' is `Smoking.no`). - * You can also use multiple values: - * `filters.features[0].value,filters.features[1].value` (e.g. - * `GoodForKids,DogsAllowed`) - */ - attrs?: string; - - /** - * Result Offset - * Parameter defines the result offset. It skips the given number of results. It's - * used for pagination. (e.g., `0` (default) is the first page of results, `10` is - * the 2nd page of results, `20` is the 3rd page of results, etc.). - */ - start?: number; -}; diff --git a/src/engines/yelp_reviews.ts b/src/engines/yelp_reviews.ts deleted file mode 100644 index 9129ebc..0000000 --- a/src/engines/yelp_reviews.ts +++ /dev/null @@ -1,54 +0,0 @@ -import type { BaseParameters } from "../types.ts"; - -export type YelpReviewsParameters = BaseParameters & { - /** - * Place ID - * Parameter defines the Yelp ID of a place. Each place has two unique IDs (e.g. - * `ED7A7vDdg8yLNKJTSVHHmg` and `arabica-brooklyn`) and you can use either of them - * as a value of the place_id. To extract the IDs of a place you can use our [Yelp - * Search API](https://serpapi.com/yelp-search-api). - */ - place_id: string; - - /** - * Domain - * Parameter defines the Yelp domain to use. It defaults to `yelp.com`. Head to the - * [Yelp domains](yelp-domains) for a full list of supported Yelp domains. - */ - yelp_domain?: string; - - /** - * Language - * Parameter defines the language to use for sorting Yelp Reviews. It's a - * two-letter language code. (e.g., `en` for English, `es` for Spanish, or `fr` for - * French) Head to the Yelp Reviews languages for a full list of supported [Yelp - * Reviews languages](https://serpapi.com/yelp-reviews-languages). - */ - hl?: string; - - /** - * Search Query - * Parameter defines the query you want to use to search through Yelp Reviews. - */ - q?: string; - - /** - * Sort By - * Parameter is used for sorting results. Available options: - * `relevance_desc` - Yelp Sort (default) - * `date_desc` - Newest First - * `date_asc` - Oldest Rated - * `rating_desc` - Highest Rated - * `rating_asc` - Lowest Rated - * `elites_desc` - Elites - */ - sortby?: string; - - /** - * Result Offset - * Parameter defines the result offset. It skips the given number of results. It's - * used for pagination. (e.g., `0` (default) is the first page of results, `10` is - * the 2nd page of results, `20` is the 3rd page of results, etc.). - */ - start?: number; -}; diff --git a/src/engines/youtube.ts b/src/engines/youtube.ts deleted file mode 100644 index a22049a..0000000 --- a/src/engines/youtube.ts +++ /dev/null @@ -1,46 +0,0 @@ -import type { BaseParameters } from "../types.ts"; - -export type YoutubeParameters = BaseParameters & { - /** - * Search Query - * Parameter defines the search query. You can use anything that you would use in a - * regular YouTube search. - */ - search_query: string; - - /** - * Country - * Parameter defines the country to use for the Google search. It's a two-letter - * country code. (e.g., `us` for the United States, `uk` for United Kingdom, or - * `fr` for France) Head to the [Google countries - * page](https://serpapi.com/google-countries) for a full list of supported Google - * countries. - */ - gl?: string; - - /** - * Language - * Parameter defines the language to use for the Youtube search. It's a two-letter - * language code. (e.g., `en` for English, `es` for Spanish, or `fr` for French). - * Head to the [Google languages page](https://serpapi.com/google-languages) for a - * full list of supported Google languages. - */ - hl?: string; - - /** - * Filter - * Parameter can be used for pagination. Youtube uses continous pagination and the - * next page token can be found in the SerpApi JSON response serpapi_pagination -> - * next_page_token and pagination -> next_page_token fields. - * Parameter can also be used to filter the search results: - * by Upload date, you need to set the sp parameter to `CAI%3D` - * by 4K, you need to set the sp parameter to `EgJwAQ%3D%3D` - * ... - * It can also be used for forcing the exact search query spelling by setting the - * sp value to `QgIIAQ%3D%3D`. - * If you are interested in passing other filters, you can visit the - * [YouTube](https://www.youtube.com/) website, set filters you want and simply - * copy sp value from their URL to SerpApi URL. - */ - sp?: string; -}; diff --git a/src/serpapi.ts b/src/serpapi.ts index 184efee..1b66347 100644 --- a/src/serpapi.ts +++ b/src/serpapi.ts @@ -1,4 +1,3 @@ -import { EngineMap } from "./engines/engine_map.ts"; import { InvalidArgumentError } from "./errors.ts"; import { AccountApiParameters, @@ -68,26 +67,24 @@ const SEARCH_ARCHIVE_PATH = `/searches`; * } * }); */ -export function getJson< - E extends keyof EngineMap, ->( +export function getJson( ...args: | [ - parameters: EngineParameters, - callback?: (json: BaseResponse) => void, + parameters: EngineParameters, + callback?: (json: BaseResponse) => void, ] | [ - engine: E | string, - parameters: EngineParameters, - callback?: (json: BaseResponse) => void, + engine: string, + parameters: EngineParameters, + callback?: (json: BaseResponse) => void, ] -): Promise> { +): Promise { if ( typeof args[0] === "string" && typeof args[1] === "object" ) { const [engine, parameters, callback] = args; - const newParameters = { ...parameters, engine } as EngineParameters; + const newParameters = { ...parameters, engine } as EngineParameters; return _getJson(newParameters, callback); } else if ( typeof args[0] === "object" && @@ -100,12 +97,10 @@ export function getJson< } } -async function _getJson< - E extends keyof EngineMap, ->( - parameters: EngineParameters, - callback?: (json: BaseResponse) => void, -): Promise> { +async function _getJson( + parameters: EngineParameters, + callback?: (json: BaseResponse) => void, +): Promise { const key = validateApiKey(parameters.api_key, true); const timeout = validateTimeout(parameters.timeout); const response = await _internals.execute( @@ -117,8 +112,8 @@ async function _getJson< }, timeout, ); - const json = JSON.parse(response) as BaseResponse; - const nextParametersFromResponse = await extractNextParameters(json); + const json = JSON.parse(response) as BaseResponse; + const nextParametersFromResponse = await extractNextParameters(json); if ( // https://github.com/serpapi/public-roadmap/issues/562 // https://github.com/serpapi/public-roadmap/issues/563 @@ -152,17 +147,15 @@ async function _getJson< * // callback * getHtml({ engine: "google", api_key: API_KEY, q: "coffee" }, console.log); */ -export function getHtml< - E extends keyof EngineMap, ->( +export function getHtml( ...args: | [ - parameters: EngineParameters, + parameters: EngineParameters, callback?: (html: string) => void, ] | [ - engine: E | string, - parameters: EngineParameters, + engine: string, + parameters: EngineParameters, callback?: (html: string) => void, ] ): Promise { @@ -171,7 +164,7 @@ export function getHtml< typeof args[1] === "object" ) { const [engine, parameters, callback] = args; - const newParameters = { ...parameters, engine } as EngineParameters; + const newParameters = { ...parameters, engine } as EngineParameters; return _getHtml(newParameters, callback); } else if ( typeof args[0] === "object" && @@ -184,10 +177,8 @@ export function getHtml< } } -async function _getHtml< - E extends keyof EngineMap, ->( - parameters: EngineParameters, +async function _getHtml( + parameters: EngineParameters, callback?: (html: string) => void, ): Promise { const key = validateApiKey(parameters.api_key, true); @@ -227,12 +218,10 @@ async function _getHtml< * // callback * getJsonBySearchId(id, { api_key: API_KEY }, console.log); */ -export async function getJsonBySearchId< - R extends BaseResponse, ->( +export async function getJsonBySearchId( searchId: string, parameters: GetBySearchIdParameters = {}, - callback?: (json: R) => void, + callback?: (json: BaseResponse) => void, ) { const key = validateApiKey(parameters.api_key); const timeout = validateTimeout(parameters.timeout); @@ -244,7 +233,7 @@ export async function getJsonBySearchId< }, timeout, ); - const json = JSON.parse(response) as R; + const json = JSON.parse(response) as BaseResponse; callback?.(json); return json; } diff --git a/src/types.ts b/src/types.ts index 3ce6c47..450a484 100644 --- a/src/types.ts +++ b/src/types.ts @@ -1,5 +1,3 @@ -import { EngineMap } from "./engines/engine_map.ts"; - export type BaseParameters = { /** * Parameter defines the device to use to get the results. It can be set to @@ -43,18 +41,14 @@ export type BaseParameters = { }; export type EngineParameters< - E extends keyof EngineMap, EngineRequired = true, > = - // https://github.com/microsoft/TypeScript/issues/29729 - // deno-lint-ignore ban-types - & (EngineRequired extends true ? { engine: E | (string & {}) } - // deno-lint-ignore ban-types - : { engine?: E | (string & {}) }) - & EngineMap[E]["parameters"] + & (EngineRequired extends true ? { engine: string } + : { engine?: string }) + & BaseParameters & Record; -export type BaseResponse = { +export type BaseResponse = { search_metadata: { id: string; status: "Queued" | "Processing" | "Success"; @@ -65,14 +59,14 @@ export type BaseResponse = { total_time_taken: number; }; search_parameters: Omit< - EngineParameters, + EngineParameters, "api_key" | "no_cache" | "async" | "timeout" >; serpapi_pagination?: { next: string }; pagination?: { next: string }; next?: ( - callback?: (json: BaseResponse) => void, - ) => Promise>; + callback?: (json: BaseResponse) => void, + ) => Promise; // deno-lint-ignore no-explicit-any [key: string]: any; // TODO(seb): use recursive type }; diff --git a/src/utils.ts b/src/utils.ts index fbaaacb..6fb50dc 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -4,7 +4,6 @@ import https from "node:https"; import qs from "node:querystring"; import url from "node:url"; import { RequestTimeoutError } from "./errors.ts"; -import { EngineMap } from "./engines/engine_map.ts"; /** * This `_internals` object is needed to support stubbing/spying of @@ -23,15 +22,15 @@ function getBaseUrl() { return "https://serpapi.com"; } -type NextParameters = { +type NextParameters = { [ K in keyof Omit< - EngineParameters, + EngineParameters, "api_key" | "no_cache" | "async" | "timeout" > ]: string; }; -export function extractNextParameters( +export function extractNextParameters( json: { serpapi_pagination?: { next: string }; pagination?: { next: string }; @@ -46,7 +45,7 @@ export function extractNextParameters( for (const [k, v] of nextUrl.searchParams.entries()) { nextParameters[k] = v; } - return nextParameters as NextParameters; + return nextParameters as NextParameters; } } From e4e9e5bb234abc93652f624b8121a386994fa6c5 Mon Sep 17 00:00:00 2001 From: zyc9012 Date: Thu, 1 Jun 2023 12:41:17 +0800 Subject: [PATCH 40/61] Simplify EngineParameters and BaseResponse --- examples/deno/basic_ts/example.ts | 4 +- examples/deno/pagination_ts/example.ts | 4 +- examples/node/basic_ts_node_14_up/example.ts | 4 +- .../node/pagination_ts_node_14_up/example.ts | 4 +- mod.ts | 1 - src/serpapi.ts | 8 +-- src/types.ts | 69 +------------------ src/utils.ts | 6 +- tests/serpapi_test.ts | 4 +- tests/utils_test.ts | 4 +- 10 files changed, 21 insertions(+), 87 deletions(-) diff --git a/examples/deno/basic_ts/example.ts b/examples/deno/basic_ts/example.ts index 5959c4c..bee7d5e 100644 --- a/examples/deno/basic_ts/example.ts +++ b/examples/deno/basic_ts/example.ts @@ -1,12 +1,12 @@ import { loadSync } from "https://deno.land/std@0.173.0/dotenv/mod.ts"; -import { config, EngineParameters, getJson } from "../../../mod.ts"; +import { config, getJson } from "../../../mod.ts"; const { API_KEY: apiKey } = loadSync(); const params = { engine: "google", q: "Coffee", api_key: apiKey, -} as EngineParameters; +}; // Show result as JSON (async/await) const response1 = await getJson(params); diff --git a/examples/deno/pagination_ts/example.ts b/examples/deno/pagination_ts/example.ts index 5444431..f431dd0 100644 --- a/examples/deno/pagination_ts/example.ts +++ b/examples/deno/pagination_ts/example.ts @@ -1,5 +1,5 @@ import { loadSync } from "https://deno.land/std@0.173.0/dotenv/mod.ts"; -import { config, EngineParameters, getJson } from "../../../mod.ts"; +import { config, getJson } from "../../../mod.ts"; const { API_KEY: apiKey } = loadSync(); @@ -10,7 +10,7 @@ const params = { engine: "google", q: "Coffee", api_key: apiKey, -} satisfies EngineParameters; +}; // Pagination (async/await) let page1 = await getJson(params); diff --git a/examples/node/basic_ts_node_14_up/example.ts b/examples/node/basic_ts_node_14_up/example.ts index 63285c6..41c36cd 100644 --- a/examples/node/basic_ts_node_14_up/example.ts +++ b/examples/node/basic_ts_node_14_up/example.ts @@ -7,7 +7,7 @@ */ import * as Dotenv from "dotenv"; -import { config, EngineParameters, getJson } from "serpapi"; +import { config, getJson } from "serpapi"; Dotenv.config(); const apiKey = process.env.API_KEY; @@ -16,7 +16,7 @@ const params = { engine: "google", q: "Coffee", api_key: apiKey, -} satisfies EngineParameters; +}; // Show result as JSON (async/await) const response1 = await getJson(params); diff --git a/examples/node/pagination_ts_node_14_up/example.ts b/examples/node/pagination_ts_node_14_up/example.ts index 8975558..dad603a 100644 --- a/examples/node/pagination_ts_node_14_up/example.ts +++ b/examples/node/pagination_ts_node_14_up/example.ts @@ -9,7 +9,7 @@ */ import * as Dotenv from "dotenv"; -import { config, EngineParameters, getJson } from "serpapi"; +import { config, getJson } from "serpapi"; Dotenv.config(); const apiKey = process.env.API_KEY; @@ -21,7 +21,7 @@ const params = { engine: "google", q: "Coffee", api_key: apiKey, -} satisfies EngineParameters; +}; // Pagination (async/await) let page1 = await getJson(params); diff --git a/mod.ts b/mod.ts index 5d03d1d..f3ccab8 100644 --- a/mod.ts +++ b/mod.ts @@ -10,7 +10,6 @@ export { export type { AccountApiParameters, AccountInformation, - BaseParameters, BaseResponse, EngineParameters, GetBySearchIdParameters, diff --git a/src/serpapi.ts b/src/serpapi.ts index 1b66347..35a2d01 100644 --- a/src/serpapi.ts +++ b/src/serpapi.ts @@ -75,7 +75,7 @@ export function getJson( ] | [ engine: string, - parameters: EngineParameters, + parameters: EngineParameters, callback?: (json: BaseResponse) => void, ] ): Promise { @@ -87,7 +87,7 @@ export function getJson( const newParameters = { ...parameters, engine } as EngineParameters; return _getJson(newParameters, callback); } else if ( - typeof args[0] === "object" && + typeof args[0] === "object" && typeof args[1] !== "object" && (typeof args[1] === "undefined" || typeof args[1] === "function") ) { const [parameters, callback] = args; @@ -155,7 +155,7 @@ export function getHtml( ] | [ engine: string, - parameters: EngineParameters, + parameters: EngineParameters, callback?: (html: string) => void, ] ): Promise { @@ -167,7 +167,7 @@ export function getHtml( const newParameters = { ...parameters, engine } as EngineParameters; return _getHtml(newParameters, callback); } else if ( - typeof args[0] === "object" && + typeof args[0] === "object" && typeof args[1] !== "object" && (typeof args[1] === "undefined" || typeof args[1] === "function") ) { const [parameters, callback] = args; diff --git a/src/types.ts b/src/types.ts index 450a484..e1d13f6 100644 --- a/src/types.ts +++ b/src/types.ts @@ -1,75 +1,12 @@ -export type BaseParameters = { - /** - * Parameter defines the device to use to get the results. It can be set to - * `desktop` (default) to use a regular browser, `tablet` to use a tablet browser - * (currently using iPads), or `mobile` to use a mobile browser (currently - * using iPhones). - */ - device?: "desktop" | "tablet" | "mobile"; - - /** - * Parameter will force SerpApi to fetch the Google results even if a cached - * version is already present. A cache is served only if the query and all - * parameters are exactly the same. Cache expires after 1h. Cached searches - * are free, and are not counted towards your searches per month. It can be set - * to `false` (default) to allow results from the cache, or `true` to disallow - * results from the cache. `no_cache` and `async` parameters should not be used together. - */ - no_cache?: boolean; - - /** - * Parameter defines the way you want to submit your search to SerpApi. It can - * be set to `false` (default) to open an HTTP connection and keep it open until - * you got your search results, or `true` to just submit your search to SerpApi - * and retrieve them later. In this case, you'll need to use our - * [Searches Archive API](https://serpapi.com/search-archive-api) to retrieve - * your results. `async` and `no_cache` parameters should not be used together. - * `async` should not be used on accounts with - * [Ludicrous Speed](https://serpapi.com/plan) enabled. - */ - async?: boolean; - - /** - * Parameter defines the SerpApi private key to use. - */ - api_key?: string | null; - - /** - * Specify the client-side timeout of the request. In milliseconds. - */ - timeout?: number; -}; - -export type EngineParameters< - EngineRequired = true, -> = - & (EngineRequired extends true ? { engine: string } - : { engine?: string }) - & BaseParameters - & Record; +// deno-lint-ignore no-explicit-any +export type EngineParameters = Record; export type BaseResponse = { - search_metadata: { - id: string; - status: "Queued" | "Processing" | "Success"; - json_endpoint: string; - created_at: string; - processed_at: string; - raw_html_file: string; - total_time_taken: number; - }; - search_parameters: Omit< - EngineParameters, - "api_key" | "no_cache" | "async" | "timeout" - >; - serpapi_pagination?: { next: string }; - pagination?: { next: string }; next?: ( callback?: (json: BaseResponse) => void, ) => Promise; // deno-lint-ignore no-explicit-any - [key: string]: any; // TODO(seb): use recursive type -}; +} & Record; export type GetBySearchIdParameters = { api_key?: string; diff --git a/src/utils.ts b/src/utils.ts index 6fb50dc..7898e7f 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -31,10 +31,8 @@ type NextParameters = { ]: string; }; export function extractNextParameters( - json: { - serpapi_pagination?: { next: string }; - pagination?: { next: string }; - }, + // deno-lint-ignore no-explicit-any + json: any, ) { const nextUrlString = json["serpapi_pagination"]?.["next"] || json["pagination"]?.["next"]; diff --git a/tests/serpapi_test.ts b/tests/serpapi_test.ts index 465af98..83ec447 100644 --- a/tests/serpapi_test.ts +++ b/tests/serpapi_test.ts @@ -293,7 +293,7 @@ describe("getJson", { it("callback", { ignore: !HAS_API_KEY, }, async () => { - const json = await new Promise>((done) => { + const json = await new Promise((done) => { getJson({ engine: "google", q: "Paris", @@ -305,7 +305,7 @@ describe("getJson", { assertExists(json.organic_results); // old API - const json2 = await new Promise>((done) => { + const json2 = await new Promise((done) => { getJson("google", { q: "Paris", api_key: SERPAPI_TEST_KEY, diff --git a/tests/utils_test.ts b/tests/utils_test.ts index c3ce4d7..ee1926f 100644 --- a/tests/utils_test.ts +++ b/tests/utils_test.ts @@ -29,7 +29,7 @@ const BASE_URL = Deno.env.get("ENV_TYPE") === "local" describe("extractNextParameters", () => { it("with serpapi_pagination property", async () => { assertEquals( - await extractNextParameters<"google">({ + await extractNextParameters({ serpapi_pagination: { next: "https://serpapi.com/search.json?device=desktop&engine=google&gl=us&google_domain=google.com&hl=en&location=Austin%2C+Texas%2C+United+States&q=coffee&start=10", @@ -50,7 +50,7 @@ describe("extractNextParameters", () => { it("with pagination property", async () => { assertEquals( - await extractNextParameters<"google_scholar_profiles">( + await extractNextParameters( { pagination: { next: From 29d1a01bf0b8e619199dcf6f4bd742c360d39cfe Mon Sep 17 00:00:00 2001 From: zyc9012 Date: Sun, 4 Jun 2023 15:51:41 +0800 Subject: [PATCH 41/61] Clear up documentation --- CHANGELOG.md | 4 ++++ README.md | 2 +- docs/migrating_from_google_search_results_nodejs.md | 6 ++++-- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d8dd34f..2755c65 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,10 @@ and this project adheres to ### Removed +- Remove all types for engine parameters. SerpApi's + [documentation](https://serpapi.com/search-api) should be the only source of + truth for valid engines and their parameters. + ## [1.1.1] - 2023-02-15 ### Added diff --git a/README.md b/README.md index 88c7889..ceb3895 100644 --- a/README.md +++ b/README.md @@ -74,7 +74,7 @@ import { getJson } from "https://deno.land/x/serpapi/mod.ts"; ## Features -- TypeScript types such as supported parameters and function argument types. +- TypeScript support. - Works out-of-the-box with [Node.js](https://www.npmjs.com/package/serpapi) and [Deno](https://deno.land/x/serpapi). - Promises and async/await support. diff --git a/docs/migrating_from_google_search_results_nodejs.md b/docs/migrating_from_google_search_results_nodejs.md index 0edab2f..a358b85 100644 --- a/docs/migrating_from_google_search_results_nodejs.md +++ b/docs/migrating_from_google_search_results_nodejs.md @@ -66,7 +66,7 @@ migrate over to the `serpapi` npm package. ## Added -- TypeScript types for supported parameters. +- TypeScript support. - First-class Promises support. ```js const json = await getJson({ engine: "google", q: "coffee" }); @@ -77,8 +77,10 @@ migrate over to the `serpapi` npm package. config.api_key = "new_api_key"; config.timeout = 20000; // 20 seconds ``` -- Error classes (`MissingApiKeyError` and `InvalidTimeoutError`). +- Error classes (`MissingApiKeyError`, `InvalidTimeoutError` and + `InvalidArgumentError`). ```js getJson({ engine: "google", api_key: "" }); // Throws `MissingApiKeyError` getAccount({ api_key: API_KEY, timeout: 0 }); // Throws `InvalidTimeoutError` + getJson("google"); // Throws `InvalidArgumentError` ``` From cde2e470ee079022ac0bb50d716e20dc053203ad Mon Sep 17 00:00:00 2001 From: zyc9012 Date: Sun, 4 Jun 2023 16:21:37 +0800 Subject: [PATCH 42/61] Overload getJson and getHtml, and clarify parameter explanation --- src/serpapi.ts | 97 ++++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 87 insertions(+), 10 deletions(-) diff --git a/src/serpapi.ts b/src/serpapi.ts index 35a2d01..5724877 100644 --- a/src/serpapi.ts +++ b/src/serpapi.ts @@ -21,12 +21,10 @@ const SEARCH_PATH = "/search"; const SEARCH_ARCHIVE_PATH = `/searches`; /** - * Get a JSON response based on search parameters. - * - Accepts an optional callback. - * - Get the next page of results by calling the `.next()` method on the returned response object. + * Get JSON response based on search parameters. * - * @param {object} parameters - search query parameters for the engine - * @param {fn=} callback - optional callback + * @param {object} parameters Search query parameters for the engine. Refer to https://serpapi.com/search-api for parameter explanations. + * @param {fn=} callback Optional callback. * @example * // single call (async/await) * const json = await getJson({ engine: "google", api_key: API_KEY, q: "coffee" }); @@ -67,6 +65,63 @@ const SEARCH_ARCHIVE_PATH = `/searches`; * } * }); */ +export function getJson( + parameters: EngineParameters, + callback?: (json: BaseResponse) => void, +): Promise; + +/** + * Get JSON response based on search parameters. + * + * @param {string} engine Engine name. Refer to https://serpapi.com/search-api for valid engines. + * @param {object} parameters Search query parameters for the engine. Refer to https://serpapi.com/search-api for parameter explanations. + * @param {fn=} callback Optional callback. + * @example + * // single call (async/await) + * const json = await getJson("google", { api_key: API_KEY, q: "coffee" }); + * + * // single call (callback) + * getJson("google", { api_key: API_KEY, q: "coffee" }, console.log); + * + * @example + * // pagination (async/await) + * const page1 = await getJson("google", { q: "coffee", start: 15 }); + * const page2 = await page1.next?.(); + * + * @example + * // pagination (callback) + * getJson("google", { q: "coffee", start: 15 }, (page1) => { + * page1.next?.((page2) => { + * console.log(page2); + * }); + * }); + * + * @example + * // pagination loop (async/await) + * const organicResults = []; + * let page = await getJson("google", { api_key: API_KEY, q: "coffee" }); + * while (page) { + * organicResults.push(...page.organic_results); + * if (organicResults.length >= 30) break; + * page = await page.next?.(); + * } + * + * @example + * // pagination loop (callback) + * const organicResults = []; + * getJson("google", { api_key: API_KEY, q: "coffee" }, (page) => { + * organicResults.push(...page.organic_results); + * if (organicResults.length < 30 && page.next) { + * page.next(); + * } + * }); + */ +export function getJson( + engine: string, + parameters: EngineParameters, + callback?: (json: BaseResponse) => void, +): Promise; + export function getJson( ...args: | [ @@ -134,12 +189,28 @@ async function _getJson( } /** - * Get a HTML response based on search parameters. - * - Accepts an optional callback. - * - Responds with a JSON string if the search request hasn't completed. + * Get raw HTML response based on search parameters. * - * @param {object} parameters - search query parameters for the engine - * @param {fn=} callback - optional callback + * @param {object} parameters Search query parameters for the engine. Refer to https://serpapi.com/search-api for parameter explanations. + * @param {fn=} callback Optional callback. + * @example + * // async/await + * const html = await getHtml({ engine: "google", api_key: API_KEY, q: "coffee" }); + * + * // callback + * getHtml({ engine: "google", api_key: API_KEY, q: "coffee" }, console.log); + */ +export function getHtml( + parameters: EngineParameters, + callback?: (html: string) => void, +): Promise; + +/** + * Get raw HTML response based on search parameters. + * + * @param {string} engine Engine name. Refer to https://serpapi.com/search-api for valid engines. + * @param {object} parameters Search query parameters for the engine. Refer to https://serpapi.com/search-api for parameter explanations. + * @param {fn=} callback Optional callback. * @example * // async/await * const html = await getHtml({ engine: "google", api_key: API_KEY, q: "coffee" }); @@ -147,6 +218,12 @@ async function _getJson( * // callback * getHtml({ engine: "google", api_key: API_KEY, q: "coffee" }, console.log); */ +export function getHtml( + engine: string, + parameters: EngineParameters, + callback?: (html: string) => void, +): Promise; + export function getHtml( ...args: | [ From 7b12208ef77280e1b8a6ee8c250f1ce12be398db Mon Sep 17 00:00:00 2001 From: zyc9012 Date: Sun, 4 Jun 2023 16:32:34 +0800 Subject: [PATCH 43/61] Clean up documentation for other APIs --- CHANGELOG.md | 2 +- mod.ts | 3 --- src/serpapi.ts | 51 +++++++++++++++++++++++++------------------------- src/types.ts | 29 ---------------------------- 4 files changed, 26 insertions(+), 59 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2755c65..1ee175a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,7 +23,7 @@ and this project adheres to ### Removed -- Remove all types for engine parameters. SerpApi's +- Remove all types for engine parameters and responses. SerpApi's [documentation](https://serpapi.com/search-api) should be the only source of truth for valid engines and their parameters. diff --git a/mod.ts b/mod.ts index f3ccab8..3eaa3f4 100644 --- a/mod.ts +++ b/mod.ts @@ -9,12 +9,9 @@ export { export type { AccountApiParameters, - AccountInformation, BaseResponse, EngineParameters, GetBySearchIdParameters, - Location, - Locations, LocationsApiParameters, } from "./src/types.ts"; export { diff --git a/src/serpapi.ts b/src/serpapi.ts index 5724877..73a0b37 100644 --- a/src/serpapi.ts +++ b/src/serpapi.ts @@ -1,11 +1,9 @@ import { InvalidArgumentError } from "./errors.ts"; import { AccountApiParameters, - AccountInformation, BaseResponse, EngineParameters, GetBySearchIdParameters, - Locations, LocationsApiParameters, } from "./types.ts"; import { @@ -277,13 +275,12 @@ async function _getHtml( * Get a JSON response given a search ID. * - This search ID can be obtained from the `search_metadata.id` key in the response. * - Typically used together with the `async` parameter. - * - Accepts an optional callback. * - * @param {string} searchId - search ID + * @param {string} searchId Search ID. * @param {object} parameters - * @param {string=} [parameters.api_key] - API key - * @param {number=} [parameters.timeout] - timeout in milliseconds - * @param {fn=} callback - optional callback + * @param {string=} [parameters.api_key] API key. + * @param {number=} [parameters.timeout] Timeout in milliseconds. + * @param {fn=} callback Optional callback. * @example * const response = await getJson({ engine: "google", api_key: API_KEY, async: true, q: "coffee" }); * const { id } = response.search_metadata; @@ -319,14 +316,12 @@ export async function getJsonBySearchId( * Get a HTML response given a search ID. * - This search ID can be obtained from the `search_metadata.id` key in the response. * - Typically used together with the `async` parameter. - * - Accepts an optional callback. - * - Responds with a JSON if the search request hasn't completed. * - * @param {string} searchId - search ID + * @param {string} searchId Search ID. * @param {object} parameters - * @param {string=} [parameters.api_key] - API key - * @param {number=} [parameters.timeout] - timeout in milliseconds - * @param {fn=} callback - optional callback + * @param {string=} [parameters.api_key] API key. + * @param {number=} [parameters.timeout] Timeout in milliseconds. + * @param {fn=} callback Optional callback. * @example * const response = await getJson({ engine: "google", api_key: API_KEY, async: true, q: "coffee" }); * const { id } = response.search_metadata; @@ -359,12 +354,13 @@ export async function getHtmlBySearchId( /** * Get account information of an API key. - * https://serpapi.com/account-api + * + * Refer to https://serpapi.com/account-api for response examples. * * @param {object} parameters - * @param {string=} [parameters.api_key] - API key - * @param {number=} [parameters.timeout] - timeout in milliseconds - * @param {fn=} callback - optional callback + * @param {string=} [parameters.api_key] API key. + * @param {number=} [parameters.timeout] Timeout in milliseconds. + * @param {fn=} callback Optional callback. * @example * // async/await * const info = await getAccount({ api_key: API_KEY }); @@ -374,27 +370,29 @@ export async function getHtmlBySearchId( */ export async function getAccount( parameters: AccountApiParameters = {}, - callback?: (info: AccountInformation) => void, + // deno-lint-ignore no-explicit-any + callback?: (info: any) => void, ) { const key = validateApiKey(parameters.api_key); const timeout = validateTimeout(parameters.timeout); const response = await _internals.execute(ACCOUNT_PATH, { api_key: key, }, timeout); - const info = JSON.parse(response) as AccountInformation; + const info = JSON.parse(response); callback?.(info); return info; } /** * Get supported locations. Does not require an API key. - * https://serpapi.com/locations-api + * + * Refer to https://serpapi.com/locations-api for response examples. * * @param {object} parameters - * @param {string=} [parameters.q] - query for a location - * @param {number=} [parameters.limit] - limit on number of locations returned - * @param {number=} [parameters.timeout] - timeout in milliseconds - * @param {fn=} callback - optional callback + * @param {string=} [parameters.q] Query for a location. + * @param {number=} [parameters.limit] Limit on number of locations returned. + * @param {number=} [parameters.timeout] Timeout in milliseconds. + * @param {fn=} callback Optional callback. * @example * // async/await * const locations = await getLocations({ limit: 3 }); @@ -404,7 +402,8 @@ export async function getAccount( */ export async function getLocations( parameters: LocationsApiParameters = {}, - callback?: (locations: Locations) => void, + // deno-lint-ignore no-explicit-any + callback?: (locations: any) => void, ) { const timeout = validateTimeout(parameters.timeout); const response = await _internals.execute( @@ -412,7 +411,7 @@ export async function getLocations( parameters, timeout, ); - const locations = JSON.parse(response) as Locations; + const locations = JSON.parse(response); callback?.(locations); return locations; } diff --git a/src/types.ts b/src/types.ts index e1d13f6..171b370 100644 --- a/src/types.ts +++ b/src/types.ts @@ -17,37 +17,8 @@ export type AccountApiParameters = { api_key?: string; timeout?: number; }; -export type AccountInformation = { - account_email: string; - account_id: string; - account_rate_limit_per_hour: number; - api_key: string; - extra_credits: number; - last_hour_searches: number; - plan_id: string; - plan_name: string; - plan_searches_left: number; - searches_per_month: number; - this_hour_searches: number; - this_month_usage: number; - total_searches_left: number; -}; - export type LocationsApiParameters = { q?: string; limit?: number; timeout?: number; }; -export type Location = { - canonical_name: string; - country_code: string; - google_id: number; - google_parent_id: number; - gps: [number, number]; - id: string; - keys: string[]; - name: string; - reach: number; - target_type: string; -}; -export type Locations = Location[]; From 75c313754ec3b3f76997ed1da1aa11235e62b8fe Mon Sep 17 00:00:00 2001 From: zyc9012 Date: Sun, 2 Jul 2023 11:51:46 +0800 Subject: [PATCH 44/61] Remove pagination support --- CHANGELOG.md | 1 + README.md | 69 ----- examples/deno/pagination_ts/.env.example | 1 - examples/deno/pagination_ts/README.md | 27 -- examples/deno/pagination_ts/example.ts | 70 ----- .../pagination_js_node_14_up/.env.example | 1 - .../node/pagination_js_node_14_up/README.md | 15 - .../node/pagination_js_node_14_up/example.js | 79 ----- .../pagination_js_node_14_up/package.json | 10 - .../node/pagination_js_node_7_up/.env.example | 1 - .../node/pagination_js_node_7_up/README.md | 15 - .../node/pagination_js_node_7_up/example.js | 81 ----- .../node/pagination_js_node_7_up/package.json | 9 - .../pagination_ts_node_14_up/.env.example | 1 - .../node/pagination_ts_node_14_up/README.md | 15 - .../node/pagination_ts_node_14_up/example.ts | 81 ----- .../pagination_ts_node_14_up/package.json | 14 - .../pagination_ts_node_14_up/tsconfig.json | 10 - smoke_tests/commonjs/commonjs.js | 51 ---- smoke_tests/esm/esm.js | 39 --- src/serpapi.ts | 124 +------- src/types.ts | 8 +- src/utils.ts | 41 --- tests/engines/apple_reviews_test.ts | 204 ------------- tests/engines/baidu_test.ts | 201 ------------ tests/engines/constants.ts | 4 - tests/engines/duckduckgo_test.ts | 204 ------------- tests/engines/ebay_test.ts | 203 ------------- tests/engines/google_maps_test.ts | 212 ------------- tests/engines/google_scholar_profiles_test.ts | 205 ------------- tests/engines/home_depot_test.ts | 285 ------------------ tests/utils_test.ts | 151 +--------- 32 files changed, 20 insertions(+), 2412 deletions(-) delete mode 100644 examples/deno/pagination_ts/.env.example delete mode 100644 examples/deno/pagination_ts/README.md delete mode 100644 examples/deno/pagination_ts/example.ts delete mode 100644 examples/node/pagination_js_node_14_up/.env.example delete mode 100644 examples/node/pagination_js_node_14_up/README.md delete mode 100644 examples/node/pagination_js_node_14_up/example.js delete mode 100644 examples/node/pagination_js_node_14_up/package.json delete mode 100644 examples/node/pagination_js_node_7_up/.env.example delete mode 100644 examples/node/pagination_js_node_7_up/README.md delete mode 100644 examples/node/pagination_js_node_7_up/example.js delete mode 100644 examples/node/pagination_js_node_7_up/package.json delete mode 100644 examples/node/pagination_ts_node_14_up/.env.example delete mode 100644 examples/node/pagination_ts_node_14_up/README.md delete mode 100644 examples/node/pagination_ts_node_14_up/example.ts delete mode 100644 examples/node/pagination_ts_node_14_up/package.json delete mode 100644 examples/node/pagination_ts_node_14_up/tsconfig.json delete mode 100644 tests/engines/apple_reviews_test.ts delete mode 100644 tests/engines/baidu_test.ts delete mode 100644 tests/engines/constants.ts delete mode 100644 tests/engines/duckduckgo_test.ts delete mode 100644 tests/engines/ebay_test.ts delete mode 100644 tests/engines/google_maps_test.ts delete mode 100644 tests/engines/google_scholar_profiles_test.ts delete mode 100644 tests/engines/home_depot_test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 1ee175a..1974c63 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,7 @@ and this project adheres to - Remove all types for engine parameters and responses. SerpApi's [documentation](https://serpapi.com/search-api) should be the only source of truth for valid engines and their parameters. +- Remove pagination support. ## [1.1.1] - 2023-02-15 diff --git a/README.md b/README.md index ceb3895..4f976d4 100644 --- a/README.md +++ b/README.md @@ -80,7 +80,6 @@ import { getJson } from "https://deno.land/x/serpapi/mod.ts"; - Promises and async/await support. - Callbacks support. - [Examples in JavaScript/TypeScript on Node.js/Deno using ESM/CommonJS, and more](https://github.com/serpapi/serpapi-javascript/tree/master/examples). -- [Pagination support](#pagination). - (Planned) More error classes. ## Configuration @@ -103,33 +102,6 @@ await getJson({ engine: "google", q: "coffee" }); // uses the API key defined in await getJson({ engine: "google", api_key: API_KEY_2, q: "coffee" }); // API_KEY_2 will be used ``` -## Pagination - -Search engines handle pagination in several different ways. Some rely on an -"offset" value to return results starting from a specific index, while some -others rely on the typical notion of a "page". These are often combined with a -"size" value to define how many results are returned in a search. - -This module helps you handle pagination easily. After receiving search results -from `getJson`, simply call the `next()` method on the returned object to -retrieve the next page of results. If there is no `next()` method, then either -pagination is not supported for the search engine or there are no more pages to -be retrieved. - -```js -const page1 = await getJson({ engine: "google", q: "coffee", start: 15 }); -const page2 = await page1.next?.(); -``` - -You may pass in the engine's supported pagination parameters as per normal. In -the above example, the first page contains the 15th to the 24th result while the -second page contains the 25th to the 34th result. - -Note that if you set `no_cache` to `true`, all subsequent `next()` calls will -not return cached results. - -Refer to the [`getJson` definition below](#getjson) for more examples. - ## Functions @@ -159,10 +131,6 @@ Refer to the [`getJson` definition below](#getjson) for more examples. Get a JSON response based on search parameters. -- Accepts an optional callback. -- Get the next page of results by calling the `.next()` method on the returned - response object. - #### Parameters - `parameters` @@ -180,43 +148,6 @@ const json = await getJson({ engine: "google", api_key: API_KEY, q: "coffee" }); getJson({ engine: "google", api_key: API_KEY, q: "coffee" }, console.log); ``` -```javascript -// pagination (async/await) -const page1 = await getJson({ engine: "google", q: "coffee", start: 15 }); -const page2 = await page1.next?.(); -``` - -```javascript -// pagination (callback) -getJson({ engine: "google", q: "coffee", start: 15 }, (page1) => { - page1.next?.((page2) => { - console.log(page2); - }); -}); -``` - -```javascript -// pagination loop (async/await) -const organicResults = []; -let page = await getJson({ engine: "google", api_key: API_KEY, q: "coffee" }); -while (page) { - organicResults.push(...page.organic_results); - if (organicResults.length >= 30) break; - page = await page.next?.(); -} -``` - -```javascript -// pagination loop (callback) -const organicResults = []; -getJson({ engine: "google", api_key: API_KEY, q: "coffee" }, (page) => { - organicResults.push(...page.organic_results); - if (organicResults.length < 30 && page.next) { - page.next(); - } -}); -``` - ### getHtml Get a HTML response based on search parameters. diff --git a/examples/deno/pagination_ts/.env.example b/examples/deno/pagination_ts/.env.example deleted file mode 100644 index fefbd78..0000000 --- a/examples/deno/pagination_ts/.env.example +++ /dev/null @@ -1 +0,0 @@ -API_KEY=YOUR_API_KEY \ No newline at end of file diff --git a/examples/deno/pagination_ts/README.md b/examples/deno/pagination_ts/README.md deleted file mode 100644 index 4402df1..0000000 --- a/examples/deno/pagination_ts/README.md +++ /dev/null @@ -1,27 +0,0 @@ -# Deno pagination TypeScript example - -## Usage - -1. Setup environment variables - -- Duplicate `.env.example` and name it `.env`. -- Replace `YOUR_API_KEY` with your SerpApi API key. - -2. Run the example - -``` -deno run example.ts -``` - -## Notes - -- Imports rely on `mod.ts` found in the root folder. -- To import the published module from deno.land, update the import to the - following: - ```ts - import { - config, - getJson, - GoogleParameters, - } from "https://deno.land/x/serpapi/mod.ts"; - ``` diff --git a/examples/deno/pagination_ts/example.ts b/examples/deno/pagination_ts/example.ts deleted file mode 100644 index f431dd0..0000000 --- a/examples/deno/pagination_ts/example.ts +++ /dev/null @@ -1,70 +0,0 @@ -import { loadSync } from "https://deno.land/std@0.173.0/dotenv/mod.ts"; -import { config, getJson } from "../../../mod.ts"; - -const { API_KEY: apiKey } = loadSync(); - -const extractLinks = (results: { link: string }[]) => - results.map((r) => r.link); - -const params = { - engine: "google", - q: "Coffee", - api_key: apiKey, -}; - -// Pagination (async/await) -let page1 = await getJson(params); -console.log( - "First page links", - extractLinks(page1.organic_results), -); -let page2 = await page1.next?.(); -console.log( - "Second page links", - extractLinks(page2?.organic_results), -); - -// Pagination (callback) -getJson(params, (page1) => { - console.log( - "First page links", - extractLinks(page1.organic_results), - ); - page1.next?.((page2) => { - console.log( - "Second page links", - extractLinks(page2.organic_results), - ); - }); -}); - -// Use global config -config.api_key = apiKey; -page1 = await getJson({ engine: "google", q: "Coffee" }); -page2 = await page1.next?.(); -console.log( - "Second page links", - extractLinks(page2?.organic_results), -); - -// Pagination loop (async/await) -let links: string[] = []; -let page; -page = await getJson({ engine: "google", q: "Coffee" }); -while (page) { - links.push(...extractLinks(page.organic_results)); - if (links.length >= 30) break; - page = await page.next?.(); -} -console.log(links); - -// Pagination loop (callback) -links = []; -getJson({ engine: "google", q: "Coffee" }, (page) => { - links.push(...extractLinks(page.organic_results)); - if (links.length < 30 && page.next) { - page.next(); - } else { - console.log(links); - } -}); diff --git a/examples/node/pagination_js_node_14_up/.env.example b/examples/node/pagination_js_node_14_up/.env.example deleted file mode 100644 index fefbd78..0000000 --- a/examples/node/pagination_js_node_14_up/.env.example +++ /dev/null @@ -1 +0,0 @@ -API_KEY=YOUR_API_KEY \ No newline at end of file diff --git a/examples/node/pagination_js_node_14_up/README.md b/examples/node/pagination_js_node_14_up/README.md deleted file mode 100644 index d49b924..0000000 --- a/examples/node/pagination_js_node_14_up/README.md +++ /dev/null @@ -1,15 +0,0 @@ -# Pagination JavaScript example for Node.js 14 and newer - -## Usage - -1. Setup environment variables - -- Duplicate `.env.example` and name it `.env`. -- Replace `YOUR_API_KEY` with your SerpApi API key. - -2. Install dependencies and run example - -```bash -npm i -npm start -``` diff --git a/examples/node/pagination_js_node_14_up/example.js b/examples/node/pagination_js_node_14_up/example.js deleted file mode 100644 index 3d1ffc6..0000000 --- a/examples/node/pagination_js_node_14_up/example.js +++ /dev/null @@ -1,79 +0,0 @@ -/** - * Example works for Node.js 14 and newer. - * - Uses ESM imports which is supported from Node.js 13.2.0. - * https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Statements/import#browser_compatibility - * - Uses top-level await which is supported from Node.js 14.8.0. - * https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Operators/await#browser_compatibility - * - Uses optional chaining which is supported from Node.js 14.0.0. - * https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Operators/Optional_chaining#browser_compatibility - */ - -import * as Dotenv from "dotenv"; -import { config, getJson } from "serpapi"; - -Dotenv.config(); -const apiKey = process.env.API_KEY; - -const extractLinks = (results) => results.map((r) => r.link); - -const params = { - engine: "google", - q: "Coffee", - api_key: apiKey, -}; - -// Pagination (async/await) -let page1 = await getJson(params); -console.log( - "First page links", - extractLinks(page1.organic_results), -); -let page2 = await page1.next?.(); -console.log( - "Second page links", - extractLinks(page2?.organic_results), -); - -// Pagination (callback) -getJson(params, (page1) => { - console.log( - "First page links", - extractLinks(page1.organic_results), - ); - page1.next?.((page2) => { - console.log( - "Second page links", - extractLinks(page2.organic_results), - ); - }); -}); - -// Use global config -config.api_key = apiKey; -page1 = await getJson({ engine: "google", q: "Coffee" }); -page2 = await page1.next?.(); -console.log( - "Second page links", - extractLinks(page2?.organic_results), -); - -// Pagination loop (async/await) -let links = []; -let page = await getJson({ engine: "google", q: "Coffee" }); -while (page) { - links.push(...extractLinks(page.organic_results)); - if (links.length >= 30) break; - page = await page.next?.(); -} -console.log(links); - -// Pagination loop (callback) -links = []; -getJson({ engine: "google", q: "Coffee" }, (page) => { - links.push(...extractLinks(page.organic_results)); - if (links.length < 30 && page.next) { - page.next(); - } else { - console.log(links); - } -}); diff --git a/examples/node/pagination_js_node_14_up/package.json b/examples/node/pagination_js_node_14_up/package.json deleted file mode 100644 index 055dcb9..0000000 --- a/examples/node/pagination_js_node_14_up/package.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "type": "module", - "dependencies": { - "dotenv": "*", - "serpapi": "*" - }, - "scripts": { - "start": "node example.js" - } -} diff --git a/examples/node/pagination_js_node_7_up/.env.example b/examples/node/pagination_js_node_7_up/.env.example deleted file mode 100644 index fefbd78..0000000 --- a/examples/node/pagination_js_node_7_up/.env.example +++ /dev/null @@ -1 +0,0 @@ -API_KEY=YOUR_API_KEY \ No newline at end of file diff --git a/examples/node/pagination_js_node_7_up/README.md b/examples/node/pagination_js_node_7_up/README.md deleted file mode 100644 index 4542a22..0000000 --- a/examples/node/pagination_js_node_7_up/README.md +++ /dev/null @@ -1,15 +0,0 @@ -# Pagination JavaScript example for Node.js 7 and newer - -## Usage - -1. Setup environment variables - -- Duplicate `.env.example` and name it `.env`. -- Replace `YOUR_API_KEY` with your SerpApi API key. - -2. Install dependencies and run example - -```bash -npm i -npm start -``` diff --git a/examples/node/pagination_js_node_7_up/example.js b/examples/node/pagination_js_node_7_up/example.js deleted file mode 100644 index e183cab..0000000 --- a/examples/node/pagination_js_node_7_up/example.js +++ /dev/null @@ -1,81 +0,0 @@ -/** - * Example works for Node.js 7 and newer. - */ - -const Dotenv = require("dotenv"); -const { config, getJson } = require("serpapi"); - -Dotenv.config(); -const apiKey = process.env.API_KEY; - -const extractLinks = (results) => results.map((r) => r.link); - -const run = async () => { - const params = { - engine: "google", - q: "Coffee", - api_key: apiKey, - }; - - // Pagination (async/await) - let page1 = await getJson(params); - console.log( - "First page links", - extractLinks(page1.organic_results) - ); - if (page1.next) { - let page2 = await page1.next(); - console.log( - "Second page links", - extractLinks(page2.organic_results) - ); - } - - // Pagination (callback) - getJson(params, (page1) => { - console.log( - "First page links", - extractLinks(page1.organic_results) - ); - if (page1.next) { - page1.next((page2) => { - console.log( - "Second page links", - extractLinks(page2.organic_results) - ); - }); - } - }); - - // Use global config - config.api_key = apiKey; - page1 = await getJson({ engine: "google", q: "Coffee" }); - page2 = await page1.next?.(); - console.log( - "Second page links", - extractLinks(page2?.organic_results), - ); - - // Pagination loop (async/await) - let links = []; - let page = await getJson({ engine: "google", q: "Coffee" }); - while (page) { - links.push(...extractLinks(page.organic_results)); - if (links.length >= 30) break; - page = page.next ? await page.next(): undefined; - } - console.log(links); - - // Pagination loop (callback) - links = []; - getJson({ engine: "google", q: "Coffee" }, (page) => { - links.push(...extractLinks(page.organic_results)); - if (links.length < 30 && page.next) { - page.next(); - } else { - console.log(links); - } - }); -}; - -run(); diff --git a/examples/node/pagination_js_node_7_up/package.json b/examples/node/pagination_js_node_7_up/package.json deleted file mode 100644 index f7ceb03..0000000 --- a/examples/node/pagination_js_node_7_up/package.json +++ /dev/null @@ -1,9 +0,0 @@ -{ - "dependencies": { - "dotenv": "*", - "serpapi": "*" - }, - "scripts": { - "start": "node example.js" - } -} diff --git a/examples/node/pagination_ts_node_14_up/.env.example b/examples/node/pagination_ts_node_14_up/.env.example deleted file mode 100644 index fefbd78..0000000 --- a/examples/node/pagination_ts_node_14_up/.env.example +++ /dev/null @@ -1 +0,0 @@ -API_KEY=YOUR_API_KEY \ No newline at end of file diff --git a/examples/node/pagination_ts_node_14_up/README.md b/examples/node/pagination_ts_node_14_up/README.md deleted file mode 100644 index 2f6cb33..0000000 --- a/examples/node/pagination_ts_node_14_up/README.md +++ /dev/null @@ -1,15 +0,0 @@ -# Pagination TypeScript example for Node.js 14 and newer - -## Usage - -1. Setup environment variables - -- Duplicate `.env.example` and name it `.env`. -- Replace `YOUR_API_KEY` with your SerpApi API key. - -2. Install dependencies and run example - -```bash -npm i -npm start -``` diff --git a/examples/node/pagination_ts_node_14_up/example.ts b/examples/node/pagination_ts_node_14_up/example.ts deleted file mode 100644 index dad603a..0000000 --- a/examples/node/pagination_ts_node_14_up/example.ts +++ /dev/null @@ -1,81 +0,0 @@ -/** - * Example works for Node.js 14 and newer. - * - Uses ESM imports which is supported from Node.js 13.2.0. - * https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Statements/import#browser_compatibility - * - Uses top-level await which is supported from Node.js 14.8.0. - * https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Operators/await#browser_compatibility - * - Uses optional chaining which is supported from Node.js 14.0.0. - * https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Operators/Optional_chaining#browser_compatibility - */ - -import * as Dotenv from "dotenv"; -import { config, getJson } from "serpapi"; - -Dotenv.config(); -const apiKey = process.env.API_KEY; - -const extractLinks = (results: { link: string }[]) => - results.map((r) => r.link); - -const params = { - engine: "google", - q: "Coffee", - api_key: apiKey, -}; - -// Pagination (async/await) -let page1 = await getJson(params); -console.log( - "First page links", - extractLinks(page1.organic_results), -); -let page2 = await page1.next?.(); -console.log( - "Second page links", - extractLinks(page2?.organic_results), -); - -// Pagination (callback) -getJson(params, (page1) => { - console.log( - "First page links", - extractLinks(page1.organic_results), - ); - page1.next?.((page2) => { - console.log( - "Second page links", - extractLinks(page2.organic_results), - ); - }); -}); - -// Use global config -config.api_key = apiKey; -page1 = await getJson({ engine: "google", q: "Coffee" }); -page2 = await page1.next?.(); -console.log( - "Second page links", - extractLinks(page2?.organic_results), -); - -// Pagination loop (async/await) -let links: string[] = []; -let page; -page = await getJson({ engine: "google", q: "Coffee" }); -while (page) { - links.push(...extractLinks(page.organic_results)); - if (links.length >= 30) break; - page = await page.next?.(); -} -console.log(links); - -// Pagination loop (callback) -links = []; -getJson({ engine: "google", q: "Coffee" }, (page) => { - links.push(...extractLinks(page.organic_results)); - if (links.length < 30 && page.next) { - page.next(); - } else { - console.log(links); - } -}); diff --git a/examples/node/pagination_ts_node_14_up/package.json b/examples/node/pagination_ts_node_14_up/package.json deleted file mode 100644 index e1ef349..0000000 --- a/examples/node/pagination_ts_node_14_up/package.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "type": "module", - "dependencies": { - "dotenv": "*", - "serpapi": "*" - }, - "devDependencies": { - "@types/node": "*", - "typescript": "*" - }, - "scripts": { - "start": "npx ts-node example.ts" - } -} diff --git a/examples/node/pagination_ts_node_14_up/tsconfig.json b/examples/node/pagination_ts_node_14_up/tsconfig.json deleted file mode 100644 index 8ce0fc8..0000000 --- a/examples/node/pagination_ts_node_14_up/tsconfig.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "compilerOptions": { - "module": "ESNext", - "target": "ESNext", - "moduleResolution": "Node" - }, - "ts-node": { - "esm": true - } -} diff --git a/smoke_tests/commonjs/commonjs.js b/smoke_tests/commonjs/commonjs.js index b5d286f..46c4ff2 100644 --- a/smoke_tests/commonjs/commonjs.js +++ b/smoke_tests/commonjs/commonjs.js @@ -31,21 +31,12 @@ const run = async () => { const page1 = await getJson(Object.assign({ engine: "google" }, params)); searchId = page1["search_metadata"]["id"]; if (!page1["organic_results"]) throw new Error("No organic results"); - if (page1.next) { - const page2 = await page1.next(); - if (!page2["organic_results"]) throw new Error("No organic results"); - } } { console.log("getJson callback"); getJson(Object.assign({ engine: "google" }, params), (page1) => { if (!page1["organic_results"]) throw new Error("No organic results"); - if (page1.next) { - page1.next((page2) => { - if (!page2["organic_results"]) throw new Error("No organic results"); - }); - } }); } @@ -54,10 +45,6 @@ const run = async () => { config.api_key = apiKey; const page1 = await getJson({ engine: "google", q: "Coffee" }); if (!page1["organic_results"]) throw new Error("No organic results"); - if (page1.next) { - const page2 = await page1.next(); - if (!page2["organic_results"]) throw new Error("No organic results"); - } } { @@ -65,21 +52,12 @@ const run = async () => { const page1 = await getJson("google", params); searchId = page1["search_metadata"]["id"]; if (!page1["organic_results"]) throw new Error("No organic results"); - if (page1.next) { - const page2 = await page1.next(); - if (!page2["organic_results"]) throw new Error("No organic results"); - } } { console.log("getJson (old API) callback"); getJson("google", params, (page1) => { if (!page1["organic_results"]) throw new Error("No organic results"); - if (page1.next) { - page1.next((page2) => { - if (!page2["organic_results"]) throw new Error("No organic results"); - }); - } }); } @@ -88,35 +66,6 @@ const run = async () => { config.api_key = apiKey; const page1 = await getJson("google", { q: "Coffee" }); if (!page1["organic_results"]) throw new Error("No organic results"); - if (page1.next) { - const page2 = await page1.next(); - if (!page2["organic_results"]) throw new Error("No organic results"); - } - } - - { - console.log("getJson pagination loop (async/await)"); - const links = []; - let page = await getJson("google", params); - while (page) { - links.push(...page.organic_results.map((r) => r.link)); - if (links.length >= 30) break; - page = page.next ? await page.next() : undefined; - } - if (links.length < 30) throw new Error("Incorrect number of links"); - } - - { - console.log("getJson pagination loop (callback)"); - const links = []; - getJson("google", params, (page) => { - links.push(...page.organic_results.map((r) => r.link)); - if (links.length < 30 && page.next) { - page.next(); - } else { - if (links.length < 30) throw new Error("Incorrect number of links"); - } - }); } { diff --git a/smoke_tests/esm/esm.js b/smoke_tests/esm/esm.js index d6bc0d6..ad4bee2 100644 --- a/smoke_tests/esm/esm.js +++ b/smoke_tests/esm/esm.js @@ -34,17 +34,12 @@ let searchId; const page1 = await getJson(Object.assign({ engine: "google" }, params)); searchId = page1["search_metadata"]["id"]; if (!page1["organic_results"]) throw new Error("No organic results"); - const page2 = await page1.next?.(); - if (!page2["organic_results"]) throw new Error("No organic results"); } { console.log("getJson callback"); getJson(Object.assign({ engine: "google" }, params), (page1) => { if (!page1["organic_results"]) throw new Error("No organic results"); - page1.next?.((page2) => { - if (!page2["organic_results"]) throw new Error("No organic results"); - }); }); } @@ -53,8 +48,6 @@ let searchId; config.api_key = apiKey; const page1 = await getJson({ engine: "google", q: "Coffee" }); if (!page1["organic_results"]) throw new Error("No organic results"); - const page2 = await page1.next?.(); - if (!page2["organic_results"]) throw new Error("No organic results"); } { @@ -62,17 +55,12 @@ let searchId; const page1 = await getJson("google", params); searchId = page1["search_metadata"]["id"]; if (!page1["organic_results"]) throw new Error("No organic results"); - const page2 = await page1.next?.(); - if (!page2["organic_results"]) throw new Error("No organic results"); } { console.log("getJson (old API) callback"); getJson("google", params, (page1) => { if (!page1["organic_results"]) throw new Error("No organic results"); - page1.next?.((page2) => { - if (!page2["organic_results"]) throw new Error("No organic results"); - }); }); } @@ -81,33 +69,6 @@ let searchId; config.api_key = apiKey; const page1 = await getJson("google", { q: "Coffee" }); if (!page1["organic_results"]) throw new Error("No organic results"); - const page2 = await page1.next?.(); - if (!page2["organic_results"]) throw new Error("No organic results"); -} - -{ - console.log("getJson pagination loop (async/await)"); - const links = []; - let page = await getJson("google", params); - while (page) { - links.push(...page.organic_results.map((r) => r.link)); - if (links.length >= 30) break; - page = await page.next?.(); - } - if (links.length < 30) throw new Error("Incorrect number of links"); -} - -{ - console.log("getJson pagination loop (callback)"); - const links = []; - getJson("google", params, (page) => { - links.push(...page.organic_results.map((r) => r.link)); - if (links.length < 30 && page.next) { - page.next(); - } else { - if (links.length < 30) throw new Error("Incorrect number of links"); - } - }); } { diff --git a/src/serpapi.ts b/src/serpapi.ts index 73a0b37..9422ea7 100644 --- a/src/serpapi.ts +++ b/src/serpapi.ts @@ -6,11 +6,7 @@ import { GetBySearchIdParameters, LocationsApiParameters, } from "./types.ts"; -import { - _internals, - extractNextParameters, - haveParametersChanged, -} from "./utils.ts"; +import { _internals } from "./utils.ts"; import { validateApiKey, validateTimeout } from "./validators.ts"; const ACCOUNT_PATH = "/account"; @@ -29,39 +25,6 @@ const SEARCH_ARCHIVE_PATH = `/searches`; * * // single call (callback) * getJson({ engine: "google", api_key: API_KEY, q: "coffee" }, console.log); - * - * @example - * // pagination (async/await) - * const page1 = await getJson({ engine: "google", q: "coffee", start: 15 }); - * const page2 = await page1.next?.(); - * - * @example - * // pagination (callback) - * getJson({ engine: "google", q: "coffee", start: 15 }, (page1) => { - * page1.next?.((page2) => { - * console.log(page2); - * }); - * }); - * - * @example - * // pagination loop (async/await) - * const organicResults = []; - * let page = await getJson({ engine: "google", api_key: API_KEY, q: "coffee" }); - * while (page) { - * organicResults.push(...page.organic_results); - * if (organicResults.length >= 30) break; - * page = await page.next?.(); - * } - * - * @example - * // pagination loop (callback) - * const organicResults = []; - * getJson({ engine: "google", api_key: API_KEY, q: "coffee" }, (page) => { - * organicResults.push(...page.organic_results); - * if (organicResults.length < 30 && page.next) { - * page.next(); - * } - * }); */ export function getJson( parameters: EngineParameters, @@ -80,39 +43,6 @@ export function getJson( * * // single call (callback) * getJson("google", { api_key: API_KEY, q: "coffee" }, console.log); - * - * @example - * // pagination (async/await) - * const page1 = await getJson("google", { q: "coffee", start: 15 }); - * const page2 = await page1.next?.(); - * - * @example - * // pagination (callback) - * getJson("google", { q: "coffee", start: 15 }, (page1) => { - * page1.next?.((page2) => { - * console.log(page2); - * }); - * }); - * - * @example - * // pagination loop (async/await) - * const organicResults = []; - * let page = await getJson("google", { api_key: API_KEY, q: "coffee" }); - * while (page) { - * organicResults.push(...page.organic_results); - * if (organicResults.length >= 30) break; - * page = await page.next?.(); - * } - * - * @example - * // pagination loop (callback) - * const organicResults = []; - * getJson("google", { api_key: API_KEY, q: "coffee" }, (page) => { - * organicResults.push(...page.organic_results); - * if (organicResults.length < 30 && page.next) { - * page.next(); - * } - * }); */ export function getJson( engine: string, @@ -122,25 +52,20 @@ export function getJson( export function getJson( ...args: - | [ - parameters: EngineParameters, - callback?: (json: BaseResponse) => void, - ] + | [parameters: EngineParameters, callback?: (json: BaseResponse) => void] | [ engine: string, parameters: EngineParameters, callback?: (json: BaseResponse) => void, ] ): Promise { - if ( - typeof args[0] === "string" && - typeof args[1] === "object" - ) { + if (typeof args[0] === "string" && typeof args[1] === "object") { const [engine, parameters, callback] = args; const newParameters = { ...parameters, engine } as EngineParameters; return _getJson(newParameters, callback); } else if ( - typeof args[0] === "object" && typeof args[1] !== "object" && + typeof args[0] === "object" && + typeof args[1] !== "object" && (typeof args[1] === "undefined" || typeof args[1] === "function") ) { const [parameters, callback] = args; @@ -166,22 +91,6 @@ async function _getJson( timeout, ); const json = JSON.parse(response) as BaseResponse; - const nextParametersFromResponse = await extractNextParameters(json); - if ( - // https://github.com/serpapi/public-roadmap/issues/562 - // https://github.com/serpapi/public-roadmap/issues/563 - parameters.engine !== "yahoo_shopping" && - nextParametersFromResponse - ) { - const nextParameters = { ...parameters, ...nextParametersFromResponse }; - if (haveParametersChanged(parameters, nextParameters)) { - json.next = (innerCallback = callback) => - getJson( - nextParameters, - innerCallback, - ); - } - } callback?.(json); return json; } @@ -224,25 +133,20 @@ export function getHtml( export function getHtml( ...args: - | [ - parameters: EngineParameters, - callback?: (html: string) => void, - ] + | [parameters: EngineParameters, callback?: (html: string) => void] | [ engine: string, parameters: EngineParameters, callback?: (html: string) => void, ] ): Promise { - if ( - typeof args[0] === "string" && - typeof args[1] === "object" - ) { + if (typeof args[0] === "string" && typeof args[1] === "object") { const [engine, parameters, callback] = args; const newParameters = { ...parameters, engine } as EngineParameters; return _getHtml(newParameters, callback); } else if ( - typeof args[0] === "object" && typeof args[1] !== "object" && + typeof args[0] === "object" && + typeof args[1] !== "object" && (typeof args[1] === "undefined" || typeof args[1] === "function") ) { const [parameters, callback] = args; @@ -375,9 +279,13 @@ export async function getAccount( ) { const key = validateApiKey(parameters.api_key); const timeout = validateTimeout(parameters.timeout); - const response = await _internals.execute(ACCOUNT_PATH, { - api_key: key, - }, timeout); + const response = await _internals.execute( + ACCOUNT_PATH, + { + api_key: key, + }, + timeout, + ); const info = JSON.parse(response); callback?.(info); return info; diff --git a/src/types.ts b/src/types.ts index 171b370..c83e861 100644 --- a/src/types.ts +++ b/src/types.ts @@ -1,12 +1,8 @@ // deno-lint-ignore no-explicit-any export type EngineParameters = Record; -export type BaseResponse = { - next?: ( - callback?: (json: BaseResponse) => void, - ) => Promise; - // deno-lint-ignore no-explicit-any -} & Record; +// deno-lint-ignore no-explicit-any +export type BaseResponse = Record; export type GetBySearchIdParameters = { api_key?: string; diff --git a/src/utils.ts b/src/utils.ts index 7898e7f..ef49a2a 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -1,8 +1,6 @@ -import type { EngineParameters } from "./types.ts"; import { version } from "../version.ts"; import https from "node:https"; import qs from "node:querystring"; -import url from "node:url"; import { RequestTimeoutError } from "./errors.ts"; /** @@ -22,45 +20,6 @@ function getBaseUrl() { return "https://serpapi.com"; } -type NextParameters = { - [ - K in keyof Omit< - EngineParameters, - "api_key" | "no_cache" | "async" | "timeout" - > - ]: string; -}; -export function extractNextParameters( - // deno-lint-ignore no-explicit-any - json: any, -) { - const nextUrlString = json["serpapi_pagination"]?.["next"] || - json["pagination"]?.["next"]; - - if (nextUrlString) { - const nextUrl = new url.URL(nextUrlString); - const nextParameters: Record = {}; - for (const [k, v] of nextUrl.searchParams.entries()) { - nextParameters[k] = v; - } - return nextParameters as NextParameters; - } -} - -export function haveParametersChanged( - parameters: Record, - nextParameters: Record, -) { - const keys = [ - ...Object.keys(parameters), - ...Object.keys(nextParameters), - ]; - const uniqueKeys = new Set(keys); - return [...uniqueKeys].some((key) => - `${parameters[key]}` !== `${nextParameters[key]}` // string comparison - ); -} - export function getSource() { const moduleSource = `serpapi@${version}`; if (typeof Deno == "object") { diff --git a/tests/engines/apple_reviews_test.ts b/tests/engines/apple_reviews_test.ts deleted file mode 100644 index 0f8660e..0000000 --- a/tests/engines/apple_reviews_test.ts +++ /dev/null @@ -1,204 +0,0 @@ -// deno-lint-ignore-file no-explicit-any -import { loadSync } from "https://deno.land/std@0.170.0/dotenv/mod.ts"; -import { - afterAll, - afterEach, - beforeAll, - beforeEach, - describe, - it, -} from "https://deno.land/std@0.170.0/testing/bdd.ts"; -import { spy, Stub, stub } from "https://deno.land/std@0.170.0/testing/mock.ts"; -import { - assert, - assertEquals, - assertExists, - assertMatch, -} from "https://deno.land/std@0.170.0/testing/asserts.ts"; -import { _internals } from "../../src/utils.ts"; -import { config, getJson } from "../../mod.ts"; -import { - MSG_ASSERT_HAS_LAST_PAGE, - MSG_ASSERT_HAS_MULTIPLE_PAGES, - MSG_ASSERT_HAS_NON_FIRST_PAGE_RESULT, -} from "./constants.ts"; - -loadSync({ export: true }); -const SERPAPI_TEST_KEY = Deno.env.get("SERPAPI_TEST_KEY") ?? ""; -const HAS_API_KEY = SERPAPI_TEST_KEY.length > 0; -const BASE_URL = Deno.env.get("ENV_TYPE") === "local" - ? "http://localhost:3000" - : "https://serpapi.com"; - -describe("apple_reviews", { - sanitizeOps: false, // TODO(seb): look into how we can avoid setting these to false - sanitizeResources: false, -}, () => { - let urlStub: Stub; - const engine = "apple_reviews"; - const productId = "534220544"; - - beforeAll(() => { - urlStub = stub(_internals, "getBaseUrl", () => BASE_URL); - }); - - beforeEach(() => { - config.api_key = SERPAPI_TEST_KEY; - }); - - afterEach(() => { - config.api_key = null; - }); - - afterAll(() => { - urlStub.restore(); - }); - - it("getJson pagination", { - ignore: !HAS_API_KEY, - }, async (t) => { - await t.step("async/await", async () => { - const ids: number[] = []; - let page; - page = await getJson({ engine, product_id: productId }); - while (page) { - ids.push(...page.reviews.map((r: any) => r.id)); - if (ids.length >= 50) break; - page = await page.next?.(); - } - assert(new Set(ids).size > 25, MSG_ASSERT_HAS_MULTIPLE_PAGES); - }); - - await t.step("callback", async () => { - const ids: number[] = []; - await new Promise((done) => { - getJson({ engine, product_id: productId }, (page) => { - ids.push(...page.reviews.map((r: any) => r.id)); - if (ids.length < 50 && page.next) { - page.next(); - } else { - assert(new Set(ids).size > 25, MSG_ASSERT_HAS_MULTIPLE_PAGES); - done(); - } - }); - }); - }); - }); - - it("getJson pagination keeps original parameter keys", { - ignore: !HAS_API_KEY, - }, async (t) => { - const executeSpy = spy(_internals, "execute"); - config.api_key = null; - - await t.step("async/await", async () => { - const page1 = await getJson({ - engine, - api_key: SERPAPI_TEST_KEY, - no_cache: false, - product_id: productId, - }); - assertMatch(executeSpy.calls[0].args[1].api_key as string, /[a-z0-9]+/); - assertEquals(executeSpy.calls[0].args[1].no_cache, false); - - assertExists(page1.next); - await page1.next(); - assertMatch(executeSpy.calls[1].args[1].api_key as string, /[a-z0-9]+/); - assertEquals(executeSpy.calls[1].args[1].no_cache, false); - }); - - await t.step("callback", async () => { - const page1 = await new Promise>>( - (res) => - getJson({ - engine, - api_key: SERPAPI_TEST_KEY, - no_cache: false, - product_id: productId, - }, res), - ); - assertMatch(executeSpy.calls[0].args[1].api_key as string, /[a-z0-9]+/); - assertEquals(executeSpy.calls[0].args[1].no_cache, false); - - assertExists(page1.next); - await new Promise((res) => page1.next?.(res)); - assertMatch(executeSpy.calls[1].args[1].api_key as string, /[a-z0-9]+/); - assertEquals(executeSpy.calls[1].args[1].no_cache, false); - }); - - executeSpy.restore(); - }); - - it("getJson pagination with page", { - ignore: !HAS_API_KEY, - }, async (t) => { - const firstPage = await getJson({ engine, product_id: productId }); - const idsOnFirstPage = firstPage.reviews.map((r: any) => r.id); - - await t.step("async/await", async () => { - const ids: number[] = []; - let page; - page = await getJson({ engine, product_id: productId, page: "2" }); - while (page) { - ids.push(...page.reviews.map((r: any) => r.id)); - if (ids.length >= 50) break; - page = await page.next?.(); - } - - assert(new Set(ids).size > 25, MSG_ASSERT_HAS_MULTIPLE_PAGES); - assert( - ids.some((id) => !idsOnFirstPage.includes(id)), - MSG_ASSERT_HAS_NON_FIRST_PAGE_RESULT, - ); - }); - - await t.step("callback", async () => { - const ids: number[] = []; - await new Promise((done) => { - getJson({ engine, product_id: productId, page: "2" }, (page) => { - ids.push(...page.reviews.map((r: any) => r.id)); - if (ids.length < 50 && page.next) { - page.next(); - } else { - assert(new Set(ids).size > 25, MSG_ASSERT_HAS_MULTIPLE_PAGES); - assert( - ids.some((id) => !idsOnFirstPage.includes(id)), - MSG_ASSERT_HAS_NON_FIRST_PAGE_RESULT, - ); - done(); - } - }); - }); - }); - }); - - it("getJson pagination has last page", { - ignore: !HAS_API_KEY, - }, async (t) => { - await t.step("async/await", async () => { - let page; - let pageNum = 0; - page = await getJson({ engine, product_id: productId, page: "99" }); - while (page && pageNum < 5) { - pageNum++; - page = await page.next?.(); - } - assert(pageNum < 5, MSG_ASSERT_HAS_LAST_PAGE); - }); - - await t.step("callback", async () => { - let pageNum = 0; - await new Promise((done) => { - getJson({ engine, product_id: productId, page: "99" }, (page) => { - pageNum++; - if (pageNum < 5 && page.next) { - page.next(); - } else { - assert(pageNum < 5, MSG_ASSERT_HAS_LAST_PAGE); - done(); - } - }); - }); - }); - }); -}); diff --git a/tests/engines/baidu_test.ts b/tests/engines/baidu_test.ts deleted file mode 100644 index 408148a..0000000 --- a/tests/engines/baidu_test.ts +++ /dev/null @@ -1,201 +0,0 @@ -// deno-lint-ignore-file no-explicit-any -import { loadSync } from "https://deno.land/std@0.170.0/dotenv/mod.ts"; -import { - afterAll, - afterEach, - beforeAll, - beforeEach, - describe, - it, -} from "https://deno.land/std@0.170.0/testing/bdd.ts"; -import { spy, Stub, stub } from "https://deno.land/std@0.170.0/testing/mock.ts"; -import { - assert, - assertEquals, - assertExists, - assertMatch, -} from "https://deno.land/std@0.170.0/testing/asserts.ts"; -import { _internals } from "../../src/utils.ts"; -import { config, getJson } from "../../mod.ts"; -import { - MSG_ASSERT_HAS_LAST_PAGE, - MSG_ASSERT_HAS_MULTIPLE_PAGES, - MSG_ASSERT_HAS_NON_FIRST_PAGE_RESULT, -} from "./constants.ts"; - -loadSync({ export: true }); -const SERPAPI_TEST_KEY = Deno.env.get("SERPAPI_TEST_KEY") ?? ""; -const HAS_API_KEY = SERPAPI_TEST_KEY.length > 0; -const BASE_URL = Deno.env.get("ENV_TYPE") === "local" - ? "http://localhost:3000" - : "https://serpapi.com"; - -describe("baidu", { - sanitizeOps: false, // TODO(seb): look into how we can avoid setting these to false - sanitizeResources: false, -}, () => { - let urlStub: Stub; - const engine = "baidu"; - const q = "coffee"; - - beforeAll(() => { - urlStub = stub(_internals, "getBaseUrl", () => BASE_URL); - }); - - beforeEach(() => { - config.api_key = SERPAPI_TEST_KEY; - }); - - afterEach(() => { - config.api_key = null; - }); - - afterAll(() => { - urlStub.restore(); - }); - - it("getJson pagination", { - ignore: !HAS_API_KEY, - }, async (t) => { - await t.step("async/await", async () => { - const links: string[] = []; - let page; - page = await getJson({ engine, q }); - while (page) { - links.push(...page.organic_results.map((r: any) => r.link)); - if (links.length >= 20) break; - page = await page.next?.(); - } - assert(new Set(links).size > 10, MSG_ASSERT_HAS_MULTIPLE_PAGES); - }); - - await t.step("callback", async () => { - const links: string[] = []; - await new Promise((done) => { - getJson({ engine, q }, (page) => { - links.push(...page.organic_results.map((r: any) => r.link)); - if (links.length < 20 && page.next) { - page.next(); - } else { - assert(new Set(links).size > 10, MSG_ASSERT_HAS_MULTIPLE_PAGES); - done(); - } - }); - }); - }); - }); - - it("getJson pagination keeps original parameter keys", { - ignore: !HAS_API_KEY, - }, async (t) => { - const executeSpy = spy(_internals, "execute"); - config.api_key = null; - - await t.step("async/await", async () => { - const page1 = await getJson({ - engine, - api_key: SERPAPI_TEST_KEY, - no_cache: false, - q, - }); - assertMatch(executeSpy.calls[0].args[1].api_key as string, /[a-z0-9]+/); - assertEquals(executeSpy.calls[0].args[1].no_cache, false); - - assertExists(page1.next); - await page1.next(); - assertMatch(executeSpy.calls[1].args[1].api_key as string, /[a-z0-9]+/); - assertEquals(executeSpy.calls[1].args[1].no_cache, false); - }); - - await t.step("callback", async () => { - const page1 = await new Promise>>( - (res) => - getJson( - { engine, api_key: SERPAPI_TEST_KEY, no_cache: false, q }, - res, - ), - ); - assertMatch(executeSpy.calls[0].args[1].api_key as string, /[a-z0-9]+/); - assertEquals(executeSpy.calls[0].args[1].no_cache, false); - - assertExists(page1.next); - await new Promise((res) => page1.next?.(res)); - assertMatch(executeSpy.calls[1].args[1].api_key as string, /[a-z0-9]+/); - assertEquals(executeSpy.calls[1].args[1].no_cache, false); - }); - - executeSpy.restore(); - }); - - it("getJson pagination with offset + size", { - ignore: !HAS_API_KEY, - }, async (t) => { - const firstPage = await getJson({ engine, q }); - const linksOnFirstPage = firstPage.organic_results.map((r: any) => r.link); - - await t.step("async/await", async () => { - const links: string[] = []; - let page; - page = await getJson({ engine, q, pn: "30", rn: "30" }); - while (page) { - links.push(...page.organic_results.map((r: any) => r.link)); - if (links.length >= 60) break; - page = await page.next?.(); - } - assert(new Set(links).size > 30, MSG_ASSERT_HAS_MULTIPLE_PAGES); - assert( - links.some((link) => !linksOnFirstPage.includes(link)), - MSG_ASSERT_HAS_NON_FIRST_PAGE_RESULT, - ); - }); - - await t.step("callback", async () => { - const links: string[] = []; - await new Promise((done) => { - getJson({ engine, q, pn: "30", rn: "30" }, (page) => { - links.push(...page.organic_results.map((r: any) => r.link)); - if (links.length < 60 && page.next) { - page.next(); - } else { - assert(new Set(links).size > 30, MSG_ASSERT_HAS_MULTIPLE_PAGES); - assert( - links.some((link) => !linksOnFirstPage.includes(link)), - MSG_ASSERT_HAS_NON_FIRST_PAGE_RESULT, - ); - done(); - } - }); - }); - }); - }); - - it("getJson pagination has last page", { - ignore: !HAS_API_KEY, - }, async (t) => { - await t.step("async/await", async () => { - let page; - let pageNum = 0; - page = await getJson({ engine, q, pn: "750" }); - while (page && pageNum < 5) { - pageNum++; - page = await page.next?.(); - } - assert(pageNum < 5, MSG_ASSERT_HAS_LAST_PAGE); - }); - - await t.step("callback", async () => { - let pageNum = 0; - await new Promise((done) => { - getJson({ engine, q, pn: "750" }, (page) => { - pageNum++; - if (pageNum < 5 && page.next) { - page.next(); - } else { - assert(pageNum < 5, MSG_ASSERT_HAS_LAST_PAGE); - done(); - } - }); - }); - }); - }); -}); diff --git a/tests/engines/constants.ts b/tests/engines/constants.ts deleted file mode 100644 index dd62efd..0000000 --- a/tests/engines/constants.ts +++ /dev/null @@ -1,4 +0,0 @@ -export const MSG_ASSERT_HAS_MULTIPLE_PAGES = "Only a single page was fetched"; -export const MSG_ASSERT_HAS_NON_FIRST_PAGE_RESULT = - "All results were from the first page"; -export const MSG_ASSERT_HAS_LAST_PAGE = "No last page was found"; diff --git a/tests/engines/duckduckgo_test.ts b/tests/engines/duckduckgo_test.ts deleted file mode 100644 index 792e5e5..0000000 --- a/tests/engines/duckduckgo_test.ts +++ /dev/null @@ -1,204 +0,0 @@ -// deno-lint-ignore-file no-explicit-any -import { loadSync } from "https://deno.land/std@0.170.0/dotenv/mod.ts"; -import { - afterAll, - afterEach, - beforeAll, - beforeEach, - describe, - it, -} from "https://deno.land/std@0.170.0/testing/bdd.ts"; -import { spy, Stub, stub } from "https://deno.land/std@0.170.0/testing/mock.ts"; -import { - assert, - assertEquals, - assertExists, - assertMatch, -} from "https://deno.land/std@0.170.0/testing/asserts.ts"; -import { _internals } from "../../src/utils.ts"; -import { config, getJson } from "../../mod.ts"; -import { - MSG_ASSERT_HAS_LAST_PAGE, - MSG_ASSERT_HAS_MULTIPLE_PAGES, - MSG_ASSERT_HAS_NON_FIRST_PAGE_RESULT, -} from "./constants.ts"; - -loadSync({ export: true }); -const SERPAPI_TEST_KEY = Deno.env.get("SERPAPI_TEST_KEY") ?? ""; -const HAS_API_KEY = SERPAPI_TEST_KEY.length > 0; -const BASE_URL = Deno.env.get("ENV_TYPE") === "local" - ? "http://localhost:3000" - : "https://serpapi.com"; - -describe("duckduckgo", { - sanitizeOps: false, // TODO(seb): look into how we can avoid setting these to false - sanitizeResources: false, -}, () => { - let urlStub: Stub; - const engine = "duckduckgo"; - const q = "coffee"; - - beforeAll(() => { - urlStub = stub(_internals, "getBaseUrl", () => BASE_URL); - }); - - beforeEach(() => { - config.api_key = SERPAPI_TEST_KEY; - }); - - afterEach(() => { - config.api_key = null; - }); - - afterAll(() => { - urlStub.restore(); - }); - - it("getJson pagination", { - ignore: !HAS_API_KEY, - }, async (t) => { - await t.step("async/await", async () => { - const links: string[] = []; - let page; - page = await getJson({ engine, q }); - while (page) { - links.push(...page.organic_results.map((r: any) => r.link)); - if (links.length >= 50) break; - page = await page.next?.(); - } - assert(new Set(links).size > 25, MSG_ASSERT_HAS_MULTIPLE_PAGES); - }); - - await t.step("callback", async () => { - const links: string[] = []; - await new Promise((done) => { - getJson({ engine, q }, (page) => { - links.push(...page.organic_results.map((r: any) => r.link)); - if (links.length < 50 && page.next) { - page.next(); - } else { - assert(new Set(links).size > 25, MSG_ASSERT_HAS_MULTIPLE_PAGES); - done(); - } - }); - }); - }); - }); - - it("getJson pagination keeps original parameter keys", { - ignore: !HAS_API_KEY, - }, async (t) => { - const executeSpy = spy(_internals, "execute"); - config.api_key = null; - - await t.step("async/await", async () => { - const page1 = await getJson({ - engine, - api_key: SERPAPI_TEST_KEY, - no_cache: false, - q, - }); - assertMatch(executeSpy.calls[0].args[1].api_key as string, /[a-z0-9]+/); - assertEquals(executeSpy.calls[0].args[1].no_cache, false); - - assertExists(page1.next); - await page1.next(); - assertMatch(executeSpy.calls[1].args[1].api_key as string, /[a-z0-9]+/); - assertEquals(executeSpy.calls[1].args[1].no_cache, false); - }); - - await t.step("callback", async () => { - const page1 = await new Promise>>( - (res) => - getJson( - { engine, api_key: SERPAPI_TEST_KEY, no_cache: false, q }, - res, - ), - ); - assertMatch(executeSpy.calls[0].args[1].api_key as string, /[a-z0-9]+/); - assertEquals(executeSpy.calls[0].args[1].no_cache, false); - - assertExists(page1.next); - await new Promise((res) => page1.next?.(res)); - assertMatch(executeSpy.calls[1].args[1].api_key as string, /[a-z0-9]+/); - assertEquals(executeSpy.calls[1].args[1].no_cache, false); - }); - - executeSpy.restore(); - }); - - it("getJson pagination with offset", { - ignore: !HAS_API_KEY, - }, async (t) => { - const firstPage = await getJson({ engine, q }); - const linksOnFirstPage: string[] = firstPage.organic_results.map(( - r: any, - ) => r.link); - - await t.step("async/await", async () => { - const links: string[] = []; - let page; - page = await getJson({ engine, q, start: 10 }); - while (page) { - links.push(...page.organic_results.map((r: any) => r.link)); - if (links.length >= 50) break; - page = await page.next?.(); - } - - assert(new Set(links).size > 25, MSG_ASSERT_HAS_MULTIPLE_PAGES); - assert( - links.some((id) => !linksOnFirstPage.includes(id)), - MSG_ASSERT_HAS_NON_FIRST_PAGE_RESULT, - ); - }); - - await t.step("callback", async () => { - const links: string[] = []; - await new Promise((done) => { - getJson({ engine, q, start: 10 }, (page) => { - links.push(...page.organic_results.map((r: any) => r.link)); - if (links.length < 50 && page.next) { - page.next(); - } else { - assert(new Set(links).size > 25, MSG_ASSERT_HAS_MULTIPLE_PAGES); - assert( - links.some((id) => !linksOnFirstPage.includes(id)), - MSG_ASSERT_HAS_NON_FIRST_PAGE_RESULT, - ); - done(); - } - }); - }); - }); - }); - - it("getJson pagination has last page", { - ignore: !HAS_API_KEY, - }, async (t) => { - await t.step("async/await", async () => { - let page; - let pageNum = 0; - page = await getJson({ engine, q, kl: "uk-en", no_cache: true }); - while (page && pageNum < 8) { - pageNum++; - page = await page.next?.(); - } - assert(pageNum < 8, MSG_ASSERT_HAS_LAST_PAGE); - }); - - await t.step("callback", async () => { - let pageNum = 0; - await new Promise((done) => { - getJson({ engine, q, kl: "uk-en" }, (page) => { - pageNum++; - if (pageNum < 8 && page.next) { - page.next(); - } else { - assert(pageNum < 8, MSG_ASSERT_HAS_LAST_PAGE); - done(); - } - }); - }); - }); - }); -}); diff --git a/tests/engines/ebay_test.ts b/tests/engines/ebay_test.ts deleted file mode 100644 index 6ebe126..0000000 --- a/tests/engines/ebay_test.ts +++ /dev/null @@ -1,203 +0,0 @@ -// deno-lint-ignore-file no-explicit-any -import { loadSync } from "https://deno.land/std@0.170.0/dotenv/mod.ts"; -import { - afterAll, - afterEach, - beforeAll, - beforeEach, - describe, - it, -} from "https://deno.land/std@0.170.0/testing/bdd.ts"; -import { spy, Stub, stub } from "https://deno.land/std@0.170.0/testing/mock.ts"; -import { - assert, - assertEquals, - assertExists, - assertMatch, -} from "https://deno.land/std@0.170.0/testing/asserts.ts"; -import { _internals } from "../../src/utils.ts"; -import { config, getJson } from "../../mod.ts"; -import { - MSG_ASSERT_HAS_LAST_PAGE, - MSG_ASSERT_HAS_MULTIPLE_PAGES, - MSG_ASSERT_HAS_NON_FIRST_PAGE_RESULT, -} from "./constants.ts"; - -loadSync({ export: true }); -const SERPAPI_TEST_KEY = Deno.env.get("SERPAPI_TEST_KEY") ?? ""; -const HAS_API_KEY = SERPAPI_TEST_KEY.length > 0; -const BASE_URL = Deno.env.get("ENV_TYPE") === "local" - ? "http://localhost:3000" - : "https://serpapi.com"; - -describe("ebay", { - sanitizeOps: false, // TODO(seb): look into how we can avoid setting these to false - sanitizeResources: false, -}, () => { - let urlStub: Stub; - const engine = "ebay"; - const nkw = "minecraft redstone"; - - beforeAll(() => { - urlStub = stub(_internals, "getBaseUrl", () => BASE_URL); - }); - - beforeEach(() => { - config.api_key = SERPAPI_TEST_KEY; - }); - - afterEach(() => { - config.api_key = null; - }); - - afterAll(() => { - urlStub.restore(); - }); - - it("getJson pagination", { - ignore: !HAS_API_KEY, - }, async (t) => { - await t.step("async/await", async () => { - const links: string[] = []; - let page; - page = await getJson({ engine, _nkw: nkw }); - while (page) { - links.push(...page.organic_results.map((r: any) => r.link)); - if (links.length >= 120) break; - page = await page.next?.(); - } - assert(new Set(links).size > 60, MSG_ASSERT_HAS_MULTIPLE_PAGES); - }); - - await t.step("callback", async () => { - const links: string[] = []; - await new Promise((done) => { - getJson({ engine, _nkw: nkw }, (page) => { - links.push(...page.organic_results.map((r: any) => r.link)); - if (links.length < 120 && page.next) { - page.next(); - } else { - assert(new Set(links).size > 60, MSG_ASSERT_HAS_MULTIPLE_PAGES); - done(); - } - }); - }); - }); - }); - - it("getJson pagination keeps original parameter keys", { - ignore: !HAS_API_KEY, - }, async (t) => { - const executeSpy = spy(_internals, "execute"); - config.api_key = null; - - await t.step("async/await", async () => { - const page1 = await getJson({ - engine, - api_key: SERPAPI_TEST_KEY, - no_cache: false, - _nkw: nkw, - }); - assertMatch(executeSpy.calls[0].args[1].api_key as string, /[a-z0-9]+/); - assertEquals(executeSpy.calls[0].args[1].no_cache, false); - - assertExists(page1.next); - await page1.next(); - assertMatch(executeSpy.calls[1].args[1].api_key as string, /[a-z0-9]+/); - assertEquals(executeSpy.calls[1].args[1].no_cache, false); - }); - - await t.step("callback", async () => { - const page1 = await new Promise>>( - (res) => - getJson({ - engine, - api_key: SERPAPI_TEST_KEY, - no_cache: false, - _nkw: nkw, - }, res), - ); - assertMatch(executeSpy.calls[0].args[1].api_key as string, /[a-z0-9]+/); - assertEquals(executeSpy.calls[0].args[1].no_cache, false); - - assertExists(page1.next); - await new Promise((res) => page1.next?.(res)); - assertMatch(executeSpy.calls[1].args[1].api_key as string, /[a-z0-9]+/); - assertEquals(executeSpy.calls[1].args[1].no_cache, false); - }); - - executeSpy.restore(); - }); - - it("getJson pagination with page + size", { - ignore: !HAS_API_KEY, - }, async (t) => { - const firstPage = await getJson({ engine, _nkw: nkw }); - const linksOnFirstPage = firstPage.organic_results.map((r: any) => r.link); - - await t.step("async/await", async () => { - const links: string[] = []; - let page; - page = await getJson({ engine, _nkw: nkw, _pgn: "2", _ipg: "100" }); - while (page) { - links.push(...page.organic_results.map((r: any) => r.link)); - if (links.length >= 200) break; - page = await page.next?.(); - } - assert(new Set(links).size > 100, MSG_ASSERT_HAS_MULTIPLE_PAGES); - assert( - links.some((link) => !linksOnFirstPage.includes(link)), - MSG_ASSERT_HAS_NON_FIRST_PAGE_RESULT, - ); - }); - - await t.step("callback", async () => { - const links: string[] = []; - await new Promise((done) => { - getJson({ engine, _nkw: nkw, _pgn: "2", _ipg: "100" }, (page) => { - links.push(...page.organic_results.map((r: any) => r.link)); - if (links.length < 200 && page.next) { - page.next(); - } else { - assert(new Set(links).size > 100, MSG_ASSERT_HAS_MULTIPLE_PAGES); - assert( - links.some((link) => !linksOnFirstPage.includes(link)), - MSG_ASSERT_HAS_NON_FIRST_PAGE_RESULT, - ); - done(); - } - }); - }); - }); - }); - - it("getJson pagination has last page", { - ignore: !HAS_API_KEY, - }, async (t) => { - await t.step("async/await", async () => { - let page; - let pageNum = 0; - page = await getJson({ engine, _nkw: nkw, _pgn: "7", _ipg: "200" }); - while (page && pageNum < 5) { - pageNum++; - page = await page.next?.(); - } - assert(pageNum < 5, MSG_ASSERT_HAS_LAST_PAGE); - }); - - await t.step("callback", async () => { - let pageNum = 0; - await new Promise((done) => { - getJson({ engine, _nkw: nkw, _pgn: "7", _ipg: "200" }, (page) => { - pageNum++; - if (pageNum < 5 && page.next) { - page.next(); - } else { - assert(pageNum < 5, MSG_ASSERT_HAS_LAST_PAGE); - done(); - } - }); - }); - }); - }); -}); diff --git a/tests/engines/google_maps_test.ts b/tests/engines/google_maps_test.ts deleted file mode 100644 index ad3f6a1..0000000 --- a/tests/engines/google_maps_test.ts +++ /dev/null @@ -1,212 +0,0 @@ -// deno-lint-ignore-file no-explicit-any -import { loadSync } from "https://deno.land/std@0.170.0/dotenv/mod.ts"; -import { - afterAll, - afterEach, - beforeAll, - beforeEach, - describe, - it, -} from "https://deno.land/std@0.170.0/testing/bdd.ts"; -import { spy, Stub, stub } from "https://deno.land/std@0.170.0/testing/mock.ts"; -import { - assert, - assertEquals, - assertExists, - assertMatch, -} from "https://deno.land/std@0.170.0/testing/asserts.ts"; -import { _internals } from "../../src/utils.ts"; -import { config, getJson } from "../../mod.ts"; -import { - MSG_ASSERT_HAS_LAST_PAGE, - MSG_ASSERT_HAS_MULTIPLE_PAGES, - MSG_ASSERT_HAS_NON_FIRST_PAGE_RESULT, -} from "./constants.ts"; - -loadSync({ export: true }); -const SERPAPI_TEST_KEY = Deno.env.get("SERPAPI_TEST_KEY") ?? ""; -const HAS_API_KEY = SERPAPI_TEST_KEY.length > 0; -const BASE_URL = Deno.env.get("ENV_TYPE") === "local" - ? "http://localhost:3000" - : "https://serpapi.com"; - -describe("google_maps", { - sanitizeOps: false, // TODO(seb): look into how we can avoid setting these to false - sanitizeResources: false, -}, () => { - let urlStub: Stub; - const engine = "google_maps"; - const q = "gas"; - const type = "search"; - const ll = "@40.7455096,-74.0083012,16z"; - - beforeAll(() => { - urlStub = stub(_internals, "getBaseUrl", () => BASE_URL); - }); - - beforeEach(() => { - config.api_key = SERPAPI_TEST_KEY; - }); - - afterEach(() => { - config.api_key = null; - }); - - afterAll(() => { - urlStub.restore(); - }); - - it("getJson pagination", { - ignore: !HAS_API_KEY, - }, async (t) => { - await t.step("async/await", async () => { - const placeIds: string[] = []; - let page; - page = await getJson({ engine, q, type, ll }); - while (page) { - placeIds.push(...page.local_results.map((r: any) => r.place_id)); - if (placeIds.length >= 40) break; - page = await page.next?.(); - } - assert(new Set(placeIds).size > 20, MSG_ASSERT_HAS_MULTIPLE_PAGES); - }); - - await t.step("callback", async () => { - const placeIds: string[] = []; - await new Promise((done) => { - getJson({ engine, q, type, ll }, (page) => { - placeIds.push(...page.local_results.map((r: any) => r.place_id)); - if (placeIds.length < 40 && page.next) { - page.next(); - } else { - assert(new Set(placeIds).size > 20, MSG_ASSERT_HAS_MULTIPLE_PAGES); - done(); - } - }); - }); - }); - }); - - it("getJson pagination keeps original parameter keys", { - ignore: !HAS_API_KEY, - }, async (t) => { - const executeSpy = spy(_internals, "execute"); - config.api_key = null; - - await t.step("async/await", async () => { - const page1 = await getJson({ - engine, - api_key: SERPAPI_TEST_KEY, - no_cache: false, - q, - type, - ll, - }); - assertMatch(executeSpy.calls[0].args[1].api_key as string, /[a-z0-9]+/); - assertEquals(executeSpy.calls[0].args[1].no_cache, false); - - assertExists(page1.next); - await page1.next(); - assertMatch(executeSpy.calls[1].args[1].api_key as string, /[a-z0-9]+/); - assertEquals(executeSpy.calls[1].args[1].no_cache, false); - }); - - await t.step("callback", async () => { - const page1 = await new Promise>>( - (res) => - getJson({ - engine, - api_key: SERPAPI_TEST_KEY, - no_cache: false, - q, - type, - ll, - }, res), - ); - assertMatch(executeSpy.calls[0].args[1].api_key as string, /[a-z0-9]+/); - assertEquals(executeSpy.calls[0].args[1].no_cache, false); - - assertExists(page1.next); - await new Promise((res) => page1.next?.(res)); - assertMatch(executeSpy.calls[1].args[1].api_key as string, /[a-z0-9]+/); - assertEquals(executeSpy.calls[1].args[1].no_cache, false); - }); - - executeSpy.restore(); - }); - - it("getJson pagination with offset", { - ignore: !HAS_API_KEY, - }, async (t) => { - const firstPage = await getJson({ engine, q, type, ll }); - const placeIdsOnFirstPage: string[] = firstPage.local_results.map(( - r: any, - ) => r.place_id); - - await t.step("async/await", async () => { - const placeIds: string[] = []; - let page; - page = await getJson({ engine, q, type, ll, start: 40 }); - while (page) { - placeIds.push(...page.local_results.map((r: any) => r.place_id)); - if (placeIds.length >= 40) break; - page = await page.next?.(); - } - - assert(new Set(placeIds).size > 20, MSG_ASSERT_HAS_MULTIPLE_PAGES); - assert( - placeIds.some((id) => !placeIdsOnFirstPage.includes(id)), - MSG_ASSERT_HAS_NON_FIRST_PAGE_RESULT, - ); - }); - - await t.step("callback", async () => { - const placeIds: string[] = []; - await new Promise((done) => { - getJson({ engine, q, type, ll, start: 20 }, (page) => { - placeIds.push(...page.local_results.map((r: any) => r.place_id)); - if (placeIds.length < 40 && page.next) { - page.next(); - } else { - assert(new Set(placeIds).size > 20, MSG_ASSERT_HAS_MULTIPLE_PAGES); - assert( - placeIds.some((id) => !placeIdsOnFirstPage.includes(id)), - MSG_ASSERT_HAS_NON_FIRST_PAGE_RESULT, - ); - done(); - } - }); - }); - }); - }); - - it("getJson pagination has last page", { - ignore: !HAS_API_KEY, - }, async (t) => { - await t.step("async/await", async () => { - let page; - let pageNum = 0; - page = await getJson({ engine, q, type, ll, start: 260 }); - while (page && pageNum < 5) { - pageNum++; - page = await page.next?.(); - } - assert(pageNum < 5, MSG_ASSERT_HAS_LAST_PAGE); - }); - - await t.step("callback", async () => { - let pageNum = 0; - await new Promise((done) => { - getJson({ engine, q, type, ll, start: 260 }, (page) => { - pageNum++; - if (pageNum < 5 && page.next) { - page.next(); - } else { - assert(pageNum < 5, MSG_ASSERT_HAS_LAST_PAGE); - done(); - } - }); - }); - }); - }); -}); diff --git a/tests/engines/google_scholar_profiles_test.ts b/tests/engines/google_scholar_profiles_test.ts deleted file mode 100644 index 63bed96..0000000 --- a/tests/engines/google_scholar_profiles_test.ts +++ /dev/null @@ -1,205 +0,0 @@ -// deno-lint-ignore-file no-explicit-any -import { loadSync } from "https://deno.land/std@0.170.0/dotenv/mod.ts"; -import { - afterAll, - afterEach, - beforeAll, - beforeEach, - describe, - it, -} from "https://deno.land/std@0.170.0/testing/bdd.ts"; -import { spy, Stub, stub } from "https://deno.land/std@0.170.0/testing/mock.ts"; -import { - assert, - assertEquals, - assertExists, - assertMatch, -} from "https://deno.land/std@0.170.0/testing/asserts.ts"; -import { _internals } from "../../src/utils.ts"; -import { config, getJson } from "../../mod.ts"; -import { - MSG_ASSERT_HAS_LAST_PAGE, - MSG_ASSERT_HAS_MULTIPLE_PAGES, - MSG_ASSERT_HAS_NON_FIRST_PAGE_RESULT, -} from "./constants.ts"; - -loadSync({ export: true }); -const SERPAPI_TEST_KEY = Deno.env.get("SERPAPI_TEST_KEY") ?? ""; -const HAS_API_KEY = SERPAPI_TEST_KEY.length > 0; -const BASE_URL = Deno.env.get("ENV_TYPE") === "local" - ? "http://localhost:3000" - : "https://serpapi.com"; - -describe("google_scholar_profiles", { - sanitizeOps: false, // TODO(seb): look into how we can avoid setting these to false - sanitizeResources: false, -}, () => { - let urlStub: Stub; - const engine = "google_scholar_profiles"; - const mauthors = "mike"; - - beforeAll(() => { - urlStub = stub(_internals, "getBaseUrl", () => BASE_URL); - }); - - beforeEach(() => { - config.api_key = SERPAPI_TEST_KEY; - }); - - afterEach(() => { - config.api_key = null; - }); - - afterAll(() => { - urlStub.restore(); - }); - - it("getJson pagination", { - ignore: !HAS_API_KEY, - }, async (t) => { - await t.step("async/await", async () => { - const authorIds: string[] = []; - let page; - page = await getJson({ engine, mauthors }); - while (page) { - authorIds.push(...page.profiles.map((r: any) => r.author_id)); - if (authorIds.length >= 20) break; - page = await page.next?.(); - } - assert(new Set(authorIds).size > 10, MSG_ASSERT_HAS_MULTIPLE_PAGES); - }); - - await t.step("callback", async () => { - const authorIds: string[] = []; - await new Promise((done) => { - getJson({ engine, mauthors }, (page) => { - authorIds.push(...page.profiles.map((r: any) => r.author_id)); - if (authorIds.length < 20 && page.next) { - page.next(); - } else { - assert(new Set(authorIds).size > 10, MSG_ASSERT_HAS_MULTIPLE_PAGES); - done(); - } - }); - }); - }); - }); - - it("getJson pagination keeps original parameter keys", { - ignore: !HAS_API_KEY, - }, async (t) => { - const executeSpy = spy(_internals, "execute"); - config.api_key = null; - - await t.step("async/await", async () => { - const page1 = await getJson({ - engine, - api_key: SERPAPI_TEST_KEY, - no_cache: false, - mauthors, - }); - assertMatch(executeSpy.calls[0].args[1].api_key as string, /[a-z0-9]+/); - assertEquals(executeSpy.calls[0].args[1].no_cache, false); - - assertExists(page1.next); - await page1.next(); - assertMatch(executeSpy.calls[1].args[1].api_key as string, /[a-z0-9]+/); - assertEquals(executeSpy.calls[1].args[1].no_cache, false); - }); - - await t.step("callback", async () => { - const page1 = await new Promise>>( - (res) => - getJson({ - engine, - api_key: SERPAPI_TEST_KEY, - no_cache: false, - mauthors, - }, res), - ); - assertMatch(executeSpy.calls[0].args[1].api_key as string, /[a-z0-9]+/); - assertEquals(executeSpy.calls[0].args[1].no_cache, false); - - assertExists(page1.next); - await new Promise((res) => page1.next?.(res)); - assertMatch(executeSpy.calls[1].args[1].api_key as string, /[a-z0-9]+/); - assertEquals(executeSpy.calls[1].args[1].no_cache, false); - }); - - executeSpy.restore(); - }); - - it("getJson pagination with token", { - ignore: !HAS_API_KEY, - }, async (t) => { - const firstPage = await getJson({ engine, mauthors }); - const authorIdsOnFirstPage = firstPage.profiles.map((r: any) => - r.author_id - ); - - await t.step("async/await", async () => { - const authorIds: string[] = []; - let page; - page = await getJson({ engine, mauthors, after_author: "rZlDAYoq__8J" }); - while (page) { - authorIds.push(...page.profiles.map((r: any) => r.author_id)); - if (authorIds.length >= 20) break; - page = await page.next?.(); - } - assert(new Set(authorIds).size > 10, MSG_ASSERT_HAS_MULTIPLE_PAGES); - assert( - authorIds.some((id) => !authorIdsOnFirstPage.includes(id)), - MSG_ASSERT_HAS_NON_FIRST_PAGE_RESULT, - ); - }); - - await t.step("callback", async () => { - const authorIds: string[] = []; - await new Promise((done) => { - getJson({ engine, mauthors, after_author: "rZlDAYoq__8J" }, (page) => { - authorIds.push(...page.profiles.map((r: any) => r.author_id)); - if (authorIds.length < 20 && page.next) { - page.next(); - } else { - assert(new Set(authorIds).size > 10, MSG_ASSERT_HAS_MULTIPLE_PAGES); - assert( - authorIds.some((id) => !authorIdsOnFirstPage.includes(id)), - MSG_ASSERT_HAS_NON_FIRST_PAGE_RESULT, - ); - done(); - } - }); - }); - }); - }); - - it("getJson pagination has last page", { - ignore: !HAS_API_KEY, - }, async (t) => { - await t.step("async/await", async () => { - let page; - let pageNum = 0; - page = await getJson({ engine, mauthors, after_author: "UXxiAf3___8J" }); - while (page && pageNum < 5) { - pageNum++; - page = await page.next?.(); - } - assert(pageNum < 5, MSG_ASSERT_HAS_LAST_PAGE); - }); - - await t.step("callback", async () => { - let pageNum = 0; - await new Promise((done) => { - getJson({ engine, mauthors, after_author: "UXxiAf3___8J" }, (page) => { - pageNum++; - if (pageNum < 5 && page.next) { - page.next(); - } else { - assert(pageNum < 5, MSG_ASSERT_HAS_LAST_PAGE); - done(); - } - }); - }); - }); - }); -}); diff --git a/tests/engines/home_depot_test.ts b/tests/engines/home_depot_test.ts deleted file mode 100644 index 77c66a1..0000000 --- a/tests/engines/home_depot_test.ts +++ /dev/null @@ -1,285 +0,0 @@ -// deno-lint-ignore-file no-explicit-any -import { loadSync } from "https://deno.land/std@0.170.0/dotenv/mod.ts"; -import { - afterAll, - afterEach, - beforeAll, - beforeEach, - describe, - it, -} from "https://deno.land/std@0.170.0/testing/bdd.ts"; -import { spy, Stub, stub } from "https://deno.land/std@0.170.0/testing/mock.ts"; -import { - assert, - assertEquals, - assertExists, - assertMatch, -} from "https://deno.land/std@0.170.0/testing/asserts.ts"; -import { _internals } from "../../src/utils.ts"; -import { config, getJson } from "../../mod.ts"; -import { - MSG_ASSERT_HAS_LAST_PAGE, - MSG_ASSERT_HAS_MULTIPLE_PAGES, - MSG_ASSERT_HAS_NON_FIRST_PAGE_RESULT, -} from "./constants.ts"; - -loadSync({ export: true }); -const SERPAPI_TEST_KEY = Deno.env.get("SERPAPI_TEST_KEY") ?? ""; -const HAS_API_KEY = SERPAPI_TEST_KEY.length > 0; -const BASE_URL = Deno.env.get("ENV_TYPE") === "local" - ? "http://localhost:3000" - : "https://serpapi.com"; - -describe("home_depot", { - sanitizeOps: false, // TODO(seb): look into how we can avoid setting these to false - sanitizeResources: false, -}, () => { - let urlStub: Stub; - const engine = "home_depot"; - const q = "coffee"; - - beforeAll(() => { - urlStub = stub(_internals, "getBaseUrl", () => BASE_URL); - }); - - beforeEach(() => { - config.api_key = SERPAPI_TEST_KEY; - }); - - afterEach(() => { - config.api_key = null; - }); - - afterAll(() => { - urlStub.restore(); - }); - - it("getJson pagination", { - ignore: !HAS_API_KEY, - }, async (t) => { - await t.step("async/await", async () => { - const ids: string[] = []; - let page; - page = await getJson({ engine, q }); - while (page) { - ids.push(...page.products.map((r: any) => r.product_id)); - if (ids.length >= 48) break; - page = await page.next?.(); - } - assert(new Set(ids).size > 24, MSG_ASSERT_HAS_MULTIPLE_PAGES); - }); - - await t.step("callback", async () => { - const ids: string[] = []; - await new Promise((done) => { - getJson({ engine, q }, (page) => { - ids.push(...page.products.map((r: any) => r.product_id)); - if (ids.length < 48 && page.next) { - page.next(); - } else { - assert(new Set(ids).size > 24, MSG_ASSERT_HAS_MULTIPLE_PAGES); - done(); - } - }); - }); - }); - }); - - it("getJson pagination keeps original parameter keys", { - ignore: !HAS_API_KEY, - }, async (t) => { - const executeSpy = spy(_internals, "execute"); - config.api_key = null; - - await t.step("async/await", async () => { - const page1 = await getJson({ - engine, - api_key: SERPAPI_TEST_KEY, - no_cache: false, - q, - }); - assertMatch(executeSpy.calls[0].args[1].api_key as string, /[a-z0-9]+/); - assertEquals(executeSpy.calls[0].args[1].no_cache, false); - - assertExists(page1.next); - await page1.next(); - assertMatch(executeSpy.calls[1].args[1].api_key as string, /[a-z0-9]+/); - assertEquals(executeSpy.calls[1].args[1].no_cache, false); - }); - - await t.step("callback", async () => { - const page1 = await new Promise>>( - (res) => - getJson( - { engine, api_key: SERPAPI_TEST_KEY, no_cache: false, q }, - res, - ), - ); - assertMatch(executeSpy.calls[0].args[1].api_key as string, /[a-z0-9]+/); - assertEquals(executeSpy.calls[0].args[1].no_cache, false); - - assertExists(page1.next); - await new Promise((res) => page1.next?.(res)); - assertMatch(executeSpy.calls[1].args[1].api_key as string, /[a-z0-9]+/); - assertEquals(executeSpy.calls[1].args[1].no_cache, false); - }); - - executeSpy.restore(); - }); - - it("getJson pagination with offset + size", { - ignore: !HAS_API_KEY, - }, async (t) => { - const firstPage = await getJson({ engine, q, ps: 3 }); - const idsOnFirstPage = firstPage.products.map((r: any) => r.product_id); - - await t.step("async/await", async () => { - const ids: string[] = []; - let page; - page = await getJson({ engine, q, nao: "6", ps: 3 }); - while (page) { - ids.push(...page.products.map((r: any) => r.product_id)); - if (ids.length >= 6) break; - page = await page.next?.(); - } - assert(new Set(ids).size > 3, MSG_ASSERT_HAS_MULTIPLE_PAGES); - assert( - ids.some((id) => !idsOnFirstPage.includes(id)), - MSG_ASSERT_HAS_NON_FIRST_PAGE_RESULT, - ); - }); - - await t.step("callback", async () => { - const ids: string[] = []; - await new Promise((done) => { - getJson({ engine, q, nao: "6", ps: 3 }, (page) => { - ids.push(...page.products.map((r: any) => r.product_id)); - if (ids.length < 6 && page.next) { - page.next(); - } else { - assert(new Set(ids).size > 3, MSG_ASSERT_HAS_MULTIPLE_PAGES); - assert( - ids.some((id) => !idsOnFirstPage.includes(id)), - MSG_ASSERT_HAS_NON_FIRST_PAGE_RESULT, - ); - done(); - } - }); - }); - }); - }); - - it("getJson pagination with page + size", { - ignore: !HAS_API_KEY, - }, async (t) => { - const firstPage = await getJson({ engine, q, ps: 3 }); - const idsOnFirstPage = firstPage.products.map((r: any) => r.product_id); - - await t.step("async/await", async () => { - const ids: string[] = []; - let page; - page = await getJson({ engine, q, page: "3", ps: 3 }); - while (page) { - ids.push(...page.products.map((r: any) => r.product_id)); - if (ids.length >= 6) break; - page = await page.next?.(); - } - assert(new Set(ids).size > 3, MSG_ASSERT_HAS_MULTIPLE_PAGES); - assert( - ids.some((id) => !idsOnFirstPage.includes(id)), - MSG_ASSERT_HAS_NON_FIRST_PAGE_RESULT, - ); - }); - - await t.step("callback", async () => { - const ids: string[] = []; - await new Promise((done) => { - getJson({ engine, q, page: "3", ps: 3 }, (page) => { - ids.push(...page.products.map((r: any) => r.product_id)); - if (ids.length < 6 && page.next) { - page.next(); - } else { - assert(new Set(ids).size > 3, MSG_ASSERT_HAS_MULTIPLE_PAGES); - assert( - ids.some((id) => !idsOnFirstPage.includes(id)), - MSG_ASSERT_HAS_NON_FIRST_PAGE_RESULT, - ); - done(); - } - }); - }); - }); - }); - - it("getJson pagination with offset + page + size", { - ignore: !HAS_API_KEY, - }, async (t) => { - const firstPage = await getJson({ engine, q, ps: 3 }); - const idsOnFirstPage = firstPage.products.map((r: any) => r.product_id); - - await t.step("async/await", async () => { - const ids: string[] = []; - let page; - page = await getJson({ engine, q, nao: "6", page: "3", ps: 3 }); - while (page) { - ids.push(...page.products.map((r: any) => r.product_id)); - if (ids.length >= 6) break; - page = await page.next?.(); - } - assert(new Set(ids).size > 3, MSG_ASSERT_HAS_MULTIPLE_PAGES); - assert( - ids.some((id) => !idsOnFirstPage.includes(id)), - MSG_ASSERT_HAS_NON_FIRST_PAGE_RESULT, - ); - }); - - await t.step("callback", async () => { - const ids: string[] = []; - await new Promise((done) => { - getJson({ engine, q, nao: "6", page: "3", ps: 3 }, (page) => { - ids.push(...page.products.map((r: any) => r.product_id)); - if (ids.length < 6 && page.next) { - page.next(); - } else { - assert(new Set(ids).size > 3, MSG_ASSERT_HAS_MULTIPLE_PAGES); - assert( - ids.some((id) => !idsOnFirstPage.includes(id)), - MSG_ASSERT_HAS_NON_FIRST_PAGE_RESULT, - ); - done(); - } - }); - }); - }); - }); - - it("getJson pagination has last page", { - ignore: !HAS_API_KEY, - }, async (t) => { - await t.step("async/await", async () => { - let page; - let pageNum = 0; - page = await getJson({ engine, q, page: "17", ps: 48 }); - while (page && pageNum < 5) { - pageNum++; - page = await page.next?.(); - } - assert(pageNum < 5, MSG_ASSERT_HAS_LAST_PAGE); - }); - - await t.step("callback", async () => { - let pageNum = 0; - await new Promise((done) => { - getJson({ engine, q, page: "17", ps: 48 }, (page) => { - pageNum++; - if (pageNum < 5 && page.next) { - page.next(); - } else { - assert(pageNum < 5, MSG_ASSERT_HAS_LAST_PAGE); - done(); - } - }); - }); - }); - }); -}); diff --git a/tests/utils_test.ts b/tests/utils_test.ts index ee1926f..f66b37e 100644 --- a/tests/utils_test.ts +++ b/tests/utils_test.ts @@ -11,14 +11,7 @@ import { assertInstanceOf, assertMatch, } from "https://deno.land/std@0.170.0/testing/asserts.ts"; -import { - _internals, - buildUrl, - execute, - extractNextParameters, - getSource, - haveParametersChanged, -} from "../src/utils.ts"; +import { _internals, buildUrl, execute, getSource } from "../src/utils.ts"; import { RequestTimeoutError } from "../src/errors.ts"; loadSync({ export: true }); @@ -26,148 +19,6 @@ const BASE_URL = Deno.env.get("ENV_TYPE") === "local" ? "http://localhost:3000" : "https://serpapi.com"; -describe("extractNextParameters", () => { - it("with serpapi_pagination property", async () => { - assertEquals( - await extractNextParameters({ - serpapi_pagination: { - next: - "https://serpapi.com/search.json?device=desktop&engine=google&gl=us&google_domain=google.com&hl=en&location=Austin%2C+Texas%2C+United+States&q=coffee&start=10", - }, - }), - { - engine: "google", - device: "desktop", - gl: "us", - google_domain: "google.com", - hl: "en", - location: "Austin, Texas, United States", - q: "coffee", - start: "10", - }, - ); - }); - - it("with pagination property", async () => { - assertEquals( - await extractNextParameters( - { - pagination: { - next: - "https://serpapi.com/search.json?after_author=rZlDAYoq__8J&engine=google_scholar_profiles&hl=en&mauthors=Mike", - }, - }, - ), - { - engine: "google_scholar_profiles", - after_author: "rZlDAYoq__8J", - hl: "en", - mauthors: "Mike", - }, - ); - }); -}); - -describe("haveParametersChanged", () => { - it("with different number of properties", () => { - assertEquals( - haveParametersChanged({ q: "coffee" }, { - kl: "us-en", - q: "coffee", - start: "26", - }), - true, - ); - assertEquals( - haveParametersChanged({ kl: "us-en", q: "coffee", start: "26" }, { - q: "coffee", - }), - true, - ); - }); - - it("with same number of properties, but different properties", () => { - assertEquals( - haveParametersChanged({ - kl: "us-en", - q: "coffee", - safe: "1", - }, { - kl: "us-en", - q: "coffee", - start: "26", - }), - true, - ); - }); - - it("with same properties, but different values", () => { - assertEquals( - haveParametersChanged({ - kl: "us-en", - q: "coffee", - start: "30", - }, { - kl: "us-en", - q: "coffee", - start: "26", - }), - true, - ); - assertEquals( - haveParametersChanged({ - kl: "us-en", - q: "coffee", - start: "26", - }, { - kl: "us-en", - q: "coffee", - start: "30", - }), - true, - ); - }); - - it("with same properties and same values, regardless of type", () => { - assertEquals( - haveParametersChanged({ - kl: "us-en", - q: "coffee", - start: "30", - }, { - kl: "us-en", - q: "coffee", - start: "30", - }), - false, - ); - assertEquals( - haveParametersChanged({ - kl: "us-en", - q: "coffee", - start: 30, - }, { - kl: "us-en", - q: "coffee", - start: "30", - }), - false, - ); - assertEquals( - haveParametersChanged({ - kl: "us-en", - q: "coffee", - start: "30", - }, { - kl: "us-en", - q: "coffee", - start: 30, - }), - false, - ); - }); -}); - describe("getSource", () => { it("use runtime version", async () => { assertMatch( From 16bc1ae06e09100f9eaca4e938933a309bb6e388 Mon Sep 17 00:00:00 2001 From: zyc9012 Date: Sun, 2 Jul 2023 12:26:40 +0800 Subject: [PATCH 45/61] Update example folder structure --- examples/deno/{basic_ts => }/.env.example | 0 examples/deno/{basic_ts => }/README.md | 0 examples/deno/{basic_ts => }/example.ts | 2 +- .../node/{basic_js_node_14_up => js_node_14_up}/.env.example | 0 examples/node/{basic_js_node_14_up => js_node_14_up}/README.md | 0 examples/node/{basic_js_node_14_up => js_node_14_up}/example.js | 0 .../node/{basic_js_node_14_up => js_node_14_up}/package.json | 0 examples/node/{basic_js_node_7_up => js_node_7_up}/.env.example | 0 examples/node/{basic_js_node_7_up => js_node_7_up}/README.md | 0 examples/node/{basic_js_node_7_up => js_node_7_up}/example.js | 0 examples/node/{basic_js_node_7_up => js_node_7_up}/package.json | 0 .../node/{basic_ts_node_14_up => ts_node_14_up}/.env.example | 0 examples/node/{basic_ts_node_14_up => ts_node_14_up}/README.md | 0 examples/node/{basic_ts_node_14_up => ts_node_14_up}/example.ts | 0 .../node/{basic_ts_node_14_up => ts_node_14_up}/package.json | 0 .../node/{basic_ts_node_14_up => ts_node_14_up}/tsconfig.json | 0 16 files changed, 1 insertion(+), 1 deletion(-) rename examples/deno/{basic_ts => }/.env.example (100%) rename examples/deno/{basic_ts => }/README.md (100%) rename examples/deno/{basic_ts => }/example.ts (91%) rename examples/node/{basic_js_node_14_up => js_node_14_up}/.env.example (100%) rename examples/node/{basic_js_node_14_up => js_node_14_up}/README.md (100%) rename examples/node/{basic_js_node_14_up => js_node_14_up}/example.js (100%) rename examples/node/{basic_js_node_14_up => js_node_14_up}/package.json (100%) rename examples/node/{basic_js_node_7_up => js_node_7_up}/.env.example (100%) rename examples/node/{basic_js_node_7_up => js_node_7_up}/README.md (100%) rename examples/node/{basic_js_node_7_up => js_node_7_up}/example.js (100%) rename examples/node/{basic_js_node_7_up => js_node_7_up}/package.json (100%) rename examples/node/{basic_ts_node_14_up => ts_node_14_up}/.env.example (100%) rename examples/node/{basic_ts_node_14_up => ts_node_14_up}/README.md (100%) rename examples/node/{basic_ts_node_14_up => ts_node_14_up}/example.ts (100%) rename examples/node/{basic_ts_node_14_up => ts_node_14_up}/package.json (100%) rename examples/node/{basic_ts_node_14_up => ts_node_14_up}/tsconfig.json (100%) diff --git a/examples/deno/basic_ts/.env.example b/examples/deno/.env.example similarity index 100% rename from examples/deno/basic_ts/.env.example rename to examples/deno/.env.example diff --git a/examples/deno/basic_ts/README.md b/examples/deno/README.md similarity index 100% rename from examples/deno/basic_ts/README.md rename to examples/deno/README.md diff --git a/examples/deno/basic_ts/example.ts b/examples/deno/example.ts similarity index 91% rename from examples/deno/basic_ts/example.ts rename to examples/deno/example.ts index bee7d5e..30e6aa3 100644 --- a/examples/deno/basic_ts/example.ts +++ b/examples/deno/example.ts @@ -1,5 +1,5 @@ import { loadSync } from "https://deno.land/std@0.173.0/dotenv/mod.ts"; -import { config, getJson } from "../../../mod.ts"; +import { config, getJson } from "../../mod.ts"; const { API_KEY: apiKey } = loadSync(); const params = { diff --git a/examples/node/basic_js_node_14_up/.env.example b/examples/node/js_node_14_up/.env.example similarity index 100% rename from examples/node/basic_js_node_14_up/.env.example rename to examples/node/js_node_14_up/.env.example diff --git a/examples/node/basic_js_node_14_up/README.md b/examples/node/js_node_14_up/README.md similarity index 100% rename from examples/node/basic_js_node_14_up/README.md rename to examples/node/js_node_14_up/README.md diff --git a/examples/node/basic_js_node_14_up/example.js b/examples/node/js_node_14_up/example.js similarity index 100% rename from examples/node/basic_js_node_14_up/example.js rename to examples/node/js_node_14_up/example.js diff --git a/examples/node/basic_js_node_14_up/package.json b/examples/node/js_node_14_up/package.json similarity index 100% rename from examples/node/basic_js_node_14_up/package.json rename to examples/node/js_node_14_up/package.json diff --git a/examples/node/basic_js_node_7_up/.env.example b/examples/node/js_node_7_up/.env.example similarity index 100% rename from examples/node/basic_js_node_7_up/.env.example rename to examples/node/js_node_7_up/.env.example diff --git a/examples/node/basic_js_node_7_up/README.md b/examples/node/js_node_7_up/README.md similarity index 100% rename from examples/node/basic_js_node_7_up/README.md rename to examples/node/js_node_7_up/README.md diff --git a/examples/node/basic_js_node_7_up/example.js b/examples/node/js_node_7_up/example.js similarity index 100% rename from examples/node/basic_js_node_7_up/example.js rename to examples/node/js_node_7_up/example.js diff --git a/examples/node/basic_js_node_7_up/package.json b/examples/node/js_node_7_up/package.json similarity index 100% rename from examples/node/basic_js_node_7_up/package.json rename to examples/node/js_node_7_up/package.json diff --git a/examples/node/basic_ts_node_14_up/.env.example b/examples/node/ts_node_14_up/.env.example similarity index 100% rename from examples/node/basic_ts_node_14_up/.env.example rename to examples/node/ts_node_14_up/.env.example diff --git a/examples/node/basic_ts_node_14_up/README.md b/examples/node/ts_node_14_up/README.md similarity index 100% rename from examples/node/basic_ts_node_14_up/README.md rename to examples/node/ts_node_14_up/README.md diff --git a/examples/node/basic_ts_node_14_up/example.ts b/examples/node/ts_node_14_up/example.ts similarity index 100% rename from examples/node/basic_ts_node_14_up/example.ts rename to examples/node/ts_node_14_up/example.ts diff --git a/examples/node/basic_ts_node_14_up/package.json b/examples/node/ts_node_14_up/package.json similarity index 100% rename from examples/node/basic_ts_node_14_up/package.json rename to examples/node/ts_node_14_up/package.json diff --git a/examples/node/basic_ts_node_14_up/tsconfig.json b/examples/node/ts_node_14_up/tsconfig.json similarity index 100% rename from examples/node/basic_ts_node_14_up/tsconfig.json rename to examples/node/ts_node_14_up/tsconfig.json From 9c716f4c1378951dde0cf59d7b94cc430cb5e3d2 Mon Sep 17 00:00:00 2001 From: zyc9012 Date: Sun, 2 Jul 2023 12:54:51 +0800 Subject: [PATCH 46/61] Merge and refine tests --- tests/engines/google_test.ts | 411 ----------------------------------- tests/serpapi_test.ts | 188 ++++++++++++++-- 2 files changed, 168 insertions(+), 431 deletions(-) delete mode 100644 tests/engines/google_test.ts diff --git a/tests/engines/google_test.ts b/tests/engines/google_test.ts deleted file mode 100644 index 145403b..0000000 --- a/tests/engines/google_test.ts +++ /dev/null @@ -1,411 +0,0 @@ -import { loadSync } from "https://deno.land/std@0.170.0/dotenv/mod.ts"; -import { - afterAll, - afterEach, - beforeAll, - describe, - it, -} from "https://deno.land/std@0.170.0/testing/bdd.ts"; -import { delay } from "https://deno.land/std@0.170.0/async/delay.ts"; -import { - assertSpyCallArg, - assertSpyCalls, - spy, - Stub, - stub, -} from "https://deno.land/std@0.170.0/testing/mock.ts"; -import { - assert, - assertArrayIncludes, - assertEquals, - assertRejects, - assertStringIncludes, -} from "https://deno.land/std@0.170.0/testing/asserts.ts"; -import { _internals } from "../../src/utils.ts"; -import { - config, - getHtml, - getHtmlBySearchId, - getJson, - getJsonBySearchId, - MissingApiKeyError, -} from "../../mod.ts"; - -loadSync({ export: true }); -const SERPAPI_TEST_KEY = Deno.env.get("SERPAPI_TEST_KEY") ?? ""; -const HAS_API_KEY = SERPAPI_TEST_KEY.length > 0; -const BASE_URL = Deno.env.get("ENV_TYPE") === "local" - ? "http://localhost:3000" - : "https://serpapi.com"; - -describe("google", { - sanitizeOps: false, // TODO(seb): look into how we can avoid setting these to false - sanitizeResources: false, -}, () => { - let urlStub: Stub; - const engine = "google"; - - beforeAll(() => { - urlStub = stub(_internals, "getBaseUrl", () => BASE_URL); - }); - - afterEach(() => { - config.api_key = null; - }); - - afterAll(() => { - urlStub.restore(); - }); - - it("getJson for an unmetered query (async/await)", async () => { - const response = await getJson({ - engine, - api_key: null, // null to support the "coffee" unmetered query - q: "coffee", - }); - assertArrayIncludes(Object.keys(response).sort(), [ - "organic_results", - "pagination", - "search_information", - "search_metadata", - "search_parameters", - "serpapi_pagination", - ]); - }); - - it("getJson for an unmetered query (callback)", async () => { - const response = await new Promise>>( - (res) => getJson({ engine, api_key: null, q: "coffee" }, res), - ); - assertArrayIncludes(Object.keys(response).sort(), [ - "organic_results", - "pagination", - "search_information", - "search_metadata", - "search_parameters", - "serpapi_pagination", - ]); - }); - - it("getJson with api_key param overrides api key from config", async () => { - const executeSpy = spy(_internals, "execute"); - config.api_key = "test_initial_api_key"; - try { - await getJson({ engine, api_key: "test_override_api_key", q: "coffee" }); - } catch { - // pass - } finally { - executeSpy.restore(); - } - assertSpyCalls(executeSpy, 1); - assertSpyCallArg(executeSpy, 0, 1, { - api_key: "test_override_api_key", - engine, - output: "json", - q: "coffee", - }); - }); - - it("getJson with no api key from params or config", () => { - assertRejects( - async () => await getJson({ engine, api_key: "", q: "coffee" }), - MissingApiKeyError, - ); - assertRejects( - async () => await getJson({ engine, q: "coffee" }), - MissingApiKeyError, - ); - assertRejects( - async () => await getJson({ engine, api_key: undefined, q: "coffee" }), - MissingApiKeyError, - ); - }); - - it("getJson with api key from config", { - ignore: !HAS_API_KEY, - }, async () => { - config.api_key = SERPAPI_TEST_KEY; - const response = await getJson({ engine, q: "serpapi" }); - assertArrayIncludes(Object.keys(response).sort(), [ - "organic_results", - "pagination", - "search_information", - "search_metadata", - "search_parameters", - "serpapi_pagination", - ]); - }); - - it("getJson with engine as first parameter (async/await)", async () => { - const response = await getJson(engine, { - api_key: null, // null to support the "coffee" unmetered query - q: "coffee", - }); - assertArrayIncludes(Object.keys(response).sort(), [ - "organic_results", - "pagination", - "search_information", - "search_metadata", - "search_parameters", - "serpapi_pagination", - ]); - }); - - it("getJson with engine as first parameter (callback)", async () => { - const response = await new Promise>>( - (res) => getJson(engine, { api_key: null, q: "coffee" }, res), - ); - assertArrayIncludes(Object.keys(response).sort(), [ - "organic_results", - "pagination", - "search_information", - "search_metadata", - "search_parameters", - "serpapi_pagination", - ]); - }); - - it("getHtml for an unmetered query (async/await)", async () => { - const response = await getHtml({ engine, api_key: null, q: "coffee" }); - assertStringIncludes(response, ""); - assertStringIncludes(response, ""); - }); - - it("getHtml for an unmetered query (callback)", async () => { - const response = await new Promise>>( - (res) => getHtml({ engine, api_key: null, q: "coffee" }, res), - ); - assertStringIncludes(response, ""); - assertStringIncludes(response, ""); - }); - - it("getHtml with api_key param overrides api key from config", async () => { - const executeSpy = spy(_internals, "execute"); - config.api_key = "test_initial_api_key"; - try { - await getHtml({ engine, api_key: "test_override_api_key", q: "coffee" }); - } catch { - // pass - } finally { - executeSpy.restore(); - } - assertSpyCalls(executeSpy, 1); - assertSpyCallArg(executeSpy, 0, 1, { - api_key: "test_override_api_key", - engine: "google", - output: "html", - q: "coffee", - }); - }); - - it("getHtml with with no api key from params or config", () => { - assertRejects( - async () => await getHtml({ engine, api_key: "", q: "coffee" }), - MissingApiKeyError, - ); - assertRejects( - async () => await getHtml({ engine, q: "coffee" }), - MissingApiKeyError, - ); - assertRejects( - async () => await getHtml({ engine, api_key: undefined, q: "coffee" }), - MissingApiKeyError, - ); - }); - - it("getHtml with api key from config", { - ignore: !HAS_API_KEY, - }, async () => { - config.api_key = SERPAPI_TEST_KEY; - const response = await getHtml({ engine, q: "serpapi" }); - assertStringIncludes(response, ""); - assertStringIncludes(response, ""); - }); - - it("getHtml with async parameter returns json", async () => { - const response = await getHtml({ - engine, - api_key: null, - async: true, - no_cache: true, - q: "coffee", - }); - const json = JSON.parse(response); - assertEquals(Object.keys(json).sort(), [ - "search_metadata", - "search_parameters", - ]); - assertEquals(json["search_metadata"]["status"], "Processing"); - }); - - it("getHtml with engine as first parameter (async/await)", async () => { - const response = await getHtml(engine, { api_key: null, q: "coffee" }); - assertStringIncludes(response, ""); - assertStringIncludes(response, ""); - }); - - it("getHtml with engine as first parameter (callback)", async () => { - const response = await new Promise>>( - (res) => getHtml(engine, { api_key: null, q: "coffee" }, res), - ); - assertStringIncludes(response, ""); - assertStringIncludes(response, ""); - }); - - // get(Json|Html)BySearchId always require a valid API key even for unmetered queries - it("get(Json|Html)BySearchId", { - ignore: !HAS_API_KEY, - }, async (t) => { - let id: string; - - await t.step("initiate async request", async () => { - const response = await getJson({ - engine, - api_key: SERPAPI_TEST_KEY, - async: true, - no_cache: true, // Ensure a new request is sent so we don't get cached results - q: "apple", - gl: "us", - hl: "en", - }); - assertEquals(Object.keys(response).sort(), [ - "search_metadata", - "search_parameters", - ]); - let status; - ({ id, status } = response["search_metadata"]); - assert(id, "Missing search id"); - assertEquals(status, "Processing"); - }); - - await t.step("getJsonBySearchId (async/await)", async () => { - let json; - while (true) { - json = await getJsonBySearchId(id, { api_key: SERPAPI_TEST_KEY }); - const status = json["search_metadata"]["status"]; - if (status === "Processing") { - await delay(500); - } else { - break; - } - } - assertArrayIncludes(Object.keys(json).sort(), [ - "organic_results", - "pagination", - "search_information", - "search_metadata", - "search_parameters", - "serpapi_pagination", - ]); - }); - - await t.step("getJsonBySearchId (callback)", async () => { - let json; - while (true) { - json = await new Promise< - Awaited> - >((res) => getJsonBySearchId(id, { api_key: SERPAPI_TEST_KEY }, res)); - const status = json["search_metadata"]["status"]; - if (status === "Processing") { - await delay(500); - } else { - break; - } - } - assertArrayIncludes(Object.keys(json).sort(), [ - "organic_results", - "pagination", - "search_information", - "search_metadata", - "search_parameters", - "serpapi_pagination", - ]); - }); - - await t.step("getJsonBySearchId with api_key from config", async () => { - let json; - config.api_key = SERPAPI_TEST_KEY; - while (true) { - json = await getJsonBySearchId(id); - const status = json["search_metadata"]["status"]; - if (status === "Processing") { - await delay(500); - } else { - break; - } - } - assertArrayIncludes(Object.keys(json).sort(), [ - "organic_results", - "pagination", - "search_information", - "search_metadata", - "search_parameters", - "serpapi_pagination", - ]); - }); - - await t.step("getHtmlBySearchId (async/await)", async () => { - let html; - while (true) { - html = await getHtmlBySearchId(id, { api_key: SERPAPI_TEST_KEY }); - try { - JSON.parse(html); - } catch { // If parsing fails, it means the request has completed - break; - } - await delay(500); - } - assertStringIncludes(html, ""); - assertStringIncludes(html, ""); - }); - - await t.step("getHtmlBySearchId (callback)", async () => { - let html; - while (true) { - html = await new Promise< - Awaited> - >((res) => getHtmlBySearchId(id, { api_key: SERPAPI_TEST_KEY }, res)); - try { - JSON.parse(html); - } catch { // If parsing fails, it means the request has completed - break; - } - await delay(500); - } - assertStringIncludes(html, ""); - assertStringIncludes(html, ""); - }); - - await t.step("getHtmlBySearchId with api_key from config", async () => { - let html; - config.api_key = SERPAPI_TEST_KEY; - while (true) { - html = await getHtmlBySearchId(id); - try { - JSON.parse(html); - } catch { // If parsing fails, it means the request has completed - break; - } - await delay(500); - } - assertStringIncludes(html, ""); - assertStringIncludes(html, ""); - }); - }); -}); diff --git a/tests/serpapi_test.ts b/tests/serpapi_test.ts index 83ec447..273f7bb 100644 --- a/tests/serpapi_test.ts +++ b/tests/serpapi_test.ts @@ -7,19 +7,30 @@ import { it, } from "https://deno.land/std@0.170.0/testing/bdd.ts"; import { + assert, + assertArrayIncludes, assertEquals, assertExists, assertInstanceOf, assertRejects, + assertStringIncludes, } from "https://deno.land/std@0.170.0/testing/asserts.ts"; -import { Stub, stub } from "https://deno.land/std@0.170.0/testing/mock.ts"; +import { + assertSpyCallArg, + assertSpyCalls, + spy, + Stub, + stub, +} from "https://deno.land/std@0.170.0/testing/mock.ts"; import { _internals } from "../src/utils.ts"; import { BaseResponse, config, getAccount, getHtml, + getHtmlBySearchId, getJson, + getJsonBySearchId, getLocations, InvalidArgumentError, InvalidTimeoutError, @@ -247,10 +258,8 @@ describe("getJson", { ); }); - it("with invalid timeout", { - ignore: !HAS_API_KEY, - }, () => { - config.api_key = SERPAPI_TEST_KEY; + it("with invalid timeout", () => { + config.api_key = "test_api_key"; assertRejects( async () => await getJson({ engine: "google", q: "Paris", timeout: 0 }), InvalidTimeoutError, @@ -315,17 +324,49 @@ describe("getJson", { assertEquals(json2.search_metadata.id, json.search_metadata.id); }); - it("rely on global config", { - ignore: !HAS_API_KEY, - }, async () => { - config.api_key = SERPAPI_TEST_KEY; - const json = await getJson({ + it("rely on global config", async () => { + const executeSpy = spy(_internals, "execute"); + config.api_key = "test_api_key"; + try { + await getJson({ + engine: "google", + q: "Paris", + }); + } catch { + // pass + } finally { + executeSpy.restore(); + } + assertSpyCalls(executeSpy, 1); + assertSpyCallArg(executeSpy, 0, 1, { + api_key: "test_api_key", engine: "google", + output: "json", + q: "Paris", + }); + }); + + it("api_key param overrides global config", async () => { + const executeSpy = spy(_internals, "execute"); + config.api_key = "test_initial_api_key"; + try { + await getJson({ + engine: "google", + api_key: "test_override_api_key", + q: "Paris", + }); + } catch { + // pass + } finally { + executeSpy.restore(); + } + assertSpyCalls(executeSpy, 1); + assertSpyCallArg(executeSpy, 0, 1, { + api_key: "test_override_api_key", + engine: "google", + output: "json", q: "Paris", - timeout: 10000, }); - assertEquals(json.search_metadata["status"], "Success"); - assertExists(json.organic_results); }); }); @@ -442,15 +483,122 @@ describe("getHtml", { assertEquals(html2, html); }); - it("rely on global config", { - ignore: !HAS_API_KEY, - }, async () => { - config.api_key = SERPAPI_TEST_KEY; - const html = await getHtml({ + it("rely on global config", async () => { + const executeSpy = spy(_internals, "execute"); + config.api_key = "test_api_key"; + try { + await getHtml({ + engine: "google", + q: "Paris", + }); + } catch { + // pass + } finally { + executeSpy.restore(); + } + assertSpyCalls(executeSpy, 1); + assertSpyCallArg(executeSpy, 0, 1, { + api_key: "test_api_key", engine: "google", + output: "html", q: "Paris", - timeout: 10000, }); - assertEquals(html.includes("Paris"), true); + }); + + it("api_key param overrides global config", async () => { + const executeSpy = spy(_internals, "execute"); + config.api_key = "test_initial_api_key"; + try { + await getHtml({ + engine: "google", + api_key: "test_override_api_key", + q: "Paris", + }); + } catch { + // pass + } finally { + executeSpy.restore(); + } + assertSpyCalls(executeSpy, 1); + assertSpyCallArg(executeSpy, 0, 1, { + api_key: "test_override_api_key", + engine: "google", + output: "html", + q: "Paris", + }); + }); +}); + +describe("getJsonBySearchId", { + sanitizeOps: false, + sanitizeResources: false, + ignore: !HAS_API_KEY, +}, () => { + let id: string; + + beforeAll(async () => { + const response = await getJson({ + engine: "google", + api_key: SERPAPI_TEST_KEY, + q: "Paris", + }); + let status; + ({ id, status } = response["search_metadata"]); + assert(id, "Missing search id"); + assertEquals(status, "Success"); + }); + + it("getJsonBySearchId (async/await)", async () => { + const json = await getJsonBySearchId(id, { api_key: SERPAPI_TEST_KEY }); + assertArrayIncludes(Object.keys(json).sort(), [ + "organic_results", + ]); + }); + + it("getJsonBySearchId (callback)", async () => { + const json = await new Promise< + Awaited> + >((res) => getJsonBySearchId(id, { api_key: SERPAPI_TEST_KEY }, res)); + assertArrayIncludes(Object.keys(json).sort(), [ + "organic_results", + ]); + }); +}); + +describe("getHtmlBySearchId", { + sanitizeOps: false, + sanitizeResources: false, + ignore: !HAS_API_KEY, +}, () => { + let id: string; + + beforeAll(async () => { + const response = await getJson({ + engine: "google", + api_key: SERPAPI_TEST_KEY, + q: "Paris", + }); + let status; + ({ id, status } = response["search_metadata"]); + assert(id, "Missing search id"); + assertEquals(status, "Success"); + }); + + it("getHtmlBySearchId (async/await)", async () => { + const html = await getHtmlBySearchId(id, { api_key: SERPAPI_TEST_KEY }); + assertStringIncludes(html, ""); + assertStringIncludes(html, ""); + }); + + it("getHtmlBySearchId (callback)", async () => { + const html = await new Promise< + Awaited> + >((res) => getHtmlBySearchId(id, { api_key: SERPAPI_TEST_KEY }, res)); + assertStringIncludes(html, ""); + assertStringIncludes(html, ""); }); }); From 06a90d5aa98167af6bfa577b755a6c7917485dd5 Mon Sep 17 00:00:00 2001 From: zyc9012 Date: Sun, 9 Jul 2023 09:03:42 +0800 Subject: [PATCH 47/61] Run CI on pull request --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 4ca5687..239d75e 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,6 +1,6 @@ name: Build -on: push +on: [push, pull_request] jobs: build: From 62d72969ccb6d5951d114c101ba27401aa34374a Mon Sep 17 00:00:00 2001 From: zyc9012 Date: Wed, 12 Jul 2023 14:34:00 +0800 Subject: [PATCH 48/61] Remove doc for unmetered query --- docs/migrating_from_google_search_results_nodejs.md | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/docs/migrating_from_google_search_results_nodejs.md b/docs/migrating_from_google_search_results_nodejs.md index a358b85..610655e 100644 --- a/docs/migrating_from_google_search_results_nodejs.md +++ b/docs/migrating_from_google_search_results_nodejs.md @@ -52,18 +52,6 @@ migrate over to the `serpapi` npm package. `getHtml` functions instead. - The `SerpApiSearch` class is removed as a public class. -## Fixed - -- Setting the `api_key` parameter to `null` works for unmetered queries. - ```js - // ❌ Previously, error is thrown when api_key is undefined or null. - const engine = new GoogleSearch(); - engine.json({ q: "coffee", api_key: undefined }); - - // ✅ Now, no error is thrown when api_key is null - getJson({ engine: "google", q: "coffee", api_key: null }); - ``` - ## Added - TypeScript support. From 8cdf7a040c13ce863f25b70f015af065af7a40ab Mon Sep 17 00:00:00 2001 From: zyc9012 Date: Wed, 12 Jul 2023 14:38:29 +0800 Subject: [PATCH 49/61] Update dnt --- scripts/build_npm.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/build_npm.ts b/scripts/build_npm.ts index b39746b..4b92248 100644 --- a/scripts/build_npm.ts +++ b/scripts/build_npm.ts @@ -1,4 +1,4 @@ -import { build, emptyDir } from "https://deno.land/x/dnt@0.34.0/mod.ts"; +import { build, emptyDir } from "https://deno.land/x/dnt@0.37.0/mod.ts"; import { version } from "../version.ts"; await emptyDir("./npm"); @@ -21,7 +21,7 @@ await build({ }, compilerOptions: { // https://github.com/microsoft/TypeScript/wiki/Node-Target-Mapping - lib: ["es2017"], + lib: ["ES2017"], target: "ES2017", }, package: { From a5d040b5bba8a337343b303da0946194f4c2bc0f Mon Sep 17 00:00:00 2001 From: zyc9012 Date: Wed, 12 Jul 2023 14:43:30 +0800 Subject: [PATCH 50/61] Use assertArrayIncludes --- tests/serpapi_test.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/serpapi_test.ts b/tests/serpapi_test.ts index 273f7bb..ea58049 100644 --- a/tests/serpapi_test.ts +++ b/tests/serpapi_test.ts @@ -98,7 +98,7 @@ describe("getAccount", { api_key: SERPAPI_TEST_KEY, timeout: 10000, }); - assertEquals(Object.keys(info).sort(), [ + assertArrayIncludes(Object.keys(info).sort(), [ "account_email", "account_id", "account_rate_limit_per_hour", @@ -122,7 +122,7 @@ describe("getAccount", { const info = await new Promise>>( (res) => getAccount({ api_key: SERPAPI_TEST_KEY, timeout: 10000 }, res), ); - assertEquals(Object.keys(info).sort(), [ + assertArrayIncludes(Object.keys(info).sort(), [ "account_email", "account_id", "account_rate_limit_per_hour", @@ -145,7 +145,7 @@ describe("getAccount", { }, async () => { config.api_key = SERPAPI_TEST_KEY; const info = await getAccount(); - assertEquals(Object.keys(info).sort(), [ + assertArrayIncludes(Object.keys(info).sort(), [ "account_email", "account_id", "account_rate_limit_per_hour", From 0c421feab2109f375747c7fdf4d8097c711cf18e Mon Sep 17 00:00:00 2001 From: zyc9012 Date: Wed, 12 Jul 2023 14:51:47 +0800 Subject: [PATCH 51/61] Don't generate lock file --- deno.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/deno.json b/deno.json index 93afa4a..1a6d5f5 100644 --- a/deno.json +++ b/deno.json @@ -28,5 +28,6 @@ "dom.iterable", "deno.ns" ] - } + }, + "lock": false } From 09624f865e6105eb350d53fc3e21bc5380517a4d Mon Sep 17 00:00:00 2001 From: zyc9012 Date: Wed, 12 Jul 2023 14:59:49 +0800 Subject: [PATCH 52/61] Doc clear up --- README.md | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 4f976d4..3cad0a5 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ more. ### Node.js - Supports Node.js 7.10.1 and newer. -- Refer to [this example](examples/node/basic_js_node_7_up) for help. +- Refer to [this example](examples/node/js_node_7_up) for help. ```bash npm install serpapi @@ -40,7 +40,7 @@ getJson({ - If you prefer using the `import` syntax and top-level `await`, you need to use at least Node.js 14.8.0. -- Refer to [this example](examples/node/basic_js_node_14_up) for help. +- Refer to [this example](examples/node/js_node_14_up) for help. You will need to add `"type": "module"` to your `package.json`: @@ -66,10 +66,17 @@ console.log(response); - Import directly from deno.land. - Usage is otherwise the same as above. -- Refer to [this example](examples/deno/basic_ts) for help. +- Refer to [this example](examples/deno) for help. ```ts import { getJson } from "https://deno.land/x/serpapi/mod.ts"; +const response = await getJson({ + engine: "google", + api_key: API_KEY, // Get your API_KEY from https://serpapi.com/manage-api-key + q: "coffee", + location: "Austin, Texas", +}); +console.log(response); ``` ## Features @@ -80,7 +87,6 @@ import { getJson } from "https://deno.land/x/serpapi/mod.ts"; - Promises and async/await support. - Callbacks support. - [Examples in JavaScript/TypeScript on Node.js/Deno using ESM/CommonJS, and more](https://github.com/serpapi/serpapi-javascript/tree/master/examples). -- (Planned) More error classes. ## Configuration From 8982c80b8eb2b8422f9dd50d96222f7c9f5ebb09 Mon Sep 17 00:00:00 2001 From: zyc9012 Date: Wed, 19 Jul 2023 15:05:43 +0800 Subject: [PATCH 53/61] Remove node example for typescript The code is identical to the javascript example --- examples/node/ts_node_14_up/.env.example | 1 - examples/node/ts_node_14_up/README.md | 15 ----------- examples/node/ts_node_14_up/example.ts | 31 ----------------------- examples/node/ts_node_14_up/package.json | 14 ---------- examples/node/ts_node_14_up/tsconfig.json | 10 -------- 5 files changed, 71 deletions(-) delete mode 100644 examples/node/ts_node_14_up/.env.example delete mode 100644 examples/node/ts_node_14_up/README.md delete mode 100644 examples/node/ts_node_14_up/example.ts delete mode 100644 examples/node/ts_node_14_up/package.json delete mode 100644 examples/node/ts_node_14_up/tsconfig.json diff --git a/examples/node/ts_node_14_up/.env.example b/examples/node/ts_node_14_up/.env.example deleted file mode 100644 index fefbd78..0000000 --- a/examples/node/ts_node_14_up/.env.example +++ /dev/null @@ -1 +0,0 @@ -API_KEY=YOUR_API_KEY \ No newline at end of file diff --git a/examples/node/ts_node_14_up/README.md b/examples/node/ts_node_14_up/README.md deleted file mode 100644 index 20e12ba..0000000 --- a/examples/node/ts_node_14_up/README.md +++ /dev/null @@ -1,15 +0,0 @@ -# Basic TypeScript example for Node.js 14 and newer - -## Usage - -1. Setup environment variables - -- Duplicate `.env.example` and name it `.env`. -- Replace `YOUR_API_KEY` with your SerpApi API key. - -2. Install dependencies and run example - -```bash -npm i -npm start -``` diff --git a/examples/node/ts_node_14_up/example.ts b/examples/node/ts_node_14_up/example.ts deleted file mode 100644 index 41c36cd..0000000 --- a/examples/node/ts_node_14_up/example.ts +++ /dev/null @@ -1,31 +0,0 @@ -/** - * Example works for Node.js 14 and newer. - * - Uses ESM imports which is supported from Node.js 13.2.0. - * https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Statements/import#browser_compatibility - * - Uses top-level await which is supported from Node.js 14.8.0. - * https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Operators/await#browser_compatibility - */ - -import * as Dotenv from "dotenv"; -import { config, getJson } from "serpapi"; - -Dotenv.config(); -const apiKey = process.env.API_KEY; - -const params = { - engine: "google", - q: "Coffee", - api_key: apiKey, -}; - -// Show result as JSON (async/await) -const response1 = await getJson(params); -console.log(response1["organic_results"]); - -// Show result as JSON (callback) -getJson(params, (json) => console.log(json["organic_results"])); - -// Use global config -config.api_key = apiKey; -const response2 = await getJson({ engine: "google", q: "Coffee" }); -console.log(response2["organic_results"]); diff --git a/examples/node/ts_node_14_up/package.json b/examples/node/ts_node_14_up/package.json deleted file mode 100644 index e1ef349..0000000 --- a/examples/node/ts_node_14_up/package.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "type": "module", - "dependencies": { - "dotenv": "*", - "serpapi": "*" - }, - "devDependencies": { - "@types/node": "*", - "typescript": "*" - }, - "scripts": { - "start": "npx ts-node example.ts" - } -} diff --git a/examples/node/ts_node_14_up/tsconfig.json b/examples/node/ts_node_14_up/tsconfig.json deleted file mode 100644 index 8ce0fc8..0000000 --- a/examples/node/ts_node_14_up/tsconfig.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "compilerOptions": { - "module": "ESNext", - "target": "ESNext", - "moduleResolution": "Node" - }, - "ts-node": { - "esm": true - } -} From b92d13849b76889135976c6970d557994d10c62b Mon Sep 17 00:00:00 2001 From: zyc9012 Date: Wed, 19 Jul 2023 15:30:21 +0800 Subject: [PATCH 54/61] Add pagination example --- CONTRIBUTING.md | 2 +- README.md | 9 +++++++ examples/deno/README.md | 3 ++- .../deno/{example.ts => basic_example.ts} | 0 examples/deno/pagination_example.ts | 15 +++++++++++ .../{example.js => basic_example.js} | 0 examples/node/js_node_14_up/package.json | 2 +- .../node/js_node_14_up/pagination_example.js | 23 +++++++++++++++++ .../{example.js => basic_example.js} | 0 examples/node/js_node_7_up/package.json | 2 +- .../node/js_node_7_up/pagination_example.js | 25 +++++++++++++++++++ 11 files changed, 77 insertions(+), 4 deletions(-) rename examples/deno/{example.ts => basic_example.ts} (100%) create mode 100644 examples/deno/pagination_example.ts rename examples/node/js_node_14_up/{example.js => basic_example.js} (100%) create mode 100644 examples/node/js_node_14_up/pagination_example.js rename examples/node/js_node_7_up/{example.js => basic_example.js} (100%) create mode 100644 examples/node/js_node_7_up/pagination_example.js diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 9fb98dd..8b52912 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -111,7 +111,7 @@ To run [examples](./examples/) on your local source files, follow these steps. "serpapi": "../../../npm" }, "scripts": { - "start": "node example.js" + "start": "node basic_example.js" } } ``` diff --git a/README.md b/README.md index 3cad0a5..02ea472 100644 --- a/README.md +++ b/README.md @@ -108,6 +108,15 @@ await getJson({ engine: "google", q: "coffee" }); // uses the API key defined in await getJson({ engine: "google", api_key: API_KEY_2, q: "coffee" }); // API_KEY_2 will be used ``` +## Pagination + +There's no built-in pagination support in this library, but you can always do it +manually. You can find pagination examples here: + +- [Pagination example (Node.js >= 7)](examples/node/js_node_7_up/pagination_example.js) +- [Pagination example (Node.js >= 14)](examples/node/js_node_14_up/pagination_example.js) +- [Pagination example (Deno)](examples/deno/pagination_example.ts) + ## Functions diff --git a/examples/deno/README.md b/examples/deno/README.md index 811641a..15f6155 100644 --- a/examples/deno/README.md +++ b/examples/deno/README.md @@ -10,7 +10,8 @@ 2. Run the example ``` -deno run example.ts +deno run basic_example.ts +deno run pagination_example.ts ``` ## Notes diff --git a/examples/deno/example.ts b/examples/deno/basic_example.ts similarity index 100% rename from examples/deno/example.ts rename to examples/deno/basic_example.ts diff --git a/examples/deno/pagination_example.ts b/examples/deno/pagination_example.ts new file mode 100644 index 0000000..9c683f9 --- /dev/null +++ b/examples/deno/pagination_example.ts @@ -0,0 +1,15 @@ +import { loadSync } from "https://deno.land/std@0.173.0/dotenv/mod.ts"; +import { config, getJson } from "../../mod.ts"; + +const { API_KEY: apiKey } = loadSync(); +config.api_key = apiKey; + +// Get the first page +const page = await getJson({ engine: "google", q: "Coffee" }); +// Parse SerpApi search URL to the next page +const nextUrl = new URL(page.serpapi_pagination.next); +// Extract the request parameters +const nextParams = Object.fromEntries(nextUrl.searchParams); +// Get the next page +const nextPage = await getJson(nextParams); +console.log(nextPage); diff --git a/examples/node/js_node_14_up/example.js b/examples/node/js_node_14_up/basic_example.js similarity index 100% rename from examples/node/js_node_14_up/example.js rename to examples/node/js_node_14_up/basic_example.js diff --git a/examples/node/js_node_14_up/package.json b/examples/node/js_node_14_up/package.json index 055dcb9..0dc44d3 100644 --- a/examples/node/js_node_14_up/package.json +++ b/examples/node/js_node_14_up/package.json @@ -5,6 +5,6 @@ "serpapi": "*" }, "scripts": { - "start": "node example.js" + "start": "node basic_example.js" } } diff --git a/examples/node/js_node_14_up/pagination_example.js b/examples/node/js_node_14_up/pagination_example.js new file mode 100644 index 0000000..07c705a --- /dev/null +++ b/examples/node/js_node_14_up/pagination_example.js @@ -0,0 +1,23 @@ +/** + * Example works for Node.js 14 and newer. + * - Uses ESM imports which is supported from Node.js 13.2.0. + * https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Statements/import#browser_compatibility + * - Uses top-level await which is supported from Node.js 14.8.0. + * https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Operators/await#browser_compatibility + */ + +import * as Dotenv from "dotenv"; +import { config, getJson } from "serpapi"; + +Dotenv.config(); +config.api_key = process.env.API_KEY; + +// Get the first page +const page = await getJson({ engine: "google", q: "Coffee" }); +// Parse SerpApi search URL to the next page +const nextUrl = new URL(page.serpapi_pagination.next); +// Extract the request parameters +const nextParams = Object.fromEntries(nextUrl.searchParams); +// Get the next page +const nextPage = await getJson(nextParams); +console.log(nextPage); \ No newline at end of file diff --git a/examples/node/js_node_7_up/example.js b/examples/node/js_node_7_up/basic_example.js similarity index 100% rename from examples/node/js_node_7_up/example.js rename to examples/node/js_node_7_up/basic_example.js diff --git a/examples/node/js_node_7_up/package.json b/examples/node/js_node_7_up/package.json index f7ceb03..ff501be 100644 --- a/examples/node/js_node_7_up/package.json +++ b/examples/node/js_node_7_up/package.json @@ -4,6 +4,6 @@ "serpapi": "*" }, "scripts": { - "start": "node example.js" + "start": "node basic_example.js" } } diff --git a/examples/node/js_node_7_up/pagination_example.js b/examples/node/js_node_7_up/pagination_example.js new file mode 100644 index 0000000..fdaebf8 --- /dev/null +++ b/examples/node/js_node_7_up/pagination_example.js @@ -0,0 +1,25 @@ +/** + * Example works for Node.js 7 and newer. + */ + +const Dotenv = require("dotenv"); +const { config, getJson } = require("serpapi"); +const url = require("url"); +const qs = require("querystring"); + +Dotenv.config(); +config.api_key = process.env.API_KEY; + +const run = async () => { + // Get the first page + const page = await getJson({ engine: "google", q: "Coffee" }); + // Parse SerpApi search URL to the next page + const nextUrl = url.parse(page.serpapi_pagination.next); + // Extract the request parameters + const nextParams = qs.parse(nextUrl.query); + // Get the next page + const nextPage = await getJson(nextParams); + console.log(nextPage); +}; + +run(); From 646b1cf6c425dad141195f6182035d0bd53e23aa Mon Sep 17 00:00:00 2001 From: tato Date: Wed, 19 Jul 2023 20:26:03 +0800 Subject: [PATCH 55/61] Update wording Co-authored-by: Milos Djurdjevic --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 02ea472..d21e738 100644 --- a/README.md +++ b/README.md @@ -110,8 +110,8 @@ await getJson({ engine: "google", api_key: API_KEY_2, q: "coffee" }); // API_KEY ## Pagination -There's no built-in pagination support in this library, but you can always do it -manually. You can find pagination examples here: +Built-in pagination is not supported. Please refer to our pagination examples +for a manual approach: - [Pagination example (Node.js >= 7)](examples/node/js_node_7_up/pagination_example.js) - [Pagination example (Node.js >= 14)](examples/node/js_node_14_up/pagination_example.js) From c0bb9e2ca41fdf3e147f1173d0cbc9d1fb05a709 Mon Sep 17 00:00:00 2001 From: zyc9012 Date: Wed, 26 Jul 2023 13:31:19 +0800 Subject: [PATCH 56/61] Bump 2.0.0 --- version.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.ts b/version.ts index fc5b7aa..f2b3ca5 100644 --- a/version.ts +++ b/version.ts @@ -4,4 +4,4 @@ * * Changing this value creates a new release. */ -export const version = "1.1.1"; +export const version = "2.0.0"; From 1cdcf24110b754106677bc51d1423b54d8bce863 Mon Sep 17 00:00:00 2001 From: Hilman Ramadhan <4522467+hilmanski@users.noreply.github.com> Date: Tue, 5 Dec 2023 10:24:37 +0800 Subject: [PATCH 57/61] Update relative link to absolute.md To make sure our Integration page didn't return 404 page: https://serpapi.com/integrations/javascript Resolves: https://github.com/serpapi/public-roadmap/issues/1254 --- README.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index d21e738..40145ec 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ more. ### Node.js - Supports Node.js 7.10.1 and newer. -- Refer to [this example](examples/node/js_node_7_up) for help. +- Refer to [this example](https://github.com/serpapi/serpapi-javascript/tree/master/examples/node/js_node_7_up) for help. ```bash npm install serpapi @@ -40,7 +40,7 @@ getJson({ - If you prefer using the `import` syntax and top-level `await`, you need to use at least Node.js 14.8.0. -- Refer to [this example](examples/node/js_node_14_up) for help. +- Refer to [this example](https://github.com/serpapi/serpapi-javascript/tree/master/examples/node/js_node_14_up) for help. You will need to add `"type": "module"` to your `package.json`: @@ -66,7 +66,7 @@ console.log(response); - Import directly from deno.land. - Usage is otherwise the same as above. -- Refer to [this example](examples/deno) for help. +- Refer to [this example](https://github.com/serpapi/serpapi-javascript/tree/master/examples/deno) for help. ```ts import { getJson } from "https://deno.land/x/serpapi/mod.ts"; @@ -113,9 +113,9 @@ await getJson({ engine: "google", api_key: API_KEY_2, q: "coffee" }); // API_KEY Built-in pagination is not supported. Please refer to our pagination examples for a manual approach: -- [Pagination example (Node.js >= 7)](examples/node/js_node_7_up/pagination_example.js) -- [Pagination example (Node.js >= 14)](examples/node/js_node_14_up/pagination_example.js) -- [Pagination example (Deno)](examples/deno/pagination_example.ts) +- [Pagination example (Node.js >= 7)](https://github.com/serpapi/serpapi-javascript/tree/master/examples/node/js_node_7_up/pagination_example.js) +- [Pagination example (Node.js >= 14)](https://github.com/serpapi/serpapi-javascript/tree/master/examples/node/js_node_14_up/pagination_example.js) +- [Pagination example (Deno)](https://github.com/serpapi/serpapi-javascript/tree/master/examples/deno/pagination_example.ts) ## Functions From 193c27b3f4831fdc60f026779c0dcbab8a24fb89 Mon Sep 17 00:00:00 2001 From: Thomas Hurst Date: Fri, 8 Mar 2024 19:51:33 +0000 Subject: [PATCH 58/61] Handle HTTP chunking across multi-byte boundaries When a response contains multi-byte UTF-8 characters that cross chunk boundaries, appending the two halves to the intermediate string results in two Unicode Replacement Characters being inserted instead of the two halves being joined. Set the response encoding to UTF-8 to ensure it buffers partial characters between chunks. Resolves #22 Co-authored-by: zyc9012 --- src/utils.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/utils.ts b/src/utils.ts index ef49a2a..a05fa05 100644 --- a/src/utils.ts +++ b/src/utils.ts @@ -63,6 +63,7 @@ export function execute( return new Promise((resolve, reject) => { let timer: number; const req = https.get(url, (resp) => { + resp.setEncoding("utf8"); let data = ""; // A chunk of data has been recieved. From 7610f9f4e9a683e9dbf439d433cc481f0b438a37 Mon Sep 17 00:00:00 2001 From: Thomas Hurst Date: Mon, 11 Mar 2024 21:29:59 +0000 Subject: [PATCH 59/61] deno fmt README.md --- README.md | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 40145ec..6838391 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,9 @@ more. ### Node.js - Supports Node.js 7.10.1 and newer. -- Refer to [this example](https://github.com/serpapi/serpapi-javascript/tree/master/examples/node/js_node_7_up) for help. +- Refer to + [this example](https://github.com/serpapi/serpapi-javascript/tree/master/examples/node/js_node_7_up) + for help. ```bash npm install serpapi @@ -40,7 +42,9 @@ getJson({ - If you prefer using the `import` syntax and top-level `await`, you need to use at least Node.js 14.8.0. -- Refer to [this example](https://github.com/serpapi/serpapi-javascript/tree/master/examples/node/js_node_14_up) for help. +- Refer to + [this example](https://github.com/serpapi/serpapi-javascript/tree/master/examples/node/js_node_14_up) + for help. You will need to add `"type": "module"` to your `package.json`: @@ -66,7 +70,9 @@ console.log(response); - Import directly from deno.land. - Usage is otherwise the same as above. -- Refer to [this example](https://github.com/serpapi/serpapi-javascript/tree/master/examples/deno) for help. +- Refer to + [this example](https://github.com/serpapi/serpapi-javascript/tree/master/examples/deno) + for help. ```ts import { getJson } from "https://deno.land/x/serpapi/mod.ts"; From d9385fcb1879fcabcd4d4db36046c0807333ce6a Mon Sep 17 00:00:00 2001 From: Thomas Hurst Date: Mon, 11 Mar 2024 21:38:19 +0000 Subject: [PATCH 60/61] Fix CI for earlier node versions Node hardcodes its default TLS certificates, meaning older versions do not support the root certificate currently used by LetsEncrypt. Use --use-openssl-ca to enable the use of the system's OpenSSL certificates. Since node v7 doesn't support NODE_OPTIONS, conditionally modify the test script with sed. --- .github/workflows/build.yml | 13 +++++++++++-- smoke_tests/commonjs/package.json | 3 ++- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 239d75e..83b4b27 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,6 +1,6 @@ name: Build -on: [push, pull_request] +on: [push] jobs: build: @@ -52,6 +52,15 @@ jobs: strategy: matrix: node-version: [7.x, 8.x, 9.x, 10.x, 11.x, 12.x, 13.x, 14.x, 15.x, 16.x, 17.x, 18.x, 19.x] + include: + - command: test + - command: test:use-openssl-ca + node-version: 7.x + - command: test:use-openssl-ca + node-version: 8.x + - command: test:use-openssl-ca + node-version: 9.x + steps: - name: Checkout repo uses: actions/checkout@v3 @@ -76,7 +85,7 @@ jobs: run: | cd smoke_tests/commonjs npm i - npm test + npm run ${{ matrix.command }} smoke-tests-esm: name: "Smoke tests (ESM)" diff --git a/smoke_tests/commonjs/package.json b/smoke_tests/commonjs/package.json index f229cf1..d1971c6 100644 --- a/smoke_tests/commonjs/package.json +++ b/smoke_tests/commonjs/package.json @@ -4,6 +4,7 @@ "serpapi": "../../npm" }, "scripts": { - "test": "node commonjs.js" + "test": "node commonjs.js", + "test:use-openssl-ca": "node --use-openssl-ca commonjs.js" } } From 38dcecba04c1a96dacf7e7847efc77e74bd97513 Mon Sep 17 00:00:00 2001 From: zyc9012 Date: Tue, 12 Mar 2024 17:00:28 +0800 Subject: [PATCH 61/61] Bump 2.1.0 --- version.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/version.ts b/version.ts index f2b3ca5..c640f4c 100644 --- a/version.ts +++ b/version.ts @@ -4,4 +4,4 @@ * * Changing this value creates a new release. */ -export const version = "2.0.0"; +export const version = "2.1.0";