diff --git a/CHANGELOG.md b/CHANGELOG.md index baff711..c2e478e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,33 @@ See [releases](https://github.com/RandomFractals/duckdb-sql-tools/releases) for the `duckdb-sql-tools-x.x.x.vsix` extension package downloads. +## v1.6.0 - [2024-07-14] + +- Update DuckDB SQL Tools Preview to support DuckDB v0.10 +- Update DuckDB SQL Tools Preview docs with DuckDB v0.10 support info +- Package and publish DuckDB SQL Tools Preview v1.6.0 with DuckDB v0.10.2 support + +## v1.5.0 - [2024-07-12] + +- Convert `BigInt` values and arrays to `Number` in DuckDB `TableData` query results for display in SQL Tools records view +- Update DuckDB version support message in DuckDB SQL Tools Preview +- Update DuckDB SQL Tools Preview `LICENSE.txt` +- Package and publish DuckDB SQL Tools Preview v1.5.0 with `BigInt` support + +## v1.4.0 - [2023-12-01] + +- Update DuckDB SQL Tools to support the latest DuckDB 0.9.2 version +- Remove VS Code Memory Limit section from DuckDB SQL Tools doc +- Add new SQLite Pro Tools info to the Pro Data Tools sections in DuckDB SQL Tools `README.md` +- Package and publish DuckDB SQL Tools v1.4.0 with DuckDB v0.9.2 support + +## v1.3.0 - [2023-10-27] + +- Update DuckDB SQL Tools and demo DuckDB configs to use new DuckDB v0.9.1 NodeJS libraries +- Update DuckDB SQL Tools `chinook-duckdb.ipynb` and generated demo `chinook.duckdb` to use DuckDB v0.9.1 +- Update DuckDB SQL Tools documentation with new DuckDB v0.9.1 support info +- Package and publish DuckDB SQL Tools v1.3.0 with DuckDB v0.9.1 support + ## v1.2.0 - [2023-10-06] - Update free public DuckDB SQL Tools Preview extension to support DuckDB v0.8.1 diff --git a/LICENSE.txt b/LICENSE.txt index 31fe905..8babd72 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -1,4 +1,4 @@ -Copyright © 2023 Random Fractals, Inc. All rights reserved. +Copyright © 2024 Random Fractals, Inc. All rights reserved. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, diff --git a/README.md b/README.md index 75b5b88..721e7f5 100644 --- a/README.md +++ b/README.md @@ -3,10 +3,8 @@ [![Version](https://img.shields.io/visual-studio-marketplace/v/RandomFractalsInc.duckdb-sql-tools.svg?color=orange&style=?style=for-the-badge&logo=visual-studio-code)](https://marketplace.visualstudio.com/items?itemName=RandomFractalsInc.duckdb-sql-tools) [![Installs](https://img.shields.io/visual-studio-marketplace/i/RandomFractalsInc.duckdb-sql-tools.svg?color=orange)](https://marketplace.visualstudio.com/items?itemName=RandomFractalsInc.duckdb-sql-tools) [![Downloads](https://img.shields.io/visual-studio-marketplace/d/RandomFractalsInc.duckdb-sql-tools.svg?color=orange)](https://marketplace.visualstudio.com/items?itemName=RandomFractalsInc.duckdb-sql-tools) - - https://ko-fi.com/dataPixy -[DuckDB SQL Tools](https://marketplace.visualstudio.com/items?itemName=RandomFractalsInc.duckdb-sql-tools) *Preview* extension adds basic [DuckDB](https://duckdb.org/) SQL support to [VS Code](https://code.visualstudio.com/) IDE, and provides database schema and SQL query API and user interfaces for the popular [SQL Tools](https://vscode-sqltools.mteixeira.dev/en/home/) extension, SQL query editor, SQL language server, and data processing tools. +[DuckDB SQL Tools](https://marketplace.visualstudio.com/items?itemName=RandomFractalsInc.duckdb-sql-tools) *Preview* extension adds basic [DuckDB](https://duckdb.org/) SQL support to [VS Code](https://code.visualstudio.com/) IDE. This DuckDB VS Code extension provides DuckDB connections and main database schema tree view, SQL query API and user interfaces for the popular [SQL Tools](https://vscode-sqltools.mteixeira.dev/en/home/) extension, SQL query editor, SQL language server, and data processing tools. ![DuckDB SQL Tools](https://github.com/RandomFractals/duckdb-sql-tools/blob/main/docs/images/duckdb-sql-tools.gif?raw=true) @@ -14,11 +12,11 @@ [DuckDB](https://duckdb.org/docs/) is an in-process SQL [OLAP](https://en.wikipedia.org/wiki/Online_analytical_processing) database management system that uses vectorized data engine for optimized analytics and parallel query processing with extensive SQL support and direct `Parquet`, `CSV`, and `JSON` data loading and query capabilites for local and remote data sources. -[DuckDB SQL Tools](https://marketplace.visualstudio.com/items?itemName=RandomFractalsInc.duckdb-sql-tools) *Peview* extension v1.2.0 adds the following basic DuckDB SQL capabilities to VS Code IDE for working with DuckDB instances: +[DuckDB SQL Tools](https://marketplace.visualstudio.com/items?itemName=RandomFractalsInc.duckdb-sql-tools) *Preview* extension v1.6.0 adds the following basic DuckDB SQL capabilities to VS Code IDE for working with DuckDB instances: -- **Connect** to a local DuckDB v0.8.1 instance +- **Connect** to a local DuckDB v0.10.2 instance - **Create** new in-memory DuckDB instance -- **View** DuckDB v0.8.1 tables, columns, and views +- **View** DuckDB v0.10.2 tables, columns, and views - **Run** SQL queries on open DuckDB connections - **Attach** SQLite database files to in-memory DuckDB instances - **Query** remote `CSV` and `Parquet` data files with [DuckDB HTTPFS](https://duckdb.org/docs/extensions/httpfs.html) extension @@ -41,15 +39,15 @@ Alternatively, you can install this extension directly in VS Code IDE from Exten Users of [VS Codium](https://vscodium.com/) and other VS Code based IDEs can install **DuckDB SQL Tools** *Preview* extension using `duckdb-sql-tools-x.x.x.vsix` extension package from the **Assets** section in published [releases](https://github.com/RandomFractals/duckdb-sql-tools/releases) of this extension on GitHub. Follow [install from .vsix](https://code.visualstudio.com/docs/editor/extension-marketplace#_install-from-a-vsix) instructions in your VS Code extensions compatible IDE or online container service to install it. -**Note**: **DuckDB SQL Tools** use [DuckDB Node.js API](https://github.com/duckdb/duckdb/tree/master/tools/nodejs) and require a local [Node.js installation](https://nodejs.org/en/download) to query DuckDB instances. +**Note**: **DuckDB SQL Tools** use [DuckDB Node.js API](https://github.com/duckdb/duckdb-node) and require a local [Node.js installation](https://nodejs.org/en/download) to query DuckDB instances. -Download and install **Node.js** from the official [node.js downloads](https://nodejs.org/en/download) page. Node.js is used as a local web server to host local data and DuckDB files. Node.js will install **npm** tool we use to install `duckdb-async` library to establish DuckDB connections. +Download and install **Node.js** from the official [Node.js downloads](https://nodejs.org/en/download) page. Node.js is used as a local web server to host local data and DuckDB files. Node.js will install **npm** tool we use to install `duckdb-async` library to establish DuckDB connections. We use Node.js DuckDB library instead of the limited DuckDB WASM browser JS library other similar DuckDB data tools use to enable faster data imports and loading via multi-threaded [`node-gyp`](https://github.com/nodejs/node-gyp) DuckDB native API interface. # VS Code SQL and Pro Data Tools -The following sections provide a summary of the popular SQL Tools VS Code extension, database drivers, our public **Random Fractals Inc.** [Data Tools](https://marketplace.visualstudio.com/publishers/RandomFractalsInc) extensions, and new premium [Pro Data Tools](https://twitter.com/search?q=(%23ProDataTools)%20(from%3ATarasNovak)&src=typed_query) available to our [Pro sponsors](https://github.com/sponsors/RandomFractals/sponsorships?sponsor=RandomFractals&tier_id=18884) on github. +The following sections provide a summary of the popular SQL Tools VS Code extension, database drivers, our public **Random Fractals Inc.** [Data Tools](https://marketplace.visualstudio.com/publishers/RandomFractalsInc) extensions, and new premium [Pro Data Tools](https://github.com/RandomFractals/pro-data-tools#pro-data-tools) available to our [Pro sponsors](https://github.com/sponsors/RandomFractals/sponsorships?tier_id=18884) on GitHub. ## SQL Tools @@ -63,67 +61,75 @@ SQL Tools support many popular databases via the [official and community databas ## Random Fractals Data Tools -[Random Fractals](https://twitter.com/search?q=(%23RandomFractalsInc)%20(from%3ATarasNovak)&src=typed_query&f=top) Inc. [**Data Tools**](https://twitter.com/search?q=(%23DataTools)%20(from%3ATarasNovak)&src=typed_query) 🛠️ is a collection of public [data visualization extensions](https://marketplace.visualstudio.com/search?term=dataViz&target=VSCode&category=All%20categories&sortBy=Relevance), [data viewers](https://marketplace.visualstudio.com/search?term=data%20viewer&target=VSCode&category=All%20categories&sortBy=Relevance), VS Code [Notebook renderers](https://marketplace.visualstudio.com/search?term=notebook%20renderer&target=VSCode&category=All%20categories&sortBy=Relevance), and [code snippets](https://marketplace.visualstudio.com/search?term=Random%20Fractals%20code%20snippets&target=VSCode&category=All%20categories&sortBy=Relevance) for devs and data scientists using VS Code IDE, published under our [Random Fractals Inc.](https://marketplace.visualstudio.com/publishers/RandomFractalsInc) ☂️ org. +[Random Fractals](https://github.com/RandomFractals) Inc. [**Data Tools**](https://marketplace.visualstudio.com/publishers/RandomFractalsInc) 🛠️ is a collection of public [data visualization extensions](https://marketplace.visualstudio.com/search?term=dataViz&target=VSCode&category=All%20categories&sortBy=Relevance), [data viewers](https://marketplace.visualstudio.com/search?term=data%20viewer&target=VSCode&category=All%20categories&sortBy=Relevance), VS Code [Notebook renderers](https://marketplace.visualstudio.com/search?term=notebook%20renderer&target=VSCode&category=All%20categories&sortBy=Relevance), and [code snippets](https://marketplace.visualstudio.com/search?term=Random%20Fractals%20code%20snippets&target=VSCode&category=All%20categories&sortBy=Relevance) for devs and data scientists using VS Code IDE, published under our [Random Fractals Inc.](https://marketplace.visualstudio.com/publishers/RandomFractalsInc) ☂️ org. ![Random Fractals Data Tools](https://github.com/RandomFractals/duckdb-sql-tools/blob/main/docs/images/random-fractals-data-tools.png?raw=true) ## Pro Data Tools -**Pro Data Tools** is a new Premium set of SQL and [PRQL](https://prql-lang.org/) Code Lenses and Data Viewers created for our monthly [**Pro sponsors**](https://github.com/sponsors/RandomFractals/sponsorships?sponsor=RandomFractals&tier_id=18884) on GitHub to enhance SQL development and SQL statements execution runtime workflow in VS Code IDE, connected to the different database management systems via [SQL Tools extension](https://marketplace.visualstudio.com/items?itemName=mtxr.sqltools), database plugins, and [SQL Tools drivers](https://marketplace.visualstudio.com/search?term=tag%3Asqltools-driver&target=VSCode&category=All%20categories&sortBy=Rating). +**Pro Data Tools** is our new Premium set of SQL and [PRQL](https://prql-lang.org/) Code Lenses and Data Viewers created for our monthly [**Pro sponsors**](https://github.com/sponsors/RandomFractals/sponsorships?tier_id=18884) on GitHub to enhance SQL development and SQL statements execution runtime workflow in VS Code IDE, connected to the different database management systems via [SQL Tools extension](https://marketplace.visualstudio.com/items?itemName=mtxr.sqltools), database plugins, and [SQL Tools drivers](https://marketplace.visualstudio.com/search?term=tag%3Asqltools-driver&target=VSCode&category=All%20categories&sortBy=Rating). ![Pro Data Tools](https://github.com/RandomFractals/pro-data-tools/blob/main/docs/images/pro-data-tools.png?raw=true) +![SQLite Pro Tools](https://github.com/RandomFractals/pro-data-tools/blob/main/docs/images/sqlite-tools-extension.png?raw=true) + ### DuckDB Pro Tools -**Pro Data Tools** include [DuckDB Pro Tools](https://www.linkedin.com/feed/hashtag/?keywords=duckdbpro) extension that adds advanced [DuckDB](https://duckdb.org/) connection features and support to VSCode IDE, and provides database schemas display, DuckDB extensions and settings views, information schema and catalog views, SQL query API and user interfaces integrated with the popular SQL Tools extension, SQL query editor, SQL language server, and VS Code data processing tools. +**Pro Data Tools** include [DuckDB Pro Tools](https://github.com/RandomFractals/pro-data-tools/blob/main/duckdb-tools.md#duckdb-pro-tools) extension that adds advanced [DuckDB](https://duckdb.org/) connection features and support to VSCode IDE, and provides database schemas display, DuckDB extensions and settings views, information schema and catalog views, SQL query API and user interfaces integrated with the popular SQL Tools extension, SQL query editor, SQL language server, and VS Code data processing tools. ![DuckDB Pro Tools Views](https://github.com/RandomFractals/duckdb-sql-tools/blob/main/docs/images/duckdb-pro-tools-views.gif?raw=true) -The latest version of [DuckDB Pro Tools](https://www.linkedin.com/feed/hashtag/?keywords=duckdbpro) extension adds the following capabilities to VS Code IDE for working with [DuckDB v0.8](https://duckdb.org/2023/05/17/announcing-duckdb-080.html) instances: +The latest version of [DuckDB Pro Tools](https://github.com/RandomFractals/pro-data-tools/blob/main/duckdb-tools.md#duckdb-pro-tools) extension adds the following capabilities to VS Code IDE for working with [DuckDB v0.10](https://duckdb.org/2024/02/13/announcing-duckdb-0100.html) instances: -- **Connect** to local DuckDB [v0.8.0](https://github.com/duckdb/duckdb/releases/tag/v0.8.0) and [v0.8.1](https://github.com/duckdb/duckdb/releases/tag/v0.8.1) instances +- **Connect** to local DuckDB [v0.10.2](https://github.com/duckdb/duckdb/releases/tag/v0.10.2) instances - **Create** new in-memory DuckDB instance - **Import** local and remote `CSV`, `JSON` and `Parquet` data files into in-memory DuckDB instance for exploratory data analysis (EDA) -- **View** DuckDB v0.8.1 databases, schemas, tables, columns, views, indexes, sequences, extensions, settings, functions, types and keywords in SQL Tools Connections Explorer +- **View** DuckDB v0.10.2 databases, schemas, tables, columns, views, indexes, sequences, extensions, settings, functions, types and keywords in SQL Tools Connections Explorer - **Run** SQL queries on active DuckDB connections - **Attach** SQLite database files to in-memory DuckDB instances to run analytical queries - **Query** remote `CSV`, `Parquet`, and `JSON` data files with [DuckDB HTTPFS](https://duckdb.org/docs/extensions/httpfs.html) extension and new [DuckDB JSON](https://duckdb.org/2023/03/03/json.html) extension - **Create** in-memory DuckDB tables from remote data sources and query results -- **Manage** DuckDB v0.8.1 connections in SQL Tools Connections Explorer +- **Manage** DuckDB v0.10.2 connections in SQL Tools Connections Explorer - **Auto-complete** SQL keywords, DuckDB instance table names, column names, and view names for active DuckDB connections in VS Code SQL editor - **Save** named SQL query Bookmarks - **Use** SQL Query History - **Export** DuckDB query results in `CSV` and `JSON` data formats -- **Use** [PRQL Code Lens](https://github.com/RandomFractals/prql-pro-tools#prql-code-lens) from our new [PRQL Pro Tools](https://twitter.com/search?q=(%23PRQLProTools)%20(from%3ATarasNovak)&src=typed_query&f=live) collection to generate and run SQL queries on active DuckDB connection -- **Explore** new [`employees.duckdb`](https://github.com/RandomFractals/duckdb-pro-tools/tree/main/data/employees) demo data, PRQL and SQL sample queries -- **Run** sample [`chicago-crimes`](https://github.com/RandomFractals/duckdb-pro-tools/tree/main/data/chicago-crimes) and [`gbif-observations`](https://github.com/RandomFractals/duckdb-pro-tools/tree/main/data/gbif-observations) [PRQL](https://prql-lang.org/) and SQL queries on Github and AWS S3 hosted `parquet` data files +- **Use** [PRQL Code Lens](https://github.com/RandomFractals/prql-pro-tools#prql-code-lens) from our new [PRQL Pro Tools](https://github.com/RandomFractals/pro-data-tools/blob/main/prql-tools.md) collection to generate and run SQL queries on active DuckDB connection +- **Explore** new `employees.duckdb` demo data, PRQL and SQL sample queries +- **Run** `chicago-crimes` and `gbif-observations` [PRQL](https://prql-lang.org/) and SQL queries on GitHub and AWS S3 hosted `parquet` data files - **Use** over 30 new `DuckDB Tools` [views and metadata shortcut commands](https://github.com/RandomFractals/pro-data-tools/blob/main/duckdb-tools.md#duckdb-view-commands) from VS Code `Command Palette...` ### DuckDB Pro Tools View Commands ![DuckDB Pro Tools View Commands](https://github.com/RandomFractals/duckdb-sql-tools/blob/main/docs/images/duckdb-pro-tools-view-commands.gif?raw=true) +## SQLite Pro Tools + +**Pro Data Tools** VS Code extensions also include [**SQLite Pro Tools**](https://github.com/RandomFractals/pro-data-tools/blob/main/sqlite-tools.md). **SQLite Pro Tools** provide over a dozen of common SQLite database shortcut [Commands](https://github.com/RandomFractals/pro-data-tools/blob/main/sqlite-tools.md#commands), display table column data type and primary key indicators, SQLite database [Indexes](https://github.com/RandomFractals/pro-data-tools/blob/main/sqlite-tools.md#indexes), [Modules](https://github.com/RandomFractals/pro-data-tools/blob/main/sqlite-tools.md#modules), [Configuration Settings](https://github.com/RandomFractals/pro-data-tools/blob/main/sqlite-tools.md#configuration-settings) and [Functions](https://github.com/RandomFractals/pro-data-tools/blob/main/sqlite-tools.md#functions). + +![SQLite Pro Tools Views](https://github.com/RandomFractals/pro-data-tools/blob/main/docs/images/sqlite-tools.png?raw=true) + ### PRQL Pro Tools -Our [**Pro**](https://github.com/sponsors/RandomFractals/sponsorships?sponsor=RandomFractals&tier_id=18884) sponsors on github also get access to the premium [PRQL Pro Tools](https://www.linkedin.com/feed/hashtag/?keywords=prqlprotools) VS Code extension. The initial private beta release of **PRQL Pro Tools** comes with custom [PRQL Code Lens](https://twitter.com/search?q=%23PRQLCodeLens&src=typed_query&f=live) SQL Tools plugin that allows you to run [PRQL](https://prql-lang.org/) queries for the supported PRQL target [SQL dialects](https://prql-lang.org/book/project/target.html#dialects) directly using any of the supported SQL Tools extension drivers for the different database management systems. +Our [**Pro**](https://github.com/sponsors/RandomFractals/sponsorships?tier_id=18884) sponsors on github also get access to the premium [PRQL Pro Tools](https://github.com/RandomFractals/pro-data-tools/blob/main/prql-tools.md) VS Code extension. The initial private beta release of **PRQL Pro Tools** comes with custom [PRQL Code Lens](https://github.com/RandomFractals/pro-data-tools/blob/main/prql-tools.md#prql-code-lens) SQL Tools plugin that allows you to run [PRQL](https://prql-lang.org/) queries for the supported PRQL target [SQL dialects](https://prql-lang.org/book/project/target.html#dialects) directly using any of the supported SQL Tools extension drivers for the different database management systems. ![PRQL Pro Tools Code Lens](https://github.com/RandomFractals/duckdb-sql-tools/blob/main/docs/images/prql-code-lens-pro-tools.gif?raw=true) ## Markdown SQL Pro Tools -Recently released private beta of [**Markdown SQL Pro Tools**](https://www.linkedin.com/feed/hashtag/?keywords=markdownsqlprotools) add custom SQL [Code Lenses](https://code.visualstudio.com/api/language-extensions/programmatic-language-features#codelens-show-actionable-context-information-within-source-code) to SQL code blocks in standard SQL and various markdown documents in VS Code IDE. +Recently released private beta of [**Markdown SQL Pro Tools**](https://github.com/RandomFractals/pro-data-tools/tree/main#markdown-sql-pro-tools) add custom SQL [Code Lenses](https://code.visualstudio.com/api/language-extensions/programmatic-language-features#codelens-show-actionable-context-information-within-source-code) to SQL code blocks in standard SQL and various markdown documents in VS Code IDE. The provided **SQL Code Lenses** include top-level SQL and markdown document `Execute All`, `Select All`, `Extract All`, `Bookmark All` SQL code lenses, `Execute`, `Select`, and `Copy` SQL code block lenses, and `Create` New SQL document from SQL code block lens. - Our **SQL Code Lenses** integrate with the popular [SQL Tools](https://vscode-sqltools.mteixeira.dev/en/home/) Connections manager, supported [SQL Tools drivers](https://marketplace.visualstudio.com/search?term=tag%3Asqltools-driver&target=VSCode&category=All%20categories&sortBy=PublishedDate), and query results viewer to execute SQL **directly** on any of the supported SQL Tools database management systems and view results. + Our **SQL Code Lenses** integrate with the popular [SQL Tools](https://vscode-sqltools.mteixeira.dev/en/home/) Connections manager, supported [SQL Tools drivers](https://marketplace.visualstudio.com/search?term=tag%3Asqltools-driver&target=VSCode&category=All%20categories&sortBy=Rating), and query results viewer to execute SQL **directly** on any of the supported SQL Tools database management systems and view results. ![Markdown SQL Pro Tools](https://github.com/RandomFractals/pro-data-tools/blob/main/docs/images/markdown-sql-pro-tools-v1.1.0.gif?raw=true) -**Markdodown SQL Pro Tools** extension is available under the new [**Markdown Sql Pro Tools**](https://github.com/sponsors/RandomFractals/sponsorships?sponsor=RandomFractals&tier_id=295482) monthly sponsor subscription on GitHub and also includes access to the [**DuckDB Pro**](https://randomfractals.github.io/pro-data-tools/#duckdb-pro-tools) SQL Tools plugin and [**PRQL Code Lens**](https://randomfractals.github.io/pro-data-tools/#prql-pro-tools) extension from our Premium [**Pro Data Tools**](https://randomfractals.github.io/pro-data-tools/#pro-data-tools) extensions pack. +**Markdodown SQL Pro Tools** extension is available under the new [**Markdown Sql Pro Tools**](https://github.com/sponsors/RandomFractals/sponsorships?tier_id=295482) monthly sponsor subscription on GitHub and also includes access to the [**DuckDB Pro**](https://randomfractals.github.io/pro-data-tools/#duckdb-pro-tools) SQL Tools plugin and [**PRQL Code Lens**](https://randomfractals.github.io/pro-data-tools/#prql-pro-tools) extension from our Premium [**Pro Data Tools**](https://randomfractals.github.io/pro-data-tools/#pro-data-tools) extensions pack. ## Data Notebook Pro Tools -The new [**Data Notebook Pro Tools**](https://twitter.com/search?q=(%23DataNotebooks)%20(from%3ATarasNovak)&src=typed_query) extension from our Premium **Pro Data Tools** extensions pack will be released later this year for our [**Pro sponsors**](https://github.com/sponsors/RandomFractals) on GitHub. +The new [**Data Notebook Pro Tools**](https://github.com/RandomFractals/pro-data-tools/tree/main#data-notebook-pro-tools) extension from our Premium **Pro Data Tools** extensions pack will be released later this year for our [**Pro sponsors**](https://github.com/sponsors/RandomFractals) on GitHub. **Data Notebook Pro** extension integrates SQL Tools connections, supported database management systems, our [**Data Table Renderers**](https://marketplace.visualstudio.com/items?itemName=RandomFractalsInc.vscode-data-table) and VS Code Jupyter Notebook user interface to view and run SQL queries from `.sql` files and `.md` markdown documents with SQL code blocks using native VS Code Notebook View. @@ -194,7 +200,7 @@ Also, note in the `sqlite_attach()` SQL function call statement above we are esc # Demo Data -**DuckDB SQL Tools** extension documentation repository contains sample [/data](https://github.com/RandomFractals/sql-duckdb-tools/tree/main/data/chinook/duckdb) folder with [`chinook.duckdb`](https://github.com/RandomFractals/duckdb-sql-tools/blob/main/data/chinook/duckdb/chinook.duckdb) and [`chinook.sqlite`](https://github.com/RandomFractals/duckdb-sql-tools/blob/main/data/chinook/sqlite/chinook.sqlite) database files, `csv`, `json` and `parquet` data files you can download to get started using DuckDB with our SQL Tools VS Code extension. +**DuckDB SQL Tools** extension documentation repository contains sample [/data](https://github.com/RandomFractals/duckdb-sql-tools/tree/main/data/chinook/duckdb) folder with [`chinook.duckdb`](https://github.com/RandomFractals/duckdb-sql-tools/blob/main/data/chinook/duckdb/chinook.duckdb) and [`chinook.sqlite`](https://github.com/RandomFractals/duckdb-sql-tools/blob/main/data/chinook/sqlite/chinook.sqlite) database files, `csv`, `json` and `parquet` data files you can download to get started using DuckDB with our SQL Tools VS Code extension. The [cninook/duckdb](https://github.com/RandomFractals/duckdb-sql-tools/tree/main/data/chinook/duckdb) demo data folder also has sample SQL query files you can try running on this well-known sample database with our VS Code extension. @@ -206,19 +212,19 @@ Due to the limited time and minimal development effort invested into building th ## DuckDB Storage -**DuckDB SQL Tools** *Preview* v1.2.0 release of this extension supports only local database instances created with [DuckDB v0.8.1](https://github.com/duckdb/duckdb/releases/tag/v0.8.1) engine. Database instances and files created with other versions of DuckDB are not supported as they use different compression and storage formats and the structure of `.duckdb` file has been changing as DuckDB engine is evolving. +**DuckDB SQL Tools** *Preview* v1.6.0 release of this extension supports only local database instances created with [DuckDB v0.10.2](https://github.com/duckdb/duckdb/releases/tag/v0.10.2) engine. Database instances and files created with older minor versions of DuckDB are not supported as they use different compression and storage formats and the structure of `.duckdb` file has been changing as DuckDB engine is evolving. -Use [DuckDB CLI](https://duckdb.org/docs/api/cli.html) to export data from the older database file versions and create new `.duckdb` file using the latest DuckDB storage implemenation. Read [Announcing DuckDB 0.8.0](https://duckdb.org/2023/05/17/announcing-duckdb-080.html) blog post for more information about DuckDB v0.8 storage improvements. +Use [DuckDB CLI](https://duckdb.org/docs/api/cli.html) to export data from the older database file versions and create new `.duckdb` file using the latest DuckDB storage implemenation. Read [Announcing DuckDB 0.10.0](https://duckdb.org/2024/02/13/announcing-duckdb-0100.html) blog post for more information about DuckDB v0.10 storage improvements, backward compatibility, and new features. -You can use prior [`v1.0.2`](https://github.com/RandomFractals/duckdb-sql-tools/releases/tag/v1.0.2) of this free **DuckDB Sql Tools** *Preview* extension to work with the older DuckDB [v0.6.1](https://github.com/duckdb/duckdb/releases/tag/v0.6.1) files. DuckDB SQL Tools [`v1.1.0`](https://github.com/RandomFractals/duckdb-sql-tools/releases/tag/v1.1.0) supports DuckDB [v0.7.1](https://github.com/duckdb/duckdb/releases/tag/v0.7.1). +You can use prior [`v1.0.2`](https://github.com/RandomFractals/duckdb-sql-tools/releases/tag/v1.0.2) of this free **DuckDB Sql Tools** *Preview* extension to work with the older DuckDB [v0.6.1](https://github.com/duckdb/duckdb/releases/tag/v0.6.1) files. DuckDB SQL Tools [`v1.1.0`](https://github.com/RandomFractals/duckdb-sql-tools/releases/tag/v1.1.0) supports DuckDB [v0.7.1](https://github.com/duckdb/duckdb/releases/tag/v0.7.1), and DuckDB SQL Tools [`v1.2.0`](https://github.com/RandomFractals/duckdb-sql-tools/releases/tag/v1.2.0) supports DuckDB [v0.8.1](https://github.com/duckdb/duckdb/releases/tag/v0.8.1). -The **Premium** [**DuckDB Pro Tools**](https://github.com/RandomFractals/pro-data-tools/tree/main#duckdb-pro-tools) extension version, available to our [**Pro** sponsors](https://github.com/sponsors/RandomFractals/sponsorships?sponsor=RandomFractals&tier_id=18884) on GitHub, supports the latest DuckDB [v0.8.1](https://github.com/duckdb/duckdb/releases/tag/v0.8.1) files with many advanced SQL Tools features, and will be updated to support the latest DuckDB [v0.9.0](https://github.com/duckdb/duckdb/releases/tag/v0.9.0) storage format and featues soon. +The **Premium** [**DuckDB Pro Tools**](https://github.com/RandomFractals/pro-data-tools/tree/main#duckdb-pro-tools) extension version, available to our [**Pro** sponsors](https://github.com/sponsors/RandomFractals/sponsorships?tier_id=18884) on GitHub, supports DuckDB [v0.10.2](https://github.com/duckdb/duckdb/releases/tag/v0.10.2) files with many advanced SQL Tools features, and will be updated to support the latest DuckDB [v1.0.0](https://github.com/duckdb/duckdb/releases) storage format and features soon. ## Read-Only DuckDB **DuckDB SQL Tools** *Preview* extension opens `.duckdb` database files in `read-only` mode. -The **Premium DuckDB Pro Tools** extension version, available to our [**Pro** sponsors](https://github.com/sponsors/RandomFractals/sponsorships?sponsor=RandomFractals&tier_id=18884) on GitHub, supports opening `.duckdb` files in write mode, provides DuckDB v0.8 support, extended DuckDB connection, schemas, systems objecs, and views display. +The **Premium DuckDB Pro Tools** extension version, available to our [**Pro** sponsors](https://github.com/sponsors/RandomFractals/sponsorships?tier_id=18884) on GitHub, supports opening `.duckdb` files in write mode, provides DuckDB v0.10 support, extended DuckDB connection, schemas, systems objecs, and views display. ## In-Memory DuckDB @@ -228,13 +234,7 @@ You can experiment with writable `:memory:` DuckDB instances in this **DuckDB SQ **Note**: only `:memory` DuckDB database instances are open in `read/write` mode in this **DuckDB SQL Tools** *Preview* extension version. -Sign up for the Premium [Pro Data Tools on GitHub](https://github.com/sponsors/RandomFractals/sponsorships?sponsor=RandomFractals&tier_id=18884) to get access to the [**DuckDB Pro Tools**](https://twitter.com/hashtag/DuckDBProTools?src=hashtag_click) and work with the latest versions of DuckDB files in read and write modes. - -## VS Code Memory Limit - -You can adjust the amount of RAM allocated to VS Code IDE to enable opening large files and load more data into memory. Go to `File -> Preferences -> Settings` and type `files.maxMemoryForLargeFilesMB` in the Setting search field to change it. For example, users with 64GB of RAM can change it to `49152` MB to allow VS Code use 48GB of available memory. - -![VS Code Max Memory Setting](https://github.com/RandomFractals/duckdb-sql-tools/blob/main/docs/images/vscode-max-memory-setting.png?raw=true) +Sign up for the Premium [Pro Data Tools on GitHub](https://github.com/sponsors/RandomFractals/sponsorships?tier_id=18884) to get access to the [**DuckDB Pro Tools**](https://github.com/RandomFractals/pro-data-tools/blob/main/duckdb-tools.md#duckdb-pro-tools) and work with the latest versions of DuckDB files in read and write modes. # Configuration @@ -267,6 +267,6 @@ Please use our public [DuckDB SQL Tools GitHub Discussions](https://github.com/R # Support -Become a [Fan or a Pro Sponsor](https://github.com/sponsors/RandomFractals) of our dev work on this and other [Random Fractals, Inc. code and data viz extensions](https://marketplace.visualstudio.com/publishers/RandomFractalsInc) if you find them useful, educational, or enhancing your daily dataViz dev code workflows and exploratory data analysis experience. +Become a [Fan or a Pro Sponsor](https://github.com/sponsors/RandomFractals) of our work on this and other [Random Fractals, Inc. code and data viz extensions](https://marketplace.visualstudio.com/publishers/RandomFractalsInc) if you find them useful, educational, or enhancing your daily dev workflows and exploratory data analysis experience in VS Code IDE. 💖 https://github.com/sponsors/RandomFractals diff --git a/data/chinook/duckdb/chinook-v0.8.1.duckdb b/data/chinook/duckdb/chinook-v0.8.1.duckdb new file mode 100644 index 0000000..7796017 Binary files /dev/null and b/data/chinook/duckdb/chinook-v0.8.1.duckdb differ diff --git a/data/chinook/duckdb/chinook.duckdb b/data/chinook/duckdb/chinook.duckdb index 7796017..bd0d5b2 100644 Binary files a/data/chinook/duckdb/chinook.duckdb and b/data/chinook/duckdb/chinook.duckdb differ diff --git a/data/chinook/duckdb/employees-by-age.prql b/data/chinook/duckdb/employees-by-age.prql index ec92fd3..9654814 100644 --- a/data/chinook/duckdb/employees-by-age.prql +++ b/data/chinook/duckdb/employees-by-age.prql @@ -1,5 +1,5 @@ from employees -derive [ - age = (@2023-02-12 - birth_date) / 365 -] -select [employee_id, first_name, last_name, birth_date, age] +derive { + age = (@2023-10-14 - birth_date) / 365 +} +select {employee_id, first_name, last_name, birth_date, age} diff --git a/data/chinook/duckdb/employees-by-age.sql b/data/chinook/duckdb/employees-by-age.sql index 0576dc6..cfd3064 100644 --- a/data/chinook/duckdb/employees-by-age.sql +++ b/data/chinook/duckdb/employees-by-age.sql @@ -3,8 +3,8 @@ SELECT first_name, last_name, birth_date, - (DATE '2023-02-12' - birth_date) / 365 AS age + ((DATE '2023-10-14' - birth_date) / 365) AS age FROM employees --- Generated by PRQL compiler version:0.5.0 target:sql.duckdb (https://prql-lang.org) +-- Generated by PRQL compiler version:0.9.1 target:sql.duckdb (https://prql-lang.org) diff --git a/data/chinook/duckdb/employees-by-birth-date.prql b/data/chinook/duckdb/employees-by-birth-date.prql index 02209ac..e127187 100644 --- a/data/chinook/duckdb/employees-by-birth-date.prql +++ b/data/chinook/duckdb/employees-by-birth-date.prql @@ -1,3 +1,3 @@ from employees -select [employee_id, first_name, last_name, birth_date] +select {employee_id, first_name, last_name, birth_date} sort birth_date diff --git a/data/chinook/duckdb/employees-by-birth-date.sql b/data/chinook/duckdb/employees-by-birth-date.sql index f723c23..47ca08b 100644 --- a/data/chinook/duckdb/employees-by-birth-date.sql +++ b/data/chinook/duckdb/employees-by-birth-date.sql @@ -8,4 +8,4 @@ FROM ORDER BY birth_date --- Generated by PRQL compiler version:0.5.0 target:sql.duckdb (https://prql-lang.org) +-- Generated by PRQL compiler version:0.9.1 target:sql.duckdb (https://prql-lang.org) diff --git a/data/chinook/duckdb/recent-employee-hires-by-title.sql b/data/chinook/duckdb/recent-employee-hires-by-title.sql index c3c5408..e87b7b7 100644 --- a/data/chinook/duckdb/recent-employee-hires-by-title.sql +++ b/data/chinook/duckdb/recent-employee-hires-by-title.sql @@ -1,19 +1,9 @@ -WITH table_1 AS ( - SELECT - *, - ROW_NUMBER() OVER ( - PARTITION BY title - ORDER BY - hire_date - ) AS _expr_0 - FROM - employees -) SELECT - * EXCLUDE (_expr_0) + DISTINCT ON (title) * FROM - table_1 -WHERE - _expr_0 <= 1 + employees +ORDER BY + title, + hire_date --- Generated by PRQL compiler version:0.5.0 target:sql.duckdb (https://prql-lang.org) +-- Generated by PRQL compiler version:0.9.1 target:sql.duckdb (https://prql-lang.org) diff --git a/data/gbif/top-10-duck-species-observations.prql b/data/gbif/top-10-duck-species-observations.prql index a7c1109..bf22cc8 100644 --- a/data/gbif/top-10-duck-species-observations.prql +++ b/data/gbif/top-10-duck-species-observations.prql @@ -2,10 +2,10 @@ from s'select * from read_parquet( "s3://gbif-open-data-us-east-1/occurrence/2023-03-01/occurrence.parquet/000000" )' filter genus == 'Anas' -group [genus, species] ( - aggregate [ - total = count - ] +group {genus, species} ( + aggregate { + total = count genus + } ) -sort [-total] +sort {-total} take 10 diff --git a/data/gbif/top-10-duck-species-observations.sql b/data/gbif/top-10-duck-species-observations.sql new file mode 100644 index 0000000..6744dac --- /dev/null +++ b/data/gbif/top-10-duck-species-observations.sql @@ -0,0 +1,25 @@ +WITH table_0 AS ( + SELECT + * + from + read_parquet( + "s3://gbif-open-data-us-east-1/occurrence/2023-03-01/occurrence.parquet/000000" + ) +) +SELECT + genus, + species, + COUNT(*) AS total +FROM + table_0 +WHERE + genus = 'Anas' +GROUP BY + genus, + species +ORDER BY + total DESC +LIMIT + 10 + +-- Generated by PRQL compiler version:0.9.1 target:sql.duckdb (https://prql-lang.org) diff --git a/notebooks/chinook-duckdb.ipynb b/notebooks/chinook-duckdb.ipynb index 9f7ae14..ed5a71a 100644 --- a/notebooks/chinook-duckdb.ipynb +++ b/notebooks/chinook-duckdb.ipynb @@ -7,16 +7,25 @@ "outputs": [], "source": [ "import sys\n", - "!{sys.executable} -m pip install duckdb" + "!{sys.executable} -m pip install -U duckdb" ] }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 1, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "0.9.1\n" + ] + } + ], "source": [ - "import duckdb" + "import duckdb\n", + "print(duckdb.__version__)" ] }, { @@ -31,16 +40,16 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 3, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "" + "" ] }, - "execution_count": 4, + "execution_count": 3, "metadata": {}, "output_type": "execute_result" } @@ -64,16 +73,16 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 4, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "" + "" ] }, - "execution_count": 5, + "execution_count": 4, "metadata": {}, "output_type": "execute_result" } @@ -204,7 +213,7 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 5, "metadata": {}, "outputs": [ { @@ -240,69 +249,134 @@ " \n", " \n", " 0\n", - " NaN\n", - " main\n", + " chinook\n", + " information_schema\n", " duckdb\n", " NaN\n", " NaN\n", " NaN\n", - " NaN\n", + " None\n", " \n", " \n", " 1\n", - " NaN\n", - " pg_catalog\n", + " chinook\n", + " main\n", " duckdb\n", " NaN\n", " NaN\n", " NaN\n", - " NaN\n", + " None\n", " \n", " \n", " 2\n", + " chinook\n", + " pg_catalog\n", + " duckdb\n", + " NaN\n", " NaN\n", + " NaN\n", + " None\n", + " \n", + " \n", + " 3\n", + " system\n", " information_schema\n", " duckdb\n", " NaN\n", " NaN\n", " NaN\n", + " None\n", + " \n", + " \n", + " 4\n", + " system\n", + " main\n", + " duckdb\n", " NaN\n", + " NaN\n", + " NaN\n", + " None\n", " \n", " \n", - " 3\n", + " 5\n", + " system\n", + " pg_catalog\n", + " duckdb\n", + " NaN\n", + " NaN\n", + " NaN\n", + " None\n", + " \n", + " \n", + " 6\n", + " temp\n", + " information_schema\n", + " duckdb\n", + " NaN\n", + " NaN\n", " NaN\n", + " None\n", + " \n", + " \n", + " 7\n", " temp\n", + " main\n", " duckdb\n", " NaN\n", " NaN\n", " NaN\n", + " None\n", + " \n", + " \n", + " 8\n", + " temp\n", + " pg_catalog\n", + " duckdb\n", + " NaN\n", + " NaN\n", " NaN\n", + " None\n", " \n", " \n", "\n", "" ], "text/plain": [ - " catalog_name schema_name schema_owner \\\n", - "0 NaN main duckdb \n", - "1 NaN pg_catalog duckdb \n", - "2 NaN information_schema duckdb \n", - "3 NaN temp duckdb \n", + " catalog_name schema_name schema_owner \\\n", + "0 chinook information_schema duckdb \n", + "1 chinook main duckdb \n", + "2 chinook pg_catalog duckdb \n", + "3 system information_schema duckdb \n", + "4 system main duckdb \n", + "5 system pg_catalog duckdb \n", + "6 temp information_schema duckdb \n", + "7 temp main duckdb \n", + "8 temp pg_catalog duckdb \n", "\n", " default_character_set_catalog default_character_set_schema \\\n", "0 NaN NaN \n", "1 NaN NaN \n", "2 NaN NaN \n", "3 NaN NaN \n", + "4 NaN NaN \n", + "5 NaN NaN \n", + "6 NaN NaN \n", + "7 NaN NaN \n", + "8 NaN NaN \n", "\n", " default_character_set_name sql_path \n", - "0 NaN NaN \n", - "1 NaN NaN \n", - "2 NaN NaN \n", - "3 NaN NaN " + "0 NaN None \n", + "1 NaN None \n", + "2 NaN None \n", + "3 NaN None \n", + "4 NaN None \n", + "5 NaN None \n", + "6 NaN None \n", + "7 NaN None \n", + "8 NaN None " ] }, - "execution_count": 13, + "execution_count": 5, "metadata": {}, "output_type": "execute_result" } @@ -315,7 +389,7 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 6, "metadata": {}, "outputs": [ { @@ -356,7 +430,7 @@ " \n", " \n", " 0\n", - " NaN\n", + " chinook\n", " main\n", " albums\n", " BASE TABLE\n", @@ -367,11 +441,11 @@ " NaN\n", " YES\n", " NO\n", - " NaN\n", + " None\n", " \n", " \n", " 1\n", - " NaN\n", + " chinook\n", " main\n", " artists\n", " BASE TABLE\n", @@ -382,11 +456,11 @@ " NaN\n", " YES\n", " NO\n", - " NaN\n", + " None\n", " \n", " \n", " 2\n", - " NaN\n", + " chinook\n", " main\n", " customers\n", " BASE TABLE\n", @@ -397,11 +471,11 @@ " NaN\n", " YES\n", " NO\n", - " NaN\n", + " None\n", " \n", " \n", " 3\n", - " NaN\n", + " chinook\n", " main\n", " employees\n", " BASE TABLE\n", @@ -412,11 +486,11 @@ " NaN\n", " YES\n", " NO\n", - " NaN\n", + " None\n", " \n", " \n", " 4\n", - " NaN\n", + " chinook\n", " main\n", " genres\n", " BASE TABLE\n", @@ -427,11 +501,11 @@ " NaN\n", " YES\n", " NO\n", - " NaN\n", + " None\n", " \n", " \n", " 5\n", - " NaN\n", + " chinook\n", " main\n", " invoices\n", " BASE TABLE\n", @@ -442,11 +516,11 @@ " NaN\n", " YES\n", " NO\n", - " NaN\n", + " None\n", " \n", " \n", " 6\n", - " NaN\n", + " chinook\n", " main\n", " invoice_items\n", " BASE TABLE\n", @@ -457,11 +531,11 @@ " NaN\n", " YES\n", " NO\n", - " NaN\n", + " None\n", " \n", " \n", " 7\n", - " NaN\n", + " chinook\n", " main\n", " media_types\n", " BASE TABLE\n", @@ -472,11 +546,11 @@ " NaN\n", " YES\n", " NO\n", - " NaN\n", + " None\n", " \n", " \n", " 8\n", - " NaN\n", + " chinook\n", " main\n", " playlists\n", " BASE TABLE\n", @@ -487,11 +561,11 @@ " NaN\n", " YES\n", " NO\n", - " NaN\n", + " None\n", " \n", " \n", " 9\n", - " NaN\n", + " chinook\n", " main\n", " playlist_track\n", " BASE TABLE\n", @@ -502,11 +576,11 @@ " NaN\n", " YES\n", " NO\n", - " NaN\n", + " None\n", " \n", " \n", " 10\n", - " NaN\n", + " chinook\n", " main\n", " tracks\n", " BASE TABLE\n", @@ -517,25 +591,25 @@ " NaN\n", " YES\n", " NO\n", - " NaN\n", + " None\n", " \n", " \n", "\n", "" ], "text/plain": [ - " table_catalog table_schema table_name table_type \\\n", - "0 NaN main albums BASE TABLE \n", - "1 NaN main artists BASE TABLE \n", - "2 NaN main customers BASE TABLE \n", - "3 NaN main employees BASE TABLE \n", - "4 NaN main genres BASE TABLE \n", - "5 NaN main invoices BASE TABLE \n", - "6 NaN main invoice_items BASE TABLE \n", - "7 NaN main media_types BASE TABLE \n", - "8 NaN main playlists BASE TABLE \n", - "9 NaN main playlist_track BASE TABLE \n", - "10 NaN main tracks BASE TABLE \n", + " table_catalog table_schema table_name table_type \\\n", + "0 chinook main albums BASE TABLE \n", + "1 chinook main artists BASE TABLE \n", + "2 chinook main customers BASE TABLE \n", + "3 chinook main employees BASE TABLE \n", + "4 chinook main genres BASE TABLE \n", + "5 chinook main invoices BASE TABLE \n", + "6 chinook main invoice_items BASE TABLE \n", + "7 chinook main media_types BASE TABLE \n", + "8 chinook main playlists BASE TABLE \n", + "9 chinook main playlist_track BASE TABLE \n", + "10 chinook main tracks BASE TABLE \n", "\n", " self_referencing_column_name reference_generation \\\n", "0 NaN NaN \n", @@ -564,20 +638,20 @@ "10 NaN NaN \n", "\n", " user_defined_type_name is_insertable_into is_typed commit_action \n", - "0 NaN YES NO NaN \n", - "1 NaN YES NO NaN \n", - "2 NaN YES NO NaN \n", - "3 NaN YES NO NaN \n", - "4 NaN YES NO NaN \n", - "5 NaN YES NO NaN \n", - "6 NaN YES NO NaN \n", - "7 NaN YES NO NaN \n", - "8 NaN YES NO NaN \n", - "9 NaN YES NO NaN \n", - "10 NaN YES NO NaN " + "0 NaN YES NO None \n", + "1 NaN YES NO None \n", + "2 NaN YES NO None \n", + "3 NaN YES NO None \n", + "4 NaN YES NO None \n", + "5 NaN YES NO None \n", + "6 NaN YES NO None \n", + "7 NaN YES NO None \n", + "8 NaN YES NO None \n", + "9 NaN YES NO None \n", + "10 NaN YES NO None " ] }, - "execution_count": 12, + "execution_count": 6, "metadata": {}, "output_type": "execute_result" } @@ -590,7 +664,7 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 7, "metadata": {}, "outputs": [ { @@ -640,12 +714,12 @@ " \n", " \n", " 0\n", - " NaN\n", + " chinook\n", " main\n", " albums\n", " album_id\n", " 1\n", - " NaN\n", + " None\n", " NO\n", " INTEGER\n", " NaN\n", @@ -664,12 +738,12 @@ " \n", " \n", " 1\n", - " NaN\n", + " chinook\n", " main\n", " albums\n", " title\n", " 2\n", - " NaN\n", + " None\n", " NO\n", " VARCHAR\n", " NaN\n", @@ -688,12 +762,12 @@ " \n", " \n", " 2\n", - " NaN\n", + " chinook\n", " main\n", " albums\n", " artist_id\n", " 3\n", - " NaN\n", + " None\n", " NO\n", " INTEGER\n", " NaN\n", @@ -712,12 +786,12 @@ " \n", " \n", " 3\n", - " NaN\n", + " chinook\n", " main\n", " artists\n", " artist_id\n", " 1\n", - " NaN\n", + " None\n", " NO\n", " INTEGER\n", " NaN\n", @@ -736,12 +810,12 @@ " \n", " \n", " 4\n", - " NaN\n", + " chinook\n", " main\n", " artists\n", " name\n", " 2\n", - " NaN\n", + " None\n", " YES\n", " VARCHAR\n", " NaN\n", @@ -784,12 +858,12 @@ " \n", " \n", " 59\n", - " NaN\n", + " chinook\n", " main\n", " tracks\n", " genre_id\n", " 5\n", - " NaN\n", + " None\n", " YES\n", " INTEGER\n", " NaN\n", @@ -808,12 +882,12 @@ " \n", " \n", " 60\n", - " NaN\n", + " chinook\n", " main\n", " tracks\n", " composer\n", " 6\n", - " NaN\n", + " None\n", " YES\n", " VARCHAR\n", " NaN\n", @@ -832,12 +906,12 @@ " \n", " \n", " 61\n", - " NaN\n", + " chinook\n", " main\n", " tracks\n", " milliseconds\n", " 7\n", - " NaN\n", + " None\n", " NO\n", " INTEGER\n", " NaN\n", @@ -856,12 +930,12 @@ " \n", " \n", " 62\n", - " NaN\n", + " chinook\n", " main\n", " tracks\n", " bytes\n", " 8\n", - " NaN\n", + " None\n", " YES\n", " INTEGER\n", " NaN\n", @@ -880,12 +954,12 @@ " \n", " \n", " 63\n", - " NaN\n", + " chinook\n", " main\n", " tracks\n", " unit_price\n", " 9\n", - " NaN\n", + " None\n", " NO\n", " DECIMAL(10,2)\n", " NaN\n", @@ -908,31 +982,31 @@ "" ], "text/plain": [ - " table_catalog table_schema table_name column_name ordinal_position \\\n", - "0 NaN main albums album_id 1 \n", - "1 NaN main albums title 2 \n", - "2 NaN main albums artist_id 3 \n", - "3 NaN main artists artist_id 1 \n", - "4 NaN main artists name 2 \n", - ".. ... ... ... ... ... \n", - "59 NaN main tracks genre_id 5 \n", - "60 NaN main tracks composer 6 \n", - "61 NaN main tracks milliseconds 7 \n", - "62 NaN main tracks bytes 8 \n", - "63 NaN main tracks unit_price 9 \n", + " table_catalog table_schema table_name column_name ordinal_position \\\n", + "0 chinook main albums album_id 1 \n", + "1 chinook main albums title 2 \n", + "2 chinook main albums artist_id 3 \n", + "3 chinook main artists artist_id 1 \n", + "4 chinook main artists name 2 \n", + ".. ... ... ... ... ... \n", + "59 chinook main tracks genre_id 5 \n", + "60 chinook main tracks composer 6 \n", + "61 chinook main tracks milliseconds 7 \n", + "62 chinook main tracks bytes 8 \n", + "63 chinook main tracks unit_price 9 \n", "\n", " column_default is_nullable data_type character_maximum_length \\\n", - "0 NaN NO INTEGER NaN \n", - "1 NaN NO VARCHAR NaN \n", - "2 NaN NO INTEGER NaN \n", - "3 NaN NO INTEGER NaN \n", - "4 NaN YES VARCHAR NaN \n", + "0 None NO INTEGER NaN \n", + "1 None NO VARCHAR NaN \n", + "2 None NO INTEGER NaN \n", + "3 None NO INTEGER NaN \n", + "4 None YES VARCHAR NaN \n", ".. ... ... ... ... \n", - "59 NaN YES INTEGER NaN \n", - "60 NaN YES VARCHAR NaN \n", - "61 NaN NO INTEGER NaN \n", - "62 NaN YES INTEGER NaN \n", - "63 NaN NO DECIMAL(10,2) NaN \n", + "59 None YES INTEGER NaN \n", + "60 None YES VARCHAR NaN \n", + "61 None NO INTEGER NaN \n", + "62 None YES INTEGER NaN \n", + "63 None NO DECIMAL(10,2) NaN \n", "\n", " character_octet_length ... is_identity identity_generation \\\n", "0 NaN ... NaN NaN \n", @@ -976,7 +1050,7 @@ "[64 rows x 44 columns]" ] }, - "execution_count": 15, + "execution_count": 7, "metadata": {}, "output_type": "execute_result" } @@ -989,16 +1063,16 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 8, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "" + "" ] }, - "execution_count": 16, + "execution_count": 8, "metadata": {}, "output_type": "execute_result" } @@ -1021,16 +1095,16 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": 9, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "" + "" ] }, - "execution_count": 19, + "execution_count": 9, "metadata": {}, "output_type": "execute_result" } @@ -1038,23 +1112,23 @@ "source": [ "# load csv data\n", "db.execute(\"\"\"\n", - "INSERT INTO artists SELECT * FROM '../data/chinook/csv/artists.csv';\n", - "INSERT INTO albums SELECT * FROM '../data/chinook//csv/albums.csv';\n", - "INSERT INTO genres SELECT * FROM '../data/chinook//csv/genres.csv';\n", - "INSERT INTO media_types SELECT * FROM '../data/chinook/csv/media_types.csv';\n", - "INSERT INTO playlists SELECT * FROM '../data/chinook/csv/playlists.csv';\n", - "INSERT INTO tracks SELECT * FROM '../data/chinook/csv/tracks.csv';\n", - "INSERT INTO playlist_track SELECT * FROM '../data/chinook/csv/playlist_track.csv';\n", - "INSERT INTO employees SELECT * FROM '../data/chinook/csv/employees.csv';\n", - "INSERT INTO customers SELECT * FROM '../data/chinook/csv/customers.csv';\n", - "INSERT INTO invoices SELECT * FROM '../data/chinook/csv/invoices.csv';\n", - "INSERT INTO invoice_items SELECT * FROM '../data/chinook/csv/invoice_items.csv';\n", + "insert into artists select * from '../data/chinook/csv/artists.csv';\n", + "insert into albums select * from '../data/chinook/csv/albums.csv';\n", + "insert into genres select * from '../data/chinook/csv/genres.csv';\n", + "insert into media_types select * from '../data/chinook/csv/media_types.csv';\n", + "insert into playlists select * from '../data/chinook/csv/playlists.csv';\n", + "insert into tracks select * from '../data/chinook/csv/tracks.csv';\n", + "insert into playlist_track select * from '../data/chinook/csv/playlist_track.csv';\n", + "insert into employees select * from '../data/chinook/csv/employees.csv';\n", + "insert into customers select * from '../data/chinook/csv/customers.csv';\n", + "insert into invoices select * from '../data/chinook/csv/invoices.csv';\n", + "insert into invoice_items select * from '../data/chinook/csv/invoice_items.csv';\n", "\"\"\")\n" ] }, { "cell_type": "code", - "execution_count": 20, + "execution_count": 10, "metadata": {}, "outputs": [ { @@ -1076,83 +1150,73 @@ } ], "source": [ - "db.execute('SELECT * FROM artists LIMIT 10')\n", + "db.execute('select * from artists limit 10')\n", "print(db.fetch_df())" ] }, { "cell_type": "code", - "execution_count": 21, + "execution_count": 11, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - " name\n", - "0 albums\n", - "1 artists\n", - "2 customers\n", - "3 employees\n", - "4 genres\n", - "5 ifk_album_artist_id\n", - "6 ifk_customer_support_rep_id\n", - "7 ifk_employee_reports_to\n", - "8 ifk_invoice_customer_id\n", - "9 ifk_invoice_item_invoice_id\n", - "10 ifk_invoice_item_track_id\n", - "11 ifk_playlist_track_track_id\n", - "12 ifk_track_album_id\n", - "13 ifk_track_genre_id\n", - "14 ifk_track_media_type_id\n", - "15 invoice_items\n", - "16 invoices\n", - "17 media_types\n", - "18 playlist_track\n", - "19 playlists\n", - "20 tracks\n" + " name\n", + "0 albums\n", + "1 artists\n", + "2 customers\n", + "3 employees\n", + "4 genres\n", + "5 invoice_items\n", + "6 invoices\n", + "7 media_types\n", + "8 playlist_track\n", + "9 playlists\n", + "10 tracks\n" ] } ], "source": [ - "db.execute('PRAGMA show_tables')\n", + "db.execute('pragma show_tables')\n", "print(db.fetch_df())" ] }, { "cell_type": "code", - "execution_count": 22, + "execution_count": 12, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - " table_catalog table_schema table_name column_name ordinal_position \\\n", - "0 NaN main albums album_id 1 \n", - "1 NaN main albums title 2 \n", - "2 NaN main albums artist_id 3 \n", - "3 NaN main artists artist_id 1 \n", - "4 NaN main artists name 2 \n", - ".. ... ... ... ... ... \n", - "59 NaN main tracks genre_id 5 \n", - "60 NaN main tracks composer 6 \n", - "61 NaN main tracks milliseconds 7 \n", - "62 NaN main tracks bytes 8 \n", - "63 NaN main tracks unit_price 9 \n", + " table_catalog table_schema table_name column_name ordinal_position \\\n", + "0 chinook main albums album_id 1 \n", + "1 chinook main albums title 2 \n", + "2 chinook main albums artist_id 3 \n", + "3 chinook main artists artist_id 1 \n", + "4 chinook main artists name 2 \n", + ".. ... ... ... ... ... \n", + "59 chinook main tracks genre_id 5 \n", + "60 chinook main tracks composer 6 \n", + "61 chinook main tracks milliseconds 7 \n", + "62 chinook main tracks bytes 8 \n", + "63 chinook main tracks unit_price 9 \n", "\n", " column_default is_nullable data_type character_maximum_length \\\n", - "0 NaN NO INTEGER NaN \n", - "1 NaN NO VARCHAR NaN \n", - "2 NaN NO INTEGER NaN \n", - "3 NaN NO INTEGER NaN \n", - "4 NaN YES VARCHAR NaN \n", + "0 None NO INTEGER NaN \n", + "1 None NO VARCHAR NaN \n", + "2 None NO INTEGER NaN \n", + "3 None NO INTEGER NaN \n", + "4 None YES VARCHAR NaN \n", ".. ... ... ... ... \n", - "59 NaN YES INTEGER NaN \n", - "60 NaN YES VARCHAR NaN \n", - "61 NaN NO INTEGER NaN \n", - "62 NaN YES INTEGER NaN \n", - "63 NaN NO DECIMAL(10,2) NaN \n", + "59 None YES INTEGER NaN \n", + "60 None YES VARCHAR NaN \n", + "61 None NO INTEGER NaN \n", + "62 None YES INTEGER NaN \n", + "63 None NO DECIMAL(10,2) NaN \n", "\n", " character_octet_length ... is_identity identity_generation \\\n", "0 NaN ... NaN NaN \n", @@ -1204,20 +1268,9 @@ }, { "cell_type": "code", - "execution_count": 23, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "" - ] - }, - "execution_count": 23, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ "# export created chinook.duckdb with data\n", "db.execute(\"export database '../data/chinook/duckdb/data'\")" @@ -1225,7 +1278,29 @@ }, { "cell_type": "code", - "execution_count": 24, + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# export chinook data in parquet format\n", + "db.execute(\"\"\"\n", + "copy artists to '../data/chinook/duckdb/data/artists.parquet';\n", + "copy albums to '../data/chinook/duckdb/data/albums.parquet';\n", + "copy genres to '../data/chinook/duckdb/data/genres.parquet';\n", + "copy media_types to '../data/chinook/duckdb/data/media_types.parquet';\n", + "copy playlists to '../data/chinook/duckdb/data/playlists.parquet';\n", + "copy tracks to '../data/chinook/duckdb/data/tracks.parquet';\n", + "copy playlist_track to '../data/chinook/duckdb/data/playlist_track.parquet';\n", + "copy employees to '../data/chinook/duckdb/data/employees.parquet';\n", + "copy customers to '../data/chinook/duckdb/data/customers.parquet';\n", + "copy invoices to '../data/chinook/duckdb/data/invoices.parquet';\n", + "copy invoice_items to '../data/chinook/duckdb/data/invoice_items.parquet';\n", + "\"\"\")" + ] + }, + { + "cell_type": "code", + "execution_count": 13, "metadata": {}, "outputs": [], "source": [ @@ -1250,7 +1325,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.13" + "version": "3.10.7" }, "orig_nbformat": 4, "vscode": {