8000 PG-1667 Validate Vault engine type by artemgavrilov · Pull Request #462 · percona/postgres · GitHub
[go: up one dir, main page]
Skip to content

PG-1667 Validate Vault engine type #462

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 1 commit into
base: TDE_REL_17_STABLE
Choose a base branch
from
Draft

PG-1667 Validate Vault engine type #462

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions ci_scripts/setup-keyring-servers.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,10 @@ export VAULT_ROOT_TOKEN_FILE=$(mktemp)
jq -r .root_token "$CLUSTER_INFO" > "$VAULT_ROOT_TOKEN_FILE"
export VAULT_CACERT_FILE=$(jq -r .ca_cert_path "$CLUSTER_INFO")
rm "$CLUSTER_INFO"

## We need to enable key/value version 1 engine for just for tests
vault secrets enable -ca-cert=$VAULT_CACERT_FILE -path=kv-v1 -version=1 kv

if [ -v GITHUB_ACTIONS ]; then
echo "VAULT_ROOT_TOKEN_FILE=$VAULT_ROOT_TOKEN_FILE" >> $GITHUB_ENV
echo "VAULT_CACERT_FILE=$VAULT_CACERT_FILE" >> $GITHUB_ENV
Expand Down
29 changes: 12 additions & 17 deletions contrib/pg_tde/expected/vault_v2_test.out
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,22 +1,17 @@
CREATE EXTENSION pg_tde;
\getenv root_token_file VAULT_ROOT_TOKEN_FILE
\getenv cacert_file VAULT_CACERT_FILE
SELECT pg_tde_add_database_key_provider_vault_v2('vault-incorrect', 'https://127.0.0.1:8200', 'DUMMY-TOKEN', :'root_token_file', :'cacert_file');
pg_tde_add_database_key_provider_vault_v2
-------------------------------------------

(1 row)

-- FAILS
SELECT pg_tde_create_key_using_database_key_provider('vault-v2-key', 'vault-incorrect');
ERROR: Invalid HTTP response from keyring provider "vault-incorrect": 404
CREATE TABLE test_enc(
id SERIAL,
k INTEGER DEFAULT '0' NOT NULL,
PRIMARY KEY (id)
) USING tde_heap;
ERROR: principal key not configured
HINT: Use pg_tde_set_key_using_database_key_provider() or pg_tde_set_key_using_global_key_provider() to configure one.
-- FAILS as mount path does not exist
SELECT pg_tde_add_database_key_provider_vault_v2('vault-incorrect', 'https://127.0.0.1:8200', 'DUMMY-MOUNT-PATH', :'root_token_file', :'cacert_file');
ERROR: failed to get mount info for "https://127.0.0.1:8200" at mountpoint "DUMMY-MOUNT-PATH" (HTTP 400)
-- FAILS as it's not supported engine type
SELECT pg_tde_add_database_key_provider_vault_v2('vault-incorrect', 'https://127.0.0.1:8200', 'cubbyhole', :'root_token_file', :'cacert_file');
ERROR: vault mount at "cubbyhole" has unsupported engine type "cubbyhole"
HINT: The only supported vault engine type is Key/Value version "2"
-- FAILS as it's not supported engine version
SELECT pg_tde_add_database_key_provider_vault_v2('vault-incorrect', 'https://127.0.0.1:8200', 'kv-v1', :'root_token_file', :'cacert_file');
ERROR: vault mount at "kv-v1" has unsupported Key/Value engine version "1"
HINT: The only supported vault engine type is Key/Value version "2"
SELECT pg_tde_add_database_key_provider_vault_v2('vault-v2', 'https://127.0.0.1:8200', 'secret', :'root_token_file', :'cacert_file');
pg_tde_add_database_key_provider_vault_v2
-------------------------------------------
Expand Down Expand Up @@ -69,5 +64,5 @@ SELECT pg_tde_change_database_key_provider_vault_v2('vault-v2', 'https://127.0.0
ERROR: HTTP(S) request to keyring provider "vault-v2" failed
-- HTTP against HTTPS server fails
SELECT pg_tde_change_database_key_provider_vault_v2('vault-v2', 'http://127.0.0.1:8200', 'secret', :'root_token_file', NULL);
ERROR: Listing secrets of "http://127.0.0.1:8200" at mountpoint "secret" failed
ERROR: failed to get mount info for "http://127.0.0.1:8200" at mountpoint "secret" (HTTP 400)
DROP EXTENSION pg_tde;
15 changes: 7 additions & 8 deletions contrib/pg_tde/sql/vault_v2_test.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,15 +3,14 @@ CREATE EXTENSION pg_tde;
\getenv root_token_file VAULT_ROOT_TOKEN_FILE
\getenv cacert_file VAULT_CACERT_FILE

SELECT pg_tde_add_database_key_provider_vault_v2('vault-incorrect', 'https://127.0.0.1:8200', 'DUMMY-TOKEN', :'root_token_file', :'cacert_file');
-- FAILS
SELECT pg_tde_create_key_using_database_key_provider('vault-v2-key', 'vault-incorrect');
-- FAILS as mount path does not exist
SELECT pg_tde_add_database_key_provider_vault_v2('vault-incorrect', 'https://127.0.0.1:8200', 'DUMMY-MOUNT-PATH', :'root_token_file', :'cacert_file');

CREATE TABLE test_enc(
id SERIAL,
k INTEGER DEFAULT '0' NOT NULL,
PRIMARY KEY (id)
) USING tde_heap;
-- FAILS as it's not supported engine type
SELECT pg_tde_add_database_key_provider_vault_v2('vault-incorrect', 'https://127.0.0.1:8200', 'cubbyhole', :'root_token_file', :'cacert_file');

-- FAILS as it's not supported engine version
SELECT pg_tde_add_database_key_provider_vault_v2('vault-incorrect', 'https://127.0.0.1:8200', 'kv-v1', :'root_token_file', :'cacert_file');

SELECT pg_tde_add_database_key_provider_vault_v2('vault-v2', 'https://127.0.0.1:8200', 'secret', :'root_token_file', :'cacert_file');
SELECT pg_tde_create_key_using_database_key_provider('vault-v2-key', 'vault-v2');
Expand Down
232 changes: 222 additions & 10 deletions contrib/pg_tde/src/keyring/keyring_vault.c
8000
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,16 @@ typedef enum
JRESP_EXPECT_KEY
} JsonVaultRespSemState;

typedef enum
{
JRESP_MOUNT_INFO_EXPECT_TOPLEVEL_FIELD,
JRESP_MOUNT_INFO_EXPECT_TYPE_VALUE,
JRESP_MOUNT_INFO_EXPECT_VERSION_VALUE,
JRESP_MOUNT_INFO_EXPECT_OPTIONS_START,
JRESP_MOUNT_INFO_EXPECT_OPTIONS_FIELD,
} JsonVaultRespMountInfoSemState;


typedef enum
{
JRESP_F_UNUSED,
Expand All @@ -49,12 +59,27 @@ typedef struct JsonVaultRespState
char *key;
} JsonVaultRespState;

typedef struct JsonVaultMountInfoState
{
JsonVaultRespMountInfoSemState state;
int level;

char *type;
char *version;
} JsonVaultMountInfoState;

static JsonParseErrorType json_resp_object_start(void *state);
static JsonParseErrorType json_resp_object_end(void *state);
static JsonParseErrorType json_resp_scalar(void *state, char *token, JsonTokenType tokentype);
static JsonParseErrorType json_resp_object_field_start(void *state, char *fname, bool isnull);
static JsonParseErrorType parse_json_response(JsonVaultRespState *parse, JsonLexContext *lex);

static JsonParseErrorType json_mountinfo_object_start(void *state);
static JsonParseErrorType json_mountinfo_object_end(void *state);
static JsonParseErrorType json_mountinfo_scalar(void *state, char *token, JsonTokenType tokentype);
static JsonParseErrorType json_mountinfo_object_field_start(void *state, char *fname, bool isnull);
static JsonParseErrorType parse_vault_mount_info(JsonVaultMountInfoState *state, JsonLexContext *lex);

static char *get_keyring_vault_url(VaultV2Keyring *keyring, const char *key_name, char *out, size_t out_size);
static bool curl_perform(VaultV2Keyring *keyring, const char *url, CurlString *outStr, long *httpCode, const char *postData);

Expand Down Expand Up @@ -292,36 +317,76 @@ validate(GenericKeyring *keyring)
char url[VAULT_URL_MAX_LEN];
CurlString str;
long httpCode = 0;
JsonParseErrorType json_error;
JsonLexContext *jlex = NULL;
JsonVaultMountInfoState parse;

/*
* Validate that the mount has the correct engine type and version.
*/
snprintf(url, VAULT_URL_MAX_LEN, "%s/v1/sys/mounts/%s", vault_keyring->vault_url, vault_keyring->vault_mount_path);

if (!curl_perform(vault_keyring, url, &str, &httpCode, NULL))
ereport(ERROR,
errmsg("HTTP(S) request to keyring provider \"%s\" failed",
vault_keyring->keyring.provider_name));

if (httpCode != 200)
ereport(ERROR,
errcode(ERRCODE_INVALID_PARAMETER_VALUE),
errmsg("failed to get mount info for \"%s\" at mountpoint \"%s\" (HTTP %ld)",
vault_keyring->vault_url, vault_keyring->vault_mount_path, httpCode));

jlex = makeJsonLexContextCstringLen(NULL, str.ptr, str.len, PG_UTF8, true);
json_error = parse_vault_mount_info(&parse, jlex);

if (json_error != JSON_SUCCESS)
ereport(ERROR,
errcode(ERRCODE_INVALID_JSON_TEXT),
errmsg("failed to parse mount info for \"%s\" at mountpoint \"%s\": %s",
vault_keyring->vault_url, vault_keyring->vault_mount_path, json_errdetail(json_error, jlex)));

if (parse.type != NULL && strcmp(parse.type, "kv") != 0)
ereport(ERROR,
errcode(ERRCODE_INVALID_PARAMETER_VALUE),
errmsg("vault mount at \"%s\" has unsupported engine type \"%s\"",
vault_keyring->vault_mount_path, parse.type),
errhint("The only supported vault engine type is Key/Value version \"2\""));

if (parse.version != NULL && strcmp(parse.version, "2") != 0)
ereport(ERROR,
errcode(ERRCODE_INVALID_PARAMETER_VALUE),
errmsg("vault mount at \"%s\" has unsupported Key/Value engine version \"%s\"",
vault_keyring->vault_mount_path, parse.version),
errhint("The only supported vault engine type is Key/Value version \"2\""));

/*
* Validate connection by listing available keys at the root level of the
* mount point
* Validate that we can read the secrets at the mount point.
*/
snprintf(url, VAULT_URL_MAX_LEN, "%s/v1/%s/metadata/?list=true",
vault_keyring->vault_url, vault_keyring->vault_mount_path);

if (!curl_perform(vault_keyring, url, &str, &httpCode, NULL))
{
ereport(ERROR,
errmsg("HTTP(S) request to keyring provider \"%s\" failed",
vault_keyring->keyring.provider_name));
}

/* If the mount point doesn't have any secrets yet, we'll get a 404. */
if (httpCode != 200 && httpCode != 404)
{
ereport(ERROR,
errcode(ERRCODE_INVALID_PARAMETER_VALUE),
errmsg("Listing secrets of \"%s\" at mountpoint \"%s\" failed",
vault_keyring->vault_url, vault_keyring->vault_mount_path));
}
errcode(ERRCODE_INVALID_PARAMETER_VALUE),
errmsg("Listing secrets of \"%s\" at mountpoint \"%s\" failed",
vault_keyring->vault_url, vault_keyring->vault_mount_path));

if (str.ptr != NULL)
pfree(str.ptr);

if (jlex != NULL)
freeJsonLexContext(jlex);
}

/*
* JSON parser routines
* JSON parser routines for key response
*
* We expect the response in the form of:
* {
A3E2 Expand Down Expand Up @@ -436,6 +501,153 @@ json_resp_object_field_start(void *state, char *fname, bool isnull)
if (strcmp(fname, "key") == 0 && parse->level == 2)
parse->field = JRESP_F_KEY;
break;
default:
/* NOP */
break;
}

return JSON_SUCCESS;
}

/*
* JSON parser routines for mount info
*
* We expect the response in the form of:
* {
* ...
* "type": "kv",
* "options": {
* "version": "2"
* }
* ...
* }
*
* the rest fields are ignored
*/

static JsonParseErrorType
parse_vault_mount_info(JsonVaultMountInfoState *state, JsonLexContext *lex)
{
JsonSemAction sem;

state->state = JRESP_MOUNT_INFO_EXPECT_TOPLEVEL_FIELD;
state->type = NULL;
state->version = NULL;
state->level = -1;


memset(&sem, 0, sizeof(sem));
sem.semstate = state;
sem.object_start = json_mountinfo_object_start;
sem.object_end = json_mountinfo_object_end;
sem.scalar = json_mountinfo_scalar;
sem.object_field_start = json_mountinfo_object_field_start;

return pg_parse_json(lex, &sem);
}

static JsonParseErrorType
json_mountinfo_object_start(void *state)
{
JsonVaultMountInfoState *parse = (JsonVaultMountInfoState *) state;

switch (parse->state)
{
case JRESP_MOUNT_INFO_EXPECT_OPTIONS_START:
parse->state = JRESP_MOUNT_INFO_EXPECT_OPTIONS_FIELD;
break;
default:
/* NOP */
break;
}

parse->level++;

return JSON_SUCCESS;
}

static JsonParseErrorType
json_mountinfo_object_end(void *state)
{
JsonVaultMountInfoState *parse = (JsonVaultMountInfoState *) state;

if (parse->state == JRESP_MOUNT_INFO_EXPECT_OPTIONS_FIELD)
parse->state = JRESP_MOUNT_INFO_EXPECT_TOPLEVEL_FIELD;

parse->level--;

return JSON_SUCCESS;
}

static JsonParseErrorType
json_mountinfo_scalar(void *state, char *token, JsonTokenType tokentype)
{
JsonVaultMountInfoState *parse = (JsonVaultMountInfoState *) state;

switch (parse->state)
{
case JRESP_MOUNT_INFO_EXPECT_TYPE_VALUE:
parse->type = token;
parse->state = JRESP_MOUNT_INFO_EXPECT_TOPLEVEL_FIELD;
break;
case JRESP_MOUNT_INFO_EXPECT_VERSION_VALUE:
parse->version = token;
parse->state = JRESP_MOUNT_INFO_EXPECT_OPTIONS_FIELD;
break;
case JRESP_MOUNT_INFO_EXPECT_OPTIONS_START:

/*
* Reset "options" object expectations if we got scalar. Most
* likely just a null.
*/
parse->state = JRESP_MOUNT_INFO_EXPECT_TOPLEVEL_FIELD;
break;
default:
/* NOP */
break;
}

return JSON_SUCCESS;
}

static JsonParseErrorType
json_mountinfo_object_field_start(void *state, char *fname, bool isnull)
{
JsonVaultMountInfoState *parse = (JsonVaultMountInfoState *) state;

switch (parse->state)
{
case JRESP_MOUNT_INFO_EXPECT_TOPLEVEL_FIELD:
if (parse->level == 0)
{
if (strcmp(fname, "type") == 0)
{
parse->state = JRESP_MOUNT_INFO_EXPECT_TYPE_VALUE;
break;
}

if (strcmp(fname, "options") == 0)
{
parse->state = JRESP_MOUNT_INFO_EXPECT_OPTIONS_START;
break;
}
}
break;

case JRESP_MOUNT_INFO_EXPECT_OPTIONS_FIELD:
if (parse->level == 1)
{
if (strcmp(fname, "version") == 0)
{
parse->state = JRESP_MOUNT_INFO_EXPECT_VERSION_VALUE;
break;
}
}
break;

default:
/* NOP */
break;
}

return JSON_SUCCESS;
Expand Down
Loading
0