Skip to content

Commit

Permalink
Merge pull request #111 from maxmind/greg/read-node-update
Browse files Browse the repository at this point in the history 
Add more fields to MMDB_search_node_s. Closes GitHub #110
  • Loading branch information
@autarch
autarch committed Mar 22, 2016
2 parents 52100fa + 4a57598 commit 0db8551
Show file tree
Hide file tree
Showing 5 changed files with 205 additions and 56 deletions.
9 changes: 9 additions & 0 deletions Changes.md
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,12 @@
## 1.2.0 - 2016-03-XX

* Four additional fields were added to the end of the `MMDB_search_node_s`
struct returned by `MMDB_read_node`. These fields allow the user to iterate
through the search tree without making undocumented assumptions about how
this library works internally and without knowing the specific details of
the database format. GitHub #110.


## 1.1.5 - 2016-03-20

* Previously, reading a database with a pointer in the metadata would cause an
Expand Down
19 changes: 19 additions & 0 deletions doc/libmaxminddb.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -332,9 +332,28 @@ tree as opposed to looking up a specific IP address.
typedef struct MMDB_search_node_s {
uint64_t left_record;
uint64_t right_record;
uint8_t left_record_type;
uint8_t right_record_type;
MMDB_entry_s left_record_entry;
MMDB_entry_s right_record_entry;
} MMDB_search_node_s;
```

The two record types will take one of the following values:

* `MMDB_RECORD_TYPE_SEARCH_NODE` - The record points to the next search node.
* `MMDB_RECORD_TYPE_EMPTY` - The record is a placeholder that indicates there
is no data for the IP address. The search should end here.
* `MMDB_RECORD_TYPE_DATA` - The record is for data in the data section of the
database. Use the entry for the record when looking up the data for the
record.
* `MMDB_RECORD_TYPE_INVALID` - The record is invalid. Either an invalid node
was looked up or the database is corrupt.

The `MMDB_entry_s` for the record is only valid if the type is
`MMDB_RECORD_TYPE_DATA`. Attempts to use an entry for other record types will
result in an error or invalid data.

# STATUS CODES

This library returns (or populates) status codes for many functions. These
Expand Down
9 changes: 9 additions & 0 deletions include/maxminddb.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,11 @@ typedef ADDRESS_FAMILY sa_family_t;
#define MMDB_DATA_TYPE_BOOLEAN (14)
#define MMDB_DATA_TYPE_FLOAT (15)

#define MMDB_RECORD_TYPE_SEARCH_NODE (0)
#define MMDB_RECORD_TYPE_EMPTY (1)
#define MMDB_RECORD_TYPE_DATA (2)
#define MMDB_RECORD_TYPE_INVALID (3)

/* flags for open */
#define MMDB_MODE_MMAP (1)
#define MMDB_MODE_MASK (7)
Expand Down Expand Up @@ -178,6 +183,10 @@ typedef struct MMDB_s {
typedef struct MMDB_search_node_s {
uint64_t left_record;
uint64_t right_record;
uint8_t left_record_type;
uint8_t right_record_type;
MMDB_entry_s left_record_entry;
MMDB_entry_s right_record_entry;
} MMDB_search_node_s;

/* *INDENT-OFF* */
Expand Down
114 changes: 85 additions & 29 deletions src/maxminddb.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -155,10 +155,13 @@ LOCAL int find_address_in_search_tree(MMDB_s *mmdb, uint8_t *address,
MMDB_lookup_result_s *result);
LOCAL record_info_s record_info_for_database(MMDB_s *mmdb);
LOCAL int find_ipv4_start_node(MMDB_s *mmdb);
LOCAL int populate_result(MMDB_s *mmdb, uint32_t node_count, uint32_t value,
uint16_t netmask, MMDB_lookup_result_s *result);
LOCAL int maybe_populate_result(MMDB_s *mmdb, uint32_t record,
uint16_t netmask, MMDB_lookup_result_s *result);
LOCAL uint8_t record_type(MMDB_s *const mmdb, uint64_t record);
LOCAL uint32_t get_left_28_bit_record(const uint8_t *record);
LOCAL uint32_t get_right_28_bit_record(const uint8_t *record);
LOCAL uint32_t data_section_offset_for_record(MMDB_s *const mmdb,
uint64_t record);
LOCAL int path_length(va_list va_path);
LOCAL int lookup_path_in_array(const char *path_elem, MMDB_s *mmdb,
MMDB_entry_data_s *entry_data);
Expand Down Expand Up @@ -880,7 +883,6 @@ LOCAL int find_address_in_search_tree(MMDB_s *mmdb, uint8_t *address,
DEBUG_NL;
DEBUG_MSG("Looking for address in search tree");

uint32_t node_count = mmdb->metadata.node_count;
uint32_t value = 0;
uint16_t max_depth0 = mmdb->depth - 1;
uint16_t start_bit = max_depth0;
Expand All @@ -893,12 +895,18 @@ LOCAL int find_address_in_search_tree(MMDB_s *mmdb, uint8_t *address,
DEBUG_MSGF("IPv4 start node is %u (netmask %u)",
mmdb->ipv4_start_node.node_value,
mmdb->ipv4_start_node.netmask);

uint8_t type = maybe_populate_result(mmdb,
mmdb->ipv4_start_node.node_value,
mmdb->ipv4_start_node.netmask,
result);
if (MMDB_RECORD_TYPE_INVALID == type) {
return MMDB_CORRUPT_SEARCH_TREE_ERROR;
}

/* We have an IPv6 database with no IPv4 data */
if (mmdb->ipv4_start_node.node_value >= node_count) {
return populate_result(mmdb, node_count,
mmdb->ipv4_start_node.node_value,
mmdb->ipv4_start_node.netmask,
result);
if (MMDB_RECORD_TYPE_SEARCH_NODE != type) {
return MMDB_SUCCESS;
}

value = mmdb->ipv4_start_node.node_value;
Expand Down Expand Up @@ -927,20 +935,16 @@ LOCAL int find_address_in_search_tree(MMDB_s *mmdb, uint8_t *address,
value = record_info.left_record_getter(record_pointer);
}

/* Ideally we'd check to make sure that a record never points to a
* previously seen value, but that's more complicated. For now, we can
* at least check that we don't end up at the top of the tree again. */
if (0 == value) {
DEBUG_MSGF(" %s record has a value of 0",
bit_is_true ? "right" : "left");
uint8_t type = maybe_populate_result(mmdb, value, current_bit, result);
if (MMDB_RECORD_TYPE_INVALID == type) {
return MMDB_CORRUPT_SEARCH_TREE_ERROR;
}

if (value >= node_count) {
return populate_result(mmdb, node_count, value, current_bit, result);
} else {
DEBUG_MSGF(" proceeding to search tree node %i", value);
if (MMDB_RECORD_TYPE_SEARCH_NODE != type) {
return MMDB_SUCCESS;
}

DEBUG_MSGF(" proceeding to search tree node %i", value);
}

DEBUG_MSG(
Expand Down Expand Up @@ -1010,22 +1014,53 @@ LOCAL int find_ipv4_start_node(MMDB_s *mmdb)
return MMDB_SUCCESS;
}

LOCAL int populate_result(MMDB_s *mmdb, uint32_t node_count, uint32_t value,
uint16_t netmask, MMDB_lookup_result_s *result)
LOCAL int maybe_populate_result(MMDB_s *mmdb, uint32_t record,
uint16_t netmask, MMDB_lookup_result_s *result)
{
// This is the offset from the end of the search tree
uint32_t offset = value - node_count;
DEBUG_MSGF(" data section offset is %i (record value = %i)", offset, value);
uint8_t type = record_type(mmdb, record);

if (offset > mmdb->data_section_size) {
return MMDB_CORRUPT_SEARCH_TREE_ERROR;
if (MMDB_RECORD_TYPE_SEARCH_NODE == type ||
MMDB_RECORD_TYPE_INVALID == type) {
return type;
}

result->netmask = mmdb->depth - netmask;
// This is the offset from the beginning of the data section
result->entry.offset = offset - MMDB_DATA_SECTION_SEPARATOR;
result->found_entry = offset > 0 ? true : false;
return MMDB_SUCCESS;

result->entry.offset = data_section_offset_for_record(mmdb, record);

// type is either MMDB_RECORD_TYPE_DATA or MMDB_RECORD_TYPE_EMPTY
// at this point
result->found_entry = MMDB_RECORD_TYPE_DATA == type;

return type;
}

LOCAL uint8_t record_type(MMDB_s *const mmdb, uint64_t record)
{
uint32_t node_count = mmdb->metadata.node_count;

/* Ideally we'd check to make sure that a record never points to a
* previously seen value, but that's more complicated. For now, we can
* at least check that we don't end up at the top of the tree again. */
if (record == 0) {
DEBUG_MSG("record has a value of 0");
return MMDB_RECORD_TYPE_INVALID;
}

if (record < node_count) {
return MMDB_RECORD_TYPE_SEARCH_NODE;
}

if (record == node_count) {
return MMDB_RECORD_TYPE_EMPTY;
}

if (record - node_count < mmdb->data_section_size) {
return MMDB_RECORD_TYPE_DATA;
}

DEBUG_MSG("record has a value that points outside of the database");
return MMDB_RECORD_TYPE_INVALID;
}

LOCAL uint32_t get_left_28_bit_record(const uint8_t *record)
Expand Down Expand Up @@ -1059,9 +1094,30 @@ int MMDB_read_node(MMDB_s *const mmdb, uint32_t node_number,
record_pointer += record_info.right_record_offset;
node->right_record = record_info.right_record_getter(record_pointer);

node->left_record_type = record_type(mmdb, node->left_record);
node->right_record_type = record_type(mmdb, node->right_record);

// Note that offset will be invalid if the record type is not
// MMDB_RECORD_TYPE_DATA, but that's ok. Any use of the record entry
// for other data types is a programming error.
node->left_record_entry = (struct MMDB_entry_s) {
.mmdb = mmdb,
.offset = data_section_offset_for_record(mmdb, node->left_record),
};
node->right_record_entry = (struct MMDB_entry_s) {
.mmdb = mmdb,
.offset = data_section_offset_for_record(mmdb, node->right_record),
};

return MMDB_SUCCESS;
}

LOCAL uint32_t data_section_offset_for_record(MMDB_s *const mmdb,
uint64_t record)
{
return record - mmdb->metadata.node_count - MMDB_DATA_SECTION_SEPARATOR;
}

int MMDB_get_value(MMDB_entry_s *const start,
MMDB_entry_data_s *const entry_data,
...)
Expand Down
110 changes: 83 additions & 27 deletions t/read_node_t.c
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,21 @@
#include "maxminddb_test_helper.h"

void run_read_node_tests(MMDB_s *mmdb, const uint32_t tests[][3],
void test_entry_data(MMDB_s *mmdb, MMDB_entry_s *entry, uint32_t node_number,
char * node_record)
{
MMDB_entry_data_s entry_data;
int status =
MMDB_get_value(entry, &entry_data, "ip",
NULL);
cmp_ok(status, "==", MMDB_SUCCESS,
"successful data lookup for node");
cmp_ok(
entry_data.type, "==", MMDB_DATA_TYPE_UTF8_STRING,
"returned entry type is UTF8_STRING for %s record of node %i",
node_record, node_number);
}

void run_read_node_tests(MMDB_s *mmdb, const uint32_t tests[][5],
int test_count,
uint8_t record_size)
{
Expand All @@ -12,9 +27,25 @@ void run_read_node_tests(MMDB_s *mmdb, const uint32_t tests[][3],
cmp_ok(node.left_record, "==", tests[i][1],
"left record for node %i is %i - %i bit DB",
node_number, tests[i][1], record_size);
cmp_ok(node.right_record, "==", tests[i][2],
"left record for node %i is %i - %i bit DB",
node_number, tests[i][2], record_size);
cmp_ok(node.left_record_type, "==", tests[i][2],
"left record type for node %i is %i", node_number,
tests[i][2]);
if (node.left_record_type == MMDB_RECORD_TYPE_DATA) {
test_entry_data(mmdb, &node.left_record_entry, node_number,
"left");
}

cmp_ok(node.right_record, "==", tests[i][3],
"right record for node %i is %i - %i bit DB",
node_number, tests[i][3], record_size);
cmp_ok(node.right_record_type, "==", tests[i][4],
"right record type for node %i is %i", node_number,
tests[i][4]);

if (node.right_record_type == MMDB_RECORD_TYPE_DATA) {
test_entry_data(mmdb, &node.right_record_entry, node_number,
"right");
}
} else {
diag("call to MMDB_read_node for node %i failed - %i bit DB",
node_number,
Expand All @@ -30,15 +61,23 @@ void run_24_bit_record_tests(int mode, const char *mode_desc)
MMDB_s *mmdb = open_ok(path, mode, mode_desc);
free((void *)path);

const uint32_t tests[5][3] = {
{ 0, 1, 242 },
{ 80, 81, 197 },
{ 96, 97, 242 },
{ 103, 242, 104 },
{ 241, 96, 242 }
const uint32_t tests[7][5] = {
{ 0, 1, MMDB_RECORD_TYPE_SEARCH_NODE, 242,
MMDB_RECORD_TYPE_EMPTY },
{ 80, 81, MMDB_RECORD_TYPE_SEARCH_NODE, 197,
MMDB_RECORD_TYPE_SEARCH_NODE, },
{ 96, 97, MMDB_RECORD_TYPE_SEARCH_NODE, 242,
MMDB_RECORD_TYPE_EMPTY, },
{ 103, 242, MMDB_RECORD_TYPE_EMPTY, 104,
MMDB_RECORD_TYPE_SEARCH_NODE, },
{ 127, 242, MMDB_RECORD_TYPE_EMPTY, 315,
MMDB_RECORD_TYPE_DATA, },
{ 132, 329, MMDB_RECORD_TYPE_DATA, 242,
MMDB_RECORD_TYPE_EMPTY, },
{ 241, 96, MMDB_RECORD_TYPE_SEARCH_NODE, 242,
MMDB_RECORD_TYPE_EMPTY, }
};

run_read_node_tests(mmdb, tests, 5, 24);
run_read_node_tests(mmdb, tests, 7, 24);

MMDB_close(mmdb);
free(mmdb);
Expand All @@ -51,15 +90,23 @@ void run_28_bit_record_tests(int mode, const char *mode_desc)
MMDB_s *mmdb = open_ok(path, mode, mode_desc);
free((void *)path);

const uint32_t tests[5][3] = {
{ 0, 1, 242 },
{ 80, 81, 197 },
{ 96, 97, 242 },
{ 103, 242, 104 },
{ 241, 96, 242 }
const uint32_t tests[7][5] = {
{ 0, 1, MMDB_RECORD_TYPE_SEARCH_NODE, 242,
MMDB_RECORD_TYPE_EMPTY },
{ 80, 81, MMDB_RECORD_TYPE_SEARCH_NODE, 197,
MMDB_RECORD_TYPE_SEARCH_NODE, },
{ 96, 97, MMDB_RECORD_TYPE_SEARCH_NODE, 242,
MMDB_RECORD_TYPE_EMPTY, },
{ 103, 242, MMDB_RECORD_TYPE_EMPTY, 104,
MMDB_RECORD_TYPE_SEARCH_NODE, },
{ 127, 242, MMDB_RECORD_TYPE_EMPTY, 315,
MMDB_RECORD_TYPE_DATA, },
{ 132, 329, MMDB_RECORD_TYPE_DATA, 242,
MMDB_RECORD_TYPE_EMPTY, },
{ 241, 96, MMDB_RECORD_TYPE_SEARCH_NODE, 242,
MMDB_RECORD_TYPE_EMPTY, }
};

run_read_node_tests(mmdb, tests, 5, 28);
run_read_node_tests(mmdb, tests, 7, 28);

MMDB_close(mmdb);
free(mmdb);
Expand All @@ -72,15 +119,24 @@ void run_32_bit_record_tests(int mode, const char *mode_desc)
MMDB_s *mmdb = open_ok(path, mode, mode_desc);
free((void *)path);

const uint32_t tests[5][3] = {
{ 0, 1, 242 },
{ 80, 81, 197 },
{ 96, 97, 242 },
{ 103, 242, 104 },
{ 241, 96, 242 }
const uint32_t tests[7][5] = {
{ 0, 1, MMDB_RECORD_TYPE_SEARCH_NODE, 242,
MMDB_RECORD_TYPE_EMPTY },
{ 80, 81, MMDB_RECORD_TYPE_SEARCH_NODE, 197,
MMDB_RECORD_TYPE_SEARCH_NODE, },
{ 96, 97, MMDB_RECORD_TYPE_SEARCH_NODE, 242,
MMDB_RECORD_TYPE_EMPTY, },
{ 103, 242, MMDB_RECORD_TYPE_EMPTY, 104,
MMDB_RECORD_TYPE_SEARCH_NODE, },
{ 127, 242, MMDB_RECORD_TYPE_EMPTY, 315,
MMDB_RECORD_TYPE_DATA, },
{ 132, 329, MMDB_RECORD_TYPE_DATA, 242,
MMDB_RECORD_TYPE_EMPTY, },
{ 241, 96, MMDB_RECORD_TYPE_SEARCH_NODE, 242,
MMDB_RECORD_TYPE_EMPTY, }
};

run_read_node_tests(mmdb, tests, 5, 32);
run_read_node_tests(mmdb, tests, 7, 32);

MMDB_close(mmdb);
free(mmdb);
Expand Down

0 comments on commit 0db8551

Please sign in to comment.