前文回顾
- 实现一个简略的Database1(译文）
- 实现一个简略的Database2(译文）
实现一个简略的Database3(译文）
译注：cstsck在github保护了一个简略的、相似sqlite的数据库实现，通过这个简略的我的项目，能够很好的了解数据库是如何运行的。本文是第三篇，次要是实现数据库的实现内存中的数据结构并存储数据
Part 3 在内存中，只追加的单表数据库
咱们从一个小型的，有许多限度的数据库开始。当初数据库将：
- 反对两个操作：插入一行并打印所有行
- 数据驻留在内存中（没有长久化到磁盘）
- 反对单个、硬编码的表
咱们的硬编码表将用来存储用户数据，看起来就行上面展现的这样：
| column | type |
|---|---|
| id | integer |
| username | varchar(32) |
| varchar(255) |
这是一个简略的计划，然而它将让咱们的数据库可能反对不同的数据类型和不同大小的文本数据类型。插入语句当初看起来像上面这样：
insert 1 cstack [email protected]这象征咱们须要降级prepare_statement()函数来解析参数：
if (strncmp(input_buffer->buffer, "insert", 6) == 0) { statement->type = STATEMENT_INSERT; + int args_assigned = sscanf( + input_buffer->buffer, "insert %d %s %s", &(statement->row_to_insert.id), + statement->row_to_insert.username, statement->row_to_insert.email); + if (args_assigned < 3) { + return PREPARE_SYNTAX_ERROR; + } return PREPARE_SUCCESS; } if (strcmp(input_buffer->buffer, "select") == 0) {咱们把这些解析出的的参数存储到Statement对象中的一个新的数据结构Row中。
+#define COLUMN_USERNAME_SIZE 32 +#define COLUMN_EMAIL_SIZE 255 +typedef struct { + uint32_t id; + char username[COLUMN_USERNAME_SIZE]; + char email[COLUMN_EMAIL_SIZE]; +} Row; + typedef struct { StatementType type; + Row row_to_insert; // only used by insert statement } Statement;当初咱们须要copy这些数据到其余一些代表table的数据结构中。SQLite为了反对疾速查找、插入和删除操作而应用B-tree。咱们将从一些简略的开始。像B-tree，它把行数据分组成页（pages），然而为了替换把这些页（pages）组织成一颗树的这种办法，这里咱们把页来组织成数组（array）。
这是我的打算：
- 存储行数据到叫做页（pages）的内存块中
- 每页尽量多的去存储适宜他的大小的数据（在页的大小范畴内，尽量多的存储数据）
- 在每页中行数据将被序列化为紧凑示意（compact representation）
- 页只有在须要时候才会被调配
- 放弃固定大小的指针数组指向页
首先咱们定义一个紧凑示意的行（row）：
+#define size_of_attribute(Struct, Attribute) sizeof(((Struct*)0)->Attribute) + +const uint32_t ID_SIZE = size_of_attribute(Row, id); +const uint32_t USERNAME_SIZE = size_of_attribute(Row, username); +const uint32_t EMAIL_SIZE = size_of_attribute(Row, email); +const uint32_t ID_OFFSET = 0; +const uint32_t USERNAME_OFFSET = ID_OFFSET + ID_SIZE; +const uint32_t EMAIL_OFFSET = USERNAME_OFFSET + USERNAME_SIZE; +const uint32_t ROW_SIZE = ID_SIZE + USERNAME_SIZE + EMAIL_SIZE;这意味着一个序列化的行的布局看起来就像上面这样：
| column | size (bytes) | offset |
|---|---|---|
| id | 4 | 0 |
| username | 32 | 4 |
| 255 | 36 | |
| total | 291 |
译注：画个图来直观的看一下这个行数据存储格局
咱们还须要编码来转换紧凑示意。（即把数据序列化与反序列化）
+void serialize_row(Row* source, void* destination) { + memcpy(destination + ID_OFFSET, &(source->id), ID_SIZE); + memcpy(destination + USERNAME_OFFSET, &(source->username), USERNAME_SIZE); + memcpy(destination + EMAIL_OFFSET, &(source->email), EMAIL_SIZE); +} + +void deserialize_row(void* source, Row* destination) { + memcpy(&(destination->id), source + ID_OFFSET, ID_SIZE); + memcpy(&(destination->username), source + USERNAME_OFFSET, USERNAME_SIZE); + memcpy(&(destination->email), source + EMAIL_OFFSET, EMAIL_SIZE); +}接下来，实现一个表的构造指向存储行的页并跟踪页中有多少行：
+const uint32_t PAGE_SIZE = 4096; +#define TABLE_MAX_PAGES 100 +const uint32_t ROWS_PER_PAGE = PAGE_SIZE / ROW_SIZE; +const uint32_t TABLE_MAX_ROWS = ROWS_PER_PAGE * TABLE_MAX_PAGES; + +typedef struct { + uint32_t num_rows; + void* pages[TABLE_MAX_PAGES]; +} Table;我把数据页大小设定为4KB，因为它与大多数计算机架构的虚拟内存零碎中应用数据页大小雷同。这意味着数据库中的一个数据页的大小和零碎中的一页大小正好雷同。操作系统在把数据页移入或者移出内存的时候会作为一个残缺的单位来操作，而不会拆散他们。
在调配page时我设置了一个很随便的限度，限度调配100个page。当切换到一个tree构造时，咱们数据库的最大限度就只是受到系统文件的大小限度了（只管我依然是限度在内存中一次能够有多少page能够放弃）。
行不能超出page的边界。因为page在内存中可能不会彼此相邻，这个假如能够让读/写行数据更简略。
说到这一点，上面是咱们如何弄清楚在内存中去哪里读/写特定行。
+void* row_slot(Table* table, uint32_t row_num) { + uint32_t page_num = row_num / ROWS_PER_PAGE; + void* page = table->pages[page_num]; + if (page == NULL) { + // Allocate memory only when we try to access page + page = table->pages[page_num] = malloc(PAGE_SIZE); + } + uint32_t row_offset = row_num % ROWS_PER_PAGE; + uint32_t byte_offset = row_offset * ROW_SIZE; + return page + byte_offset; +}当初咱们通过execute_statement()函数能够从表构造中读/写了。
-void execute_statement(Statement* statement) { +ExecuteResult execute_insert(Statement* statement, Table* table) { + if (table->num_rows >= TABLE_MAX_ROWS) { + return EXECUTE_TABLE_FULL; + } + + Row* row_to_insert = &(statement->row_to_insert); + + serialize_row(row_to_insert, row_slot(table, table->num_rows)); + table->num_rows += 1; + + return EXECUTE_SUCCESS; +} + +ExecuteResult execute_select(Statement* statement, Table* table) { + Row row; + for (uint32_t i = 0; i < table->num_rows; i++) { + deserialize_row(row_slot(table, i), &row); + print_row(&row); + } + return EXECUTE_SUCCESS; +} + +ExecuteResult execute_statement(Statement* statement, Table* table) { switch (statement->type) { case (STATEMENT_INSERT): - printf("This is where we would do an insert.\n"); - break; + return execute_insert(statement, table); case (STATEMENT_SELECT): - printf("This is where we would do a select.\n"); - break; + return execute_select(statement, table); } }最初，咱们须要初始化table，创立各自的内存开释函数并且须要解决一些报错状况：
+ Table* new_table() { + Table* table = (Table*)malloc(sizeof(Table)); + table->num_rows = 0; + for (uint32_t i = 0; i < TABLE_MAX_PAGES; i++) { + table->pages[i] = NULL; + } + return table; +} + +void free_table(Table* table) { + for (int i = 0; table->pages[i]; i++) { + free(table->pages[i]); + } + free(table); +}在主函数中调用table初始化，并解决报错：
int main(int argc, char* argv[]) { + Table* table = new_table(); InputBuffer* input_buffer = new_input_buffer(); while (true) { print_prompt(); @@ -105,13 +203,22 @@ int main(int argc, char* argv[]) { switch (prepare_statement(input_buffer, &statement)) { case (PREPARE_SUCCESS): break; + case (PREPARE_SYNTAX_ERROR): + printf("Syntax error. Could not parse statement.\n"); + continue; case (PREPARE_UNRECOGNIZED_STATEMENT): printf("Unrecognized keyword at start of '%s'.\n", input_buffer->buffer); continue; } - execute_statement(&statement); - printf("Executed.\n"); + switch (execute_statement(&statement, table)) { + case (EXECUTE_SUCCESS): + printf("Executed.\n"); + break; + case (EXECUTE_TABLE_FULL): + printf("Error: Table full.\n"); + break; + } } }做了这些批改后咱们就能理论保留数据到数据库了。
~ ./db db > insert 1 cstack [email protected] Executed. db > insert 2 bob [email protected] Executed. db > select (1, cstack, [email protected]) (2, bob, [email protected]) Executed. db > insert foo bar 1 Syntax error. Could not parse statement. db > .exit ~当初是写一些测试的好时机，有几个起因：
- 咱们打算大幅度批改存储表的数据结构，并且测试是可捕捉回归
- 还有一些边界条件咱们没有手动测试（例如填满一张表）
咱们将在下一部分中解决这些问题。当初，看一下这一部分残缺的区别（与上一部分比照，行结尾“+”为新增，“-”为删除）：
@@ -2,6 +2,7 @@ #include <stdio.h> #include <stdlib.h> #include <string.h>+#include <stdint.h> typedef struct { char* buffer;@@ -10,6 +11,105 @@ typedef struct { } InputBuffer;+typedef enum { EXECUTE_SUCCESS, EXECUTE_TABLE_FULL } ExecuteResult;++typedef enum {+ META_COMMAND_SUCCESS,+ META_COMMAND_UNRECOGNIZED_COMMAND+} MetaCommandResult;++typedef enum {+ PREPARE_SUCCESS,+ PREPARE_SYNTAX_ERROR,+ PREPARE_UNRECOGNIZED_STATEMENT+ } PrepareResult;++typedef enum { STATEMENT_INSERT, STATEMENT_SELECT } StatementType;++#define COLUMN_USERNAME_SIZE 32+#define COLUMN_EMAIL_SIZE 255+typedef struct {+ uint32_t id;+ char username[COLUMN_USERNAME_SIZE];+ char email[COLUMN_EMAIL_SIZE];+} Row;++typedef struct {+ StatementType type;+ Row row_to_insert; //only used by insert statement+} Statement;++#define size_of_attribute(Struct, Attribute) sizeof(((Struct*)0)->Attribute)++const uint32_t ID_SIZE = size_of_attribute(Row, id);+const uint32_t USERNAME_SIZE = size_of_attribute(Row, username);+const uint32_t EMAIL_SIZE = size_of_attribute(Row, email);+const uint32_t ID_OFFSET = 0;+const uint32_t USERNAME_OFFSET = ID_OFFSET + ID_SIZE;+const uint32_t EMAIL_OFFSET = USERNAME_OFFSET + USERNAME_SIZE;+const uint32_t ROW_SIZE = ID_SIZE + USERNAME_SIZE + EMAIL_SIZE;++const uint32_t PAGE_SIZE = 4096;+#define TABLE_MAX_PAGES 100+const uint32_t ROWS_PER_PAGE = PAGE_SIZE / ROW_SIZE;+const uint32_t TABLE_MAX_ROWS = ROWS_PER_PAGE * TABLE_MAX_PAGES;++typedef struct {+ uint32_t num_rows;+ void* pages[TABLE_MAX_PAGES];+} Table;++void print_row(Row* row) {+ printf("(%d, %s, %s)\n", row->id, row->username, row->email);+}++void serialize_row(Row* source, void* destination) {+ memcpy(destination + ID_OFFSET, &(source->id), ID_SIZE);+ memcpy(destination + USERNAME_OFFSET, &(source->username), USERNAME_SIZE);+ memcpy(destination + EMAIL_OFFSET, &(source->email), EMAIL_SIZE);+}++void deserialize_row(void *source, Row* destination) {+ memcpy(&(destination->id), source + ID_OFFSET, ID_SIZE);+ memcpy(&(destination->username), source + USERNAME_OFFSET, USERNAME_SIZE);+ memcpy(&(destination->email), source + EMAIL_OFFSET, EMAIL_SIZE);+}++void* row_slot(Table* table, uint32_t row_num) {+ uint32_t page_num = row_num / ROWS_PER_PAGE;+ void *page = table->pages[page_num];+ if (page == NULL) {+ // Allocate memory only when we try to access page+ page = table->pages[page_num] = malloc(PAGE_SIZE);+ }+ uint32_t row_offset = row_num % ROWS_PER_PAGE;+ uint32_t byte_offset = row_offset * ROW_SIZE;+ return page + byte_offset;+}++Table* new_table() {+ Table* table = (Table*)malloc(sizeof(Table));+ table->num_rows = 0;+ for (uint32_t i = 0; i < TABLE_MAX_PAGES; i++) {+ table->pages[i] = NULL;+ }+ return table;+}++void free_table(Table* table) {+ for (int i = 0; table->pages[i]; i++) {+ free(table->pages[i]);+ }+ free(table);+}+ InputBuffer* new_input_buffer() { InputBuffer* input_buffer = (InputBuffer*)malloc(sizeof(InputBuffer)); input_buffer->buffer = NULL;@@ -40,17 +140,105 @@ void close_input_buffer(InputBuffer* input_buffer) { free(input_buffer); }+MetaCommandResult do_meta_command(InputBuffer* input_buffer, Table *table) {+ if (strcmp(input_buffer->buffer, ".exit") == 0) {+ close_input_buffer(input_buffer);+ free_table(table);+ exit(EXIT_SUCCESS);+ } else {+ return META_COMMAND_UNRECOGNIZED_COMMAND;+ }+}++PrepareResult prepare_statement(InputBuffer* input_buffer,+ Statement* statement) {+ if (strncmp(input_buffer->buffer, "insert", 6) == 0) {+ statement->type = STATEMENT_INSERT;+ int args_assigned = sscanf(+ input_buffer->buffer, "insert %d %s %s", &(statement->row_to_insert.id),+ statement->row_to_insert.username, statement->row_to_insert.email+ );+ if (args_assigned < 3) {+ return PREPARE_SYNTAX_ERROR;+ }+ return PREPARE_SUCCESS;+ }+ if (strcmp(input_buffer->buffer, "select") == 0) {+ statement->type = STATEMENT_SELECT;+ return PREPARE_SUCCESS;+ }++ return PREPARE_UNRECOGNIZED_STATEMENT;+}++ExecuteResult execute_insert(Statement* statement, Table* table) {+ if (table->num_rows >= TABLE_MAX_ROWS) {+ return EXECUTE_TABLE_FULL;+ }++ Row* row_to_insert = &(statement->row_to_insert);++ serialize_row(row_to_insert, row_slot(table, table->num_rows));+ table->num_rows += 1;++ return EXECUTE_SUCCESS;+}++ExecuteResult execute_select(Statement* statement, Table* table) {+ Row row;+ for (uint32_t i = 0; i < table->num_rows; i++) {+ deserialize_row(row_slot(table, i), &row);+ print_row(&row);+ }+ return EXECUTE_SUCCESS;+}++ExecuteResult execute_statement(Statement* statement, Table *table) {+ switch (statement->type) {+ case (STATEMENT_INSERT):+ return execute_insert(statement, table);+ case (STATEMENT_SELECT):+ return execute_select(statement, table);+ }+}+ int main(int argc, char* argv[]) {+ Table* table = new_table(); InputBuffer* input_buffer = new_input_buffer(); while (true) { print_prompt(); read_input(input_buffer);- if (strcmp(input_buffer->buffer, ".exit") == 0) {- close_input_buffer(input_buffer);- exit(EXIT_SUCCESS);- } else {- printf("Unrecognized command '%s'.\n", input_buffer->buffer);+ if (input_buffer->buffer[0] == '.') {+ switch (do_meta_command(input_buffer, table)) {+ case (META_COMMAND_SUCCESS):+ continue;+ case (META_COMMAND_UNRECOGNIZED_COMMAND):+ printf("Unrecognized command '%s'\n", input_buffer->buffer);+ continue;+ }+ }++ Statement statement;+ switch (prepare_statement(input_buffer, &statement)) {+ case (PREPARE_SUCCESS):+ break;+ case (PREPARE_SYNTAX_ERROR):+ printf("Syntax error. Could not parse statement.\n");+ continue;+ case (PREPARE_UNRECOGNIZED_STATEMENT):+ printf("Unrecognized keyword at start of '%s'.\n",+ input_buffer->buffer);+ continue;+ }++ switch (execute_statement(&statement, table)) {+ case (EXECUTE_SUCCESS):+ printf("Executed.\n");+ break;+ case (EXECUTE_TABLE_FULL):+ printf("Error: Table full.\n");+ break; } }Enjoy GreatSQL :)
## 对于 GreatSQL
GreatSQL是由万里数据库保护的MySQL分支，专一于晋升MGR可靠性及性能，反对InnoDB并行查问个性，是实用于金融级利用的MySQL分支版本。
相干链接： GreatSQL社区 Gitee GitHub Bilibili
GreatSQL社区：
欢送来GreatSQL社区发帖发问
https://greatsql.cn/
技术交换群：
微信：扫码增加GreatSQL社区助手微信好友，发送验证信息加群。