Skip to content

Commit c9bc948

Browse files
siara-ccsiara-in
siara-cc
authored and
siara-in
committed
Add console and shox examples
1 parent ee8b7d1 commit c9bc948

File tree

8 files changed

+1058
-3
lines changed

8 files changed

+1058
-3
lines changed

‎README.md‎

Lines changed: 38 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,36 @@ The stack size of ESP8266 Arduino core is to be increased to atleast 6144 bytes
5555
* Before opening database files from SPIFFS, the `vfs_set_spiffs_file_obj()` should be called with a reference to SPIFFS file object
5656
* A prefix (in front of filenames) such as `/FLASH/` is to be used for SPIFFS and `/SD0/` is to be used for Micro SD, for opening databases.
5757
58+
## Compression with Shox96
59+
60+
This implementation of `sqlite3` includes two functions `shox96_0_2c()` and `shox96_0_2d()` for compressing and decompressing text data.
61+
62+
Shox96 is a compression technique developed for reducing storage size of Short Strings. Details of how it works can be found [here](https://github.com/siara-cc/Shox96).
63+
64+
As of now it can work on only strings made of 'A to Z', 'a to z', '0-9', Special Characters such as &*() etc. found on keyboard, CR, LF, TAB and Space.
65+
66+
In general it can achieve upto 40% size reduction for Short Strings.
67+
68+
### Usage
69+
70+
The following set of commands demonstrate how compression can be accomplished:
71+
72+
```sql
73+
create table test (b1 blob);
74+
insert into test values (shox96_0_2c('Hello World'));
75+
insert into test values (shox96_0_2c('Lorem Ipsum is simply dummy text of the printing and typesetting industry. Lorem Ipsum has been the industry''s standard dummy text ever since the 1500s, when an unknown printer took a galley of type and scrambled it to make a type specimen book.'));
76+
select txt, length(txt) txt_len from (select shox96_0_2d(b1) txt from test);
77+
select length(b1) compressed_len from test;
78+
```
79+
80+
See screenshots section for output.
81+
82+
### Limitations (for Shox96)
83+
84+
- Trying to decompress any blob that was not compressed using `shox96_0_2c()` will crash the program.
85+
- It does not work if the string has binary characters. that is, other than ASCII 32 to 126, CR, LF and Tab.
86+
- Dictionary based compression / decompression is not yet implemented.
87+
5888
## Acknowledgements
5989
* This library was developed by modifying the VFS layer for Sqlite3 developed by [Luiz Felipe Silva](https://github.com/luizfeliperj). The documentation can be found [here](https://nodemcu.readthedocs.io/en/master/en/modules/sqlite3/).
6090
* The census2000 and baby names databases were taken from here: http://2016.padjo.org/tutorials/sqlite-data-starterpacks/. But no license information is available.
@@ -64,21 +94,26 @@ The stack size of ESP8266 Arduino core is to be increased to atleast 6144 bytes
6494

6595
## Screenshots
6696

67-
Output of Micro SD example:
97+
### Output of Micro SD example
6898

6999
![](output_screenshot.png?raw=true)
70100

71-
Output of [SD Card database query through WebServer example](https://github.com/siara-cc/esp_arduino_sqlite3_lib/tree/master/examples/sqlite3_webquery):
101+
### Output of [SD Card database query through WebServer example](https://github.com/siara-cc/esp_arduino_sqlite3_lib/tree/master/examples/sqlite3_webquery):
72102

73103
![](output_web_1.png?raw=true)
74104
![](output_web_2.png?raw=true)
75105

76-
Output of [StackOverflow Users db query example](https://github.com/siara-cc/esp_arduino_sqlite3_lib/tree/master/examples/sqlite3_stackoverflow_users):
106+
### Output of [StackOverflow Users db query example](https://github.com/siara-cc/esp_arduino_sqlite3_lib/tree/master/examples/sqlite3_stackoverflow_users):
77107

78108
![](output_web_so.png?raw=true)
79109
![](output_web_so_id.png?raw=true)
80110
![](output_web_so_name.png?raw=true)
81111
![](output_web_so_loc.png?raw=true)
82112

113+
### Output of Shox96 Compression example
114+
115+
![](output_shox.png?raw=true)
116+
83117
## Issues
118+
84119
Please contact the author (Arundale Ramanathan) at arun@siara.cc if you find any problem (or create issue here).

‎examples/sqlite3_console_sd/sqlite3_console_sd.ino‎

Lines changed: 198 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,198 @@
1+
/*
2+
Open SQLite databass on the SD Card
3+
and execute SQL statements throught this console.
4+
Output is in tab-delimited format.
5+
Increase stack size in cores/esp8266/cont.h
6+
to atleast 6144 (from 4096)
7+
*/
8+
#include <stdio.h>
9+
#include <stdlib.h>
10+
#include <string.h>
11+
#include <sqlite3.h>
12+
#include <vfs.h>
13+
#include <SPI.h>
14+
#include "SdFat.h"
15+
extern "C" {
16+
#include "user_interface.h"
17+
}
18+
#include <ESP8266WiFi.h>
19+
20+
#define MAX_FILE_NAME_LEN 100
21+
#define MAX_STR_LEN 500
22+
23+
void WiFiOff() {
24+
wifi_station_disconnect();
25+
wifi_set_opmode(NULL_MODE);
26+
wifi_set_sleep_type(MODEM_SLEEP_T);
27+
wifi_fpm_open();
28+
wifi_fpm_do_sleep(0xFFFFFFF);
29+
}
30+
31+
char db_file_name[MAX_FILE_NAME_LEN] = "\0";
32+
sqlite3 *db = NULL;
33+
int rc;
34+
35+
bool first_time = false;
36+
static int callback(void *data, int argc, char **argv, char **azColName) {
37+
int i;
38+
if (first_time) {
39+
Serial.println((const char *) data);
40+
for (i = 0; i<argc; i++) {
41+
if (i)
42+
Serial.print((char) '\t');
43+
Serial.printf("%s", azColName[i]);
44+
}
45+
Serial.printf("\n");
46+
first_time = false;
47+
}
48+
for (i = 0; i<argc; i++) {
49+
if (i)
50+
Serial.print((char) '\t');
51+
Serial.printf("%s", argv[i] ? argv[i] : "NULL");
52+
}
53+
Serial.printf("\n");
54+
return 0;
55+
}
56+
57+
int db_open() {
58+
if (db != NULL)
59+
sqlite3_close(db);
60+
int rc = sqlite3_open(db_file_name, &db);
61+
if (rc) {
62+
Serial.print(F("Can't open database: "));
63+
Serial.println(sqlite3_errmsg(db));
64+
return rc;
65+
} else
66+
Serial.println(F("Opened database successfully"));
67+
return rc;
68+
}
69+
70+
char *zErrMsg = 0;
71+
const char* data = "Output:";
72+
int db_exec(const char *sql) {
73+
if (db == NULL) {
74+
Serial.println("No database open");
75+
return 0;
76+
}
77+
first_time = true;
78+
long start = micros();
79+
int rc = sqlite3_exec(db, sql, callback, (void*)data, &zErrMsg);
80+
if (rc != SQLITE_OK) {
81+
Serial.print(F("SQL error: "));
82+
Serial.println(zErrMsg);
83+
sqlite3_free(zErrMsg);
84+
} else
85+
Serial.println(F("Operation done successfully"));
86+
Serial.print(F("Time taken:"));
87+
Serial.print(micros()-start);
88+
Serial.println(F(" us"));
89+
return rc;
90+
}
91+
92+
int input_string(char *str, int max_len) {
93+
max_len--;
94+
int ctr = 0;
95+
str[ctr] = 0;
96+
while (str[ctr] != '\n') {
97+
if (Serial.available()) {
98+
str[ctr] = Serial.read();
99+
if (str[ctr] >= ' ' && str[ctr] <= '~')
100+
ctr++;
101+
if (ctr >= max_len)
102+
break;
103+
}
104+
}
105+
str[ctr] = 0;
106+
Serial.println(str);
107+
}
108+
109+
int input_num() {
110+
char in[20];
111+
int ctr = 0;
112+
in[ctr] = 0;
113+
while (in[ctr] != '\n') {
114+
if (Serial.available()) {
115+
in[ctr] = Serial.read();
116+
if (in[ctr] >= '0' && in[ctr] <= '9')
117+
ctr++;
118+
if (ctr >= sizeof(in))
119+
break;
120+
}
121+
}
122+
in[ctr] = 0;
123+
int ret = atoi(in);
124+
Serial.println(ret);
125+
return ret;
126+
}
127+
128+
enum {CHOICE_OPEN_DB = 1, CHOICE_EXEC_SQL, CHOICE_EXEC_MULTI_SQL, CHOICE_CLOSE_DB,
129+
CHOICE_LIST_FOLDER, CHOICE_RENAME_FILE, CHOICE_DELETE_FILE};
130+
int askChoice() {
131+
Serial.println();
132+
Serial.println(F("Welcome to SQLite console!!"));
133+
Serial.println(F("---------------------------"));
134+
Serial.println();
135+
Serial.print(F("Database file: "));
136+
Serial.println(db_file_name);
137+
Serial.println();
138+
Serial.println(F("1. Open database 2. Execute SQL"));
139+
Serial.println(F("3. Execute Multiple SQL 4. Close database"));
140+
Serial.println();
141+
Serial.print(F("Enter choice: "));
142+
return input_num();
143+
}
144+
145+
void displayPrompt(const char *title) {
146+
Serial.print(F("Enter "));
147+
Serial.println(title);
148+
}
149+
150+
void setup() {
151+
Serial.begin(74880);
152+
system_update_cpu_freq(SYS_CPU_160MHZ);
153+
WiFiOff();
154+
SPI.begin();
155+
vfs_mount("/SD0", SS);
156+
sqlite3_initialize();
157+
}
158+
159+
char str[MAX_STR_LEN];
160+
void loop() {
161+
int choice = askChoice();
162+
switch (choice) {
163+
case CHOICE_OPEN_DB:
164+
displayPrompt("file name (prefix /SD0/): ");
165+
input_string(str, MAX_FILE_NAME_LEN);
166+
if (str[0] != 0) {
167+
strncpy(db_file_name, str, MAX_FILE_NAME_LEN);
168+
db_open();
169+
}
170+
break;
171+
case CHOICE_EXEC_SQL:
172+
Serial.print(F("Enter SQL (max "));
173+
Serial.print(MAX_STR_LEN);
174+
Serial.println(F(" characters):"));
175+
input_string(str, MAX_STR_LEN);
176+
if (str[0] != 0)
177+
db_exec(str);
178+
break;
179+
case CHOICE_EXEC_MULTI_SQL:
180+
Serial.println(F("(Copy paste may not always work due to limited serial buffer)"));
181+
Serial.println(F("Keep entering SQL, empty to stop:"));
182+
do {
183+
input_string(str, MAX_STR_LEN);
184+
if (str[0] != 0)
185+
db_exec(str);
186+
} while (str[0] != 0);
187+
break;
188+
case CHOICE_CLOSE_DB:
189+
if (db_file_name[0] != 0) {
190+
db_file_name[0] = 0;
191+
sqlite3_close(db);
192+
}
193+
break;
194+
default:
195+
Serial.println(F("Invalid choice. Try again."));
196+
}
197+
198+
}

0 commit comments

Comments
 (0)