Skip to content

Commit 1e5f986

Browse files
jelujelu
committed
MaxMind DB
- MaxMind DB support - Refactor to have functions always present and be more C++ - Use pkg-config to detect library - Move function to .cpp to avoid including config.h in .h - Add `PACKETQ_MAXMIND_PATH` setting and search for database files in common paths - Only open the databases once - Use same stderr warning format everywhere
1 parent 3967c31 commit 1e5f986

File tree

6 files changed

+226
-204
lines changed

6 files changed

+226
-204
lines changed

‎FUNCTIONS.md

+23-6
Original file line numberDiff line numberDiff line change
@@ -51,15 +51,18 @@ Defaults to 24 for IPv4 and 48 for IPv6 (/24 and /48 respectively)
5151

5252
### CC(address)
5353

54-
Returns the 2-letter ISO country code associated with the address from a MaxMind database.
55-
The database must be specified in the environment variable "PACKETQ_MAXMIND_CC_DB". Returns
56-
an empty string for lookup failures.
54+
Returns the 2-letter ISO country code associated with the address from
55+
a MaxMind database (see MaxMind Database below on selecting database).
56+
57+
Returns an empty string on lookup failures or if this feature was not
58+
built in.
5759

5860
### ASN(address)
5961

60-
Returns the autonomous system number associated with the address from a MaxMind database as
61-
an integer. The database must be specified in the environment variable "PACKETQ_MAXMIND_ASN_DB".
62-
Returns -1 for lookup failures.
62+
Returns the autonomous system number associated with the address from
63+
a MaxMind database (see MaxMind Database below on selecting database).
64+
65+
Returns -1 on lookup failures or if this feature was not built in.
6366

6467
## String operations
6568

@@ -81,3 +84,17 @@ i.e: `trim('se.domains.se', 'se')` returns `.domains.`.
8184
### LOWER(string)
8285

8386
Turns `string` into lowercase.
87+
88+
# MaxMind Database
89+
90+
PacketQ will try to open MaxMind databases that resides in common path on
91+
major distributions, but you can also specify paths and database files
92+
using environment variables.
93+
94+
`PACKETQ_MAXMIND_PATH` sets the path to look for the databases in addition
95+
to the common paths, it will try `GeoLite2-Country.mmdb` for `CC()` and
96+
`GeoLite2-ASN.mmdb` for `ASN()`.
97+
98+
You can also specify the full path to the database file you wish to use
99+
for each function with `PACKETQ_MAXMIND_CC_DB` and `PACKETQ_MAXMIND_ASN_DB`,
100+
these settings will override path settings.

‎configure.ac

+4-10
Original file line numberDiff line numberDiff line change
@@ -50,17 +50,11 @@ AC_ARG_ENABLE([gcov], [AS_HELP_STRING([--enable-gcov], [Enable coverage testing]
5050
AM_CONDITIONAL([ENABLE_GCOV], [test "x$enable_gcov" != "xno"])
5151
AM_EXTRA_RECURSIVE_TARGETS([gcov])
5252

53-
# Check for MaxMindDB
54-
AC_CHECK_LIB([maxminddb], [MMDB_open], [
55-
AC_MSG_NOTICE([enabling MaxMindDB])
56-
AS_VAR_APPEND(LDFLAGS, [" -lmaxminddb"])
57-
AC_DEFINE([MAXMINDDB], [1], [Enable MaxMindDB])], [
58-
AC_MSG_NOTICE([disabling MaxMindDB])])
59-
6053
# Checks for libraries.
6154
AC_CHECK_LIB([z], [deflate])
6255
AC_CHECK_LIB([socket], socket)
6356
AC_CHECK_LIB([nsl], inet_ntop)
57+
PKG_CHECK_MODULES([libmaxminddb], [libmaxminddb], [AC_DEFINE([HAVE_LIBMAXMINDDB], [1], [Define to 1 if you have libmaxminddb.])], [:])
6458

6559
# Checks for header files.
6660
AC_HEADER_STDC
@@ -88,8 +82,8 @@ AC_CHECK_HEADERS([syslog.h])
8882
AC_LANG([C])
8983

9084
AC_CONFIG_FILES([
91-
Makefile
92-
src/Makefile
93-
src/test/Makefile
85+
Makefile
86+
src/Makefile
87+
src/test/Makefile
9488
])
9589
AC_OUTPUT

‎src/Makefile.am

+2
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ SUBDIRS = test
2525
AM_CXXFLAGS = -I$(srcdir) \
2626
-I$(srcdir)/Murmur \
2727
-I$(top_srcdir) \
28+
$(libmaxminddb_CFLAGS) \
2829
-std=c++0x \
2930
-Wall -Wno-parentheses -Wno-switch -Wno-sign-compare -Wno-char-subscripts
3031

@@ -34,6 +35,7 @@ packetq_SOURCES = dns.cpp dns.h icmp.cpp icmp.h output.h packet_handler.cpp \
3435
packet_handler.h packetq.cpp packetq.h pcap.cpp pcap.h reader.cpp \
3536
reader.h refcountstring.h segzip.h server.cpp server.h sql.cpp sql.h \
3637
tcp.cpp tcp.h variant.h
38+
packetq_LDADD = $(libmaxminddb_LIBS)
3739

3840
dist_packetq_SOURCES = Murmur/MurmurHash3.cpp Murmur/MurmurHash3.h
3941

‎src/packet_handler.cpp

+2-2
Original file line numberDiff line numberDiff line change
@@ -375,7 +375,7 @@ bool Packet::parse_transport(unsigned char* data, int len)
375375
data += dataoffs;
376376
len -= dataoffs;
377377
if (len < 0) {
378-
fprintf(stderr, "warning: Found TCP packet with bad length\n");
378+
fprintf(stderr, "Warning: Found TCP packet with bad length\n");
379379
return false;
380380
}
381381

@@ -393,7 +393,7 @@ bool Packet::parse_transport(unsigned char* data, int len)
393393
len -= 8;
394394

395395
if (len < 0) {
396-
fprintf(stderr, "warning: Found UDP packet with bad length\n");
396+
fprintf(stderr, "Warning: Found UDP packet with bad length\n");
397397
return false;
398398
}
399399
}

‎src/sql.cpp

+189-2
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,8 @@
1919
* along with PacketQ. If not, see <http://www.gnu.org/licenses/>.
2020
*/
2121

22+
#include "config.h"
23+
2224
#include "sql.h"
2325
#include "output.h"
2426
#include "packet_handler.h"
@@ -31,6 +33,14 @@
3133
#ifdef WIN32
3234
#include <windows.h>
3335
#endif
36+
#ifdef HAVE_LIBMAXMINDDB
37+
#include <maxminddb.h>
38+
#include <sys/types.h>
39+
#include <sys/stat.h>
40+
#include <unistd.h>
41+
static MMDB_s* __cc_mmdb = 0;
42+
static MMDB_s* __asn_mmdb = 0;
43+
#endif
3444

3545
namespace packetq {
3646

@@ -1884,14 +1894,12 @@ OP* OP::compile(const std::vector<Table*>& tables, const std::vector<int>& searc
18841894
} else if (cmpi(get_token(), "netmask")) {
18851895
m_t = Coltype::_text;
18861896
ret = new Netmask_func(*this);
1887-
#ifdef MAXMINDDB
18881897
} else if (cmpi(get_token(), "cc")) {
18891898
m_t = Coltype::_text;
18901899
ret = new Cc_func(*this);
18911900
} else if (cmpi(get_token(), "asn")) {
18921901
m_t = Coltype::_int;
18931902
ret = new Asn_func(*this);
1894-
#endif /* MAXMIND */
18951903
} else if (cmpi(get_token(), "count")) {
18961904
m_t = Coltype::_int;
18971905
ret = new Count_func(*this, dest_table);
@@ -2511,6 +2519,185 @@ void Trim_func::evaluate(Row** rows, Variant& v)
25112519
}
25122520
}
25132521

2522+
Cc_func::Cc_func(const OP& op)
2523+
: OP(op)
2524+
{
2525+
#ifdef HAVE_LIBMAXMINDDB
2526+
if (__cc_mmdb) {
2527+
return;
2528+
}
2529+
2530+
std::string db;
2531+
char* env = getenv("PACKETQ_MAXMIND_CC_DB");
2532+
if (env) {
2533+
db = env;
2534+
}
2535+
2536+
if (db.empty()) {
2537+
std::list<std::string> paths = {
2538+
"/var/lib/GeoIP", "/usr/share/GeoIP", "/usr/local/share/GeoIP"
2539+
};
2540+
2541+
if ((env = getenv("PACKETQ_MAXMIND_PATH"))) {
2542+
paths.push_front(std::string(env));
2543+
}
2544+
2545+
std::list<std::string>::iterator i = paths.begin();
2546+
for (; i != paths.end(); i++) {
2547+
db = (*i) + "/GeoLite2-Country.mmdb";
2548+
struct stat s;
2549+
if (!stat(db.c_str(), &s)) {
2550+
break;
2551+
}
2552+
}
2553+
if (i == paths.end()) {
2554+
return;
2555+
}
2556+
}
2557+
2558+
MMDB_s* mmdb = new MMDB_s;
2559+
if (!mmdb) {
2560+
return;
2561+
}
2562+
2563+
int ret = MMDB_open(db.c_str(), 0, mmdb);
2564+
if (ret != MMDB_SUCCESS) {
2565+
fprintf(stderr, "Warning: cannot open MaxMind CC database \"%s\": %s\n", db.c_str(), MMDB_strerror(ret));
2566+
free(mmdb);
2567+
return;
2568+
}
2569+
2570+
__cc_mmdb = mmdb;
2571+
#endif
2572+
}
2573+
2574+
void Cc_func::evaluate(Row** rows, Variant& v)
2575+
{
2576+
#ifdef HAVE_LIBMAXMINDDB
2577+
if (!__cc_mmdb) {
2578+
RefCountStringHandle res(RefCountString::construct(""));
2579+
v = *res;
2580+
return;
2581+
}
2582+
2583+
Variant str;
2584+
m_param[0]->evaluate(rows, str);
2585+
RefCountStringHandle str_handle(str.get_text());
2586+
2587+
int gai_error, ret;
2588+
2589+
MMDB_lookup_result_s mmdb_result = MMDB_lookup_string(__cc_mmdb, (*str_handle)->data, &gai_error, &ret);
2590+
2591+
if (gai_error || ret != MMDB_SUCCESS || !mmdb_result.found_entry) {
2592+
RefCountStringHandle res(RefCountString::construct(""));
2593+
v = *res;
2594+
return;
2595+
}
2596+
2597+
MMDB_entry_data_s entry_data;
2598+
ret = MMDB_get_value(&mmdb_result.entry, &entry_data, "country", "iso_code", NULL);
2599+
2600+
if (ret != MMDB_SUCCESS || !entry_data.has_data || entry_data.type != MMDB_DATA_TYPE_UTF8_STRING) {
2601+
RefCountStringHandle res(RefCountString::construct(""));
2602+
v = *res;
2603+
return;
2604+
}
2605+
2606+
RefCountStringHandle res(RefCountString::construct(entry_data.utf8_string, 0, entry_data.data_size));
2607+
v = *res;
2608+
#else
2609+
RefCountStringHandle res(RefCountString::construct(""));
2610+
v = *res;
2611+
#endif
2612+
}
2613+
2614+
Asn_func::Asn_func(const OP& op)
2615+
: OP(op)
2616+
{
2617+
#ifdef HAVE_LIBMAXMINDDB
2618+
if (__asn_mmdb) {
2619+
return;
2620+
}
2621+
2622+
std::string db;
2623+
char* env = getenv("PACKETQ_MAXMIND_ASN_DB");
2624+
if (env) {
2625+
db = env;
2626+
}
2627+
2628+
if (db.empty()) {
2629+
std::list<std::string> paths = {
2630+
"/var/lib/GeoIP", "/usr/share/GeoIP", "/usr/local/share/GeoIP"
2631+
};
2632+
2633+
if ((env = getenv("PACKETQ_MAXMIND_PATH"))) {
2634+
paths.push_front(std::string(env));
2635+
}
2636+
2637+
std::list<std::string>::iterator i = paths.begin();
2638+
for (; i != paths.end(); i++) {
2639+
db = (*i) + "/GeoLite2-ASN.mmdb";
2640+
struct stat s;
2641+
if (!stat(db.c_str(), &s)) {
2642+
break;
2643+
}
2644+
}
2645+
if (i == paths.end()) {
2646+
return;
2647+
}
2648+
}
2649+
2650+
MMDB_s* mmdb = new MMDB_s;
2651+
if (!mmdb) {
2652+
return;
2653+
}
2654+
2655+
int ret = MMDB_open(db.c_str(), 0, mmdb);
2656+
if (ret != MMDB_SUCCESS) {
2657+
fprintf(stderr, "Warning: cannot open MaxMind ASN database \"%s\": %s\n", db.c_str(), MMDB_strerror(ret));
2658+
free(mmdb);
2659+
return;
2660+
}
2661+
2662+
__asn_mmdb = mmdb;
2663+
#endif
2664+
}
2665+
2666+
void Asn_func::evaluate(Row** rows, Variant& v)
2667+
{
2668+
#ifdef HAVE_LIBMAXMINDDB
2669+
if (!__asn_mmdb) {
2670+
v = -1;
2671+
return;
2672+
}
2673+
2674+
Variant str;
2675+
m_param[0]->evaluate(rows, str);
2676+
RefCountStringHandle str_handle(str.get_text());
2677+
2678+
int gai_error, ret;
2679+
2680+
MMDB_lookup_result_s mmdb_result = MMDB_lookup_string(__asn_mmdb, (*str_handle)->data, &gai_error, &ret);
2681+
2682+
if (gai_error || ret != MMDB_SUCCESS || !mmdb_result.found_entry) {
2683+
v = -1;
2684+
return;
2685+
}
2686+
2687+
MMDB_entry_data_s entry_data;
2688+
ret = MMDB_get_value(&mmdb_result.entry, &entry_data, "autonomous_system_number", NULL);
2689+
2690+
if (ret != MMDB_SUCCESS || !entry_data.has_data || entry_data.type != MMDB_DATA_TYPE_UINT32) {
2691+
v = -1;
2692+
return;
2693+
}
2694+
2695+
v = (int_column)entry_data.uint32;
2696+
#else
2697+
v = -1;
2698+
#endif
2699+
}
2700+
25142701
DB g_db;
25152702

25162703
Coldef Column::m_coldefs[COLTYPE_MAX];

0 commit comments

Comments
 (0)