rapidsai · aocsa · Jul 17, 2018 · Jul 18, 2018 · Jul 20, 2018 · Jul 20, 2018
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -1,6 +1,7 @@
 #=============================================================================
 # Copyright 2018 BlazingDB, Inc.
 #     Copyright 2018 Percy Camilo Triveño Aucahuasi <[email protected]>
+#     Copyright 2018 Cristhian Alberto Gonzales Castillo <[email protected]>
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -26,7 +27,7 @@
 
 PROJECT(libgdf)
 
-cmake_minimum_required(VERSION 2.8)  # not sure about version required
+cmake_minimum_required(VERSION 3.3)  # not sure about version required
 
 set(CMAKE_CXX_STANDARD 11)
 message(STATUS "Using C++ standard: c++${CMAKE_CXX_STANDARD}")
@@ -43,6 +44,7 @@ include(CTest)
 # Include custom modules (see cmake directory)
 include(ConfigureGoogleTest)
 include(ConfigureArrow)
+include(ConfigureParquetCpp)
 
 find_package(CUDA)
 set_package_properties(

diff --git a/cmake/Modules/ConfigureArrow.cmake b/cmake/Modules/ConfigureArrow.cmake
@@ -1,6 +1,7 @@
 #=============================================================================
 # Copyright 2018 BlazingDB, Inc.
 #     Copyright 2018 Percy Camilo Triveño Aucahuasi <[email protected]>
+#     Copyright 2018 Cristhian Alberto Gonzales Castillo <[email protected]>
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -15,7 +16,7 @@
 # limitations under the License.
 #=============================================================================
 
-set(ARROW_DOWNLOAD_BINARY_DIR ${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/thirdparty/arrow-download/)
+set(ARROW_DOWNLOAD_BINARY_DIR ${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/thirdparty/arrow-download)
 
 # Download and unpack arrow at configure time
 configure_file(${CMAKE_SOURCE_DIR}/cmake/Templates/Arrow.CMakeLists.txt.cmake ${ARROW_DOWNLOAD_BINARY_DIR}/CMakeLists.txt COPYONLY)
@@ -46,7 +47,7 @@ endif()
 set(ARROW_ROOT ${ARROW_DOWNLOAD_BINARY_DIR}/arrow-prefix/src/arrow-install/usr/local/)
 
 # Copy the arrow-format flatbuffer headers to include/ipc using configure_file (will sync if input file changes)
-set(ARROW_GENERATED_IPC_DIR ${ARROW_DOWNLOAD_BINARY_DIR}/arrow-prefix/src/arrow/cpp/src/arrow/ipc/)
+set(ARROW_GENERATED_IPC_DIR ${ARROW_DOWNLOAD_BINARY_DIR}/arrow-prefix/src/arrow/cpp/src/arrow/ipc)
 
 configure_file(${ARROW_GENERATED_IPC_DIR}/File_generated.h ${CMAKE_SOURCE_DIR}/include/gdf/ipc/File_generated.h COPYONLY)
 configure_file(${ARROW_GENERATED_IPC_DIR}/Message_generated.h ${CMAKE_SOURCE_DIR}/include/gdf/ipc/Message_generated.h COPYONLY)

diff --git a/cmake/Modules/ConfigureParquetCpp.cmake b/cmake/Modules/ConfigureParquetCpp.cmake
@@ -0,0 +1,85 @@
+#=============================================================================
+# Copyright 2018 BlazingDB, Inc.
+#     Copyright 2018 Cristhian Alberto Gonzales Castillo <[email protected]>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#=============================================================================
+
+# Download and unpack ParquetCpp at configure time
+configure_file(${CMAKE_SOURCE_DIR}/cmake/Templates/ParquetCpp.CMakeLists.txt.cmake ${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/thirdparty/parquetcpp-download/CMakeLists.txt)
+
+execute_process(
+    COMMAND ${CMAKE_COMMAND} -F "${CMAKE_GENERATOR}" .
+    RESULT_VARIABLE result
+    WORKING_DIRECTORY ${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/thirdparty/parquetcpp-download/
+)
+
+if(result)
+    message(FATAL_ERROR "CMake step for ParquetCpp failed: ${result}")
+endif()
+
+# Transitive dependencies
+set(ARROW_TRANSITIVE_DEPENDENCIES_PREFIX ${ARROW_DOWNLOAD_BINARY_DIR}/arrow-prefix/src/arrow-build)
+set(BROTLI_TRANSITIVE_DEPENDENCY_PREFIX ${ARROW_TRANSITIVE_DEPENDENCIES_PREFIX}/brotli_ep/src/brotli_ep-install/lib/x86_64-linux-gnu)
+set(BROTLI_STATIC_LIB_ENC ${BROTLI_TRANSITIVE_DEPENDENCY_PREFIX}/libbrotlienc.a)
+set(BROTLI_STATIC_LIB_DEC ${BROTLI_TRANSITIVE_DEPENDENCY_PREFIX}/libbrotlidec.a)
+set(BROTLI_STATIC_LIB_COMMON ${BROTLI_TRANSITIVE_DEPENDENCY_PREFIX}/libbrotlicommon.a)
+set(SNAPPY_STATIC_LIB ${ARROW_TRANSITIVE_DEPENDENCIES_PREFIX}/snappy_ep/src/snappy_ep-install/lib/libsnappy.a)
+set(ZLIB_STATIC_LIB ${ARROW_TRANSITIVE_DEPENDENCIES_PREFIX}/zlib_ep/src/zlib_ep-install/lib/libz.a)
+set(ARROW_HOME ${ARROW_ROOT})
+
+set(ENV{BROTLI_STATIC_LIB_ENC} ${BROTLI_STATIC_LIB_ENC})
+set(ENV{BROTLI_STATIC_LIB_DEC} ${BROTLI_STATIC_LIB_DEC})
+set(ENV{BROTLI_STATIC_LIB_COMMON} ${BROTLI_STATIC_LIB_COMMON})
+set(ENV{SNAPPY_STATIC_LIB} ${SNAPPY_STATIC_LIB})
+set(ENV{ZLIB_STATIC_LIB} ${ZLIB_STATIC_LIB})
+set(ENV{ARROW_HOME} ${ARROW_HOME})
+
+execute_process(
+    COMMAND ${CMAKE_COMMAND} --build .
+    RESULT_VARIABLE result
+    WORKING_DIRECTORY ${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/thirdparty/parquetcpp-download)
+
+if(result)
+    message(FATAL_ERROR "Build step for ParquetCpp failed: ${result}")
+endif()
+
+# Add transitive dependency: Thrift
+set(THRIFT_ROOT ${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/thirdparty/parquetcpp-build/thrift_ep/src/thrift_ep-install)
+
+# Locate ParquetCpp package
+set(PARQUETCPP_ROOT ${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/thirdparty/parquetcpp-install)
+set(PARQUETCPP_BINARY_DIR ${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/thirdparty/parquetcpp-build)
+set(PARQUETCPP_SOURCE_DIR ${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/thirdparty/parquetcpp-src)
+
+# Dependency interfaces
+find_package(Boost REQUIRED COMPONENTS regex)
+
+add_library(Apache::Thrift INTERFACE IMPORTED)
+set_target_properties(Apache::Thrift
+    PROPERTIES INTERFACE_INCLUDE_DIRECTORIES ${THRIFT_ROOT}/include)
+set_target_properties(Apache::Thrift
+    PROPERTIES INTERFACE_LINK_LIBRARIES ${THRIFT_ROOT}/lib/libthrift.a)
+
+add_library(Apache::Arrow INTERFACE IMPORTED)
+set_target_properties(Apache::Arrow
+    PROPERTIES INTERFACE_INCLUDE_DIRECTORIES ${ARROW_ROOT}/include)
+set_target_properties(Apache::Arrow
+    PROPERTIES INTERFACE_LINK_LIBRARIES "${ARROW_ROOT}/lib/libarrow.a;${BROTLI_STATIC_LIB_ENC};${BROTLI_STATIC_LIB_DEC};${BROTLI_STATIC_LIB_COMMON};${SNAPPY_STATIC_LIB};${ZLIB_STATIC_LIB}")
+
+add_library(Apache::ParquetCpp INTERFACE IMPORTED)
+set_target_properties(Apache::ParquetCpp
+    PROPERTIES INTERFACE_INCLUDE_DIRECTORIES
+        "${PARQUETCPP_ROOT}/include;${PARQUETCPP_BINARY_DIR}/src;${PARQUETCPP_SOURCE_DIR}/src")
+set_target_properties(Apache::ParquetCpp
+    PROPERTIES INTERFACE_LINK_LIBRARIES "${PARQUETCPP_ROOT}/lib/libparquet.a;Apache::Arrow;Apache::Thrift;Boost::regex")
diff --git a/cmake/Templates/Arrow.CMakeLists.txt.cmake b/cmake/Templates/Arrow.CMakeLists.txt.cmake
@@ -1,6 +1,7 @@
 #=============================================================================
 # Copyright 2018 BlazingDB, Inc.
 #     Copyright 2018 Percy Camilo Triveño Aucahuasi <[email protected]>
+#     Copyright 2018 Cristhian Alberto Gonzales Castillo <[email protected]>
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -34,27 +35,25 @@ message(STATUS "Using Apache Arrow version: ${ARROW_VERSION}")
 set(ARROW_URL "https://github.com/apache/arrow/archive/${ARROW_VERSION}.tar.gz")
 
 set(ARROW_CMAKE_ARGS
-    #Arrow dependencies
-    -DARROW_WITH_LZ4=OFF
-    -DARROW_WITH_ZSTD=OFF
-    -DARROW_WITH_BROTLI=OFF
-    -DARROW_WITH_SNAPPY=OFF
-    -DARROW_WITH_ZLIB=OFF
-
     #Build settings
     -DARROW_BUILD_STATIC=ON
     -DARROW_BUILD_SHARED=OFF
     -DARROW_BOOST_USE_SHARED=ON
     -DARROW_BUILD_TESTS=OFF
     -DARROW_TEST_MEMCHECK=OFF
     -DARROW_BUILD_BENCHMARKS=OFF
+    -DARROW_BUILD_UTILITIES=OFF
+    -DARROW_JEMALLOC=OFF
+    -DARROW_WITH_LZ4=OFF
+    -DARROW_WITH_ZSTD=OFF
 
     #Arrow modules
     -DARROW_IPC=ON
-    -DARROW_COMPUTE=OFF
+    -DARROW_COMPUTE=ON
     -DARROW_GPU=OFF
     -DARROW_JEMALLOC=OFF
     -DARROW_HDFS=OFF
+    -DARROW_BOOST_USE_SHARED=ON
     -DARROW_BOOST_VENDORED=OFF
     -DARROW_PYTHON=OFF
 )

diff --git a/cmake/Templates/ParquetCpp.CMakeLists.txt.cmake b/cmake/Templates/ParquetCpp.CMakeLists.txt.cmake
@@ -0,0 +1,44 @@
+#=============================================================================
+# Copyright 2018 BlazingDB, Inc.
+#     Copyright 2018 Cristhian Alberto Gonzales Castillo <[email protected]>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#=============================================================================
+
+cmake_minimum_required(VERSION 2.8.12)
+
+project(parquetcpp-download NONE)
+
+include(ExternalProject)
+
+set(PARQUET_VERSION apache-parquet-cpp-1.3.1)
+
+if (NOT $ENV{PARQUET_VERSION} STREQUAL "")
+    set(PARQUET_VERSION $ENV{PARQUET_VETSION})
+endif()
+
+message(STATUS "Using Apache ParquetCpp version: ${PARQUET_VERSION}")
+
+ExternalProject_Add(parquetcpp
+    BINARY_DIR "${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/thirdparty/parquetcpp-build"
+    CMAKE_ARGS
+        -DCMAKE_BUILD_TYPE=RELEASE
+        -DCMAKE_INSTALL_PREFIX=${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/thirdparty/parquetcpp-install
+        -DPARQUET_ARROW_LINKAGE=static
+        -DPARQUET_BUILD_SHARED=OFF
+        -DPARQUET_BUILD_TESTS=OFF
+    GIT_REPOSITORY https://github.com/apache/parquet-cpp.git
+    GIT_TAG apache-parquet-cpp-1.3.1
+    INSTALL_DIR "${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/thirdparty/parquetcpp-install"
+    SOURCE_DIR "${CMAKE_BINARY_DIR}${CMAKE_FILES_DIRECTORY}/thirdparty/parquetcpp-src"
+)
diff --git a/conda_environments/dev_py35.yml b/conda_environments/dev_py35.yml
@@ -26,3 +26,5 @@ dependencies:
 - numba=0.34.0.dev=np112py35_316
 - cmake=3.6.3=0
 - arrow-cpp=0.7.1
+- flex=2.6.4
+- bison=3.0.5
diff --git a/include/gdf/parquet/api.h b/include/gdf/parquet/api.h
@@ -0,0 +1,39 @@
+/*
+ * Copyright 2018 BlazingDB, Inc.
+ *     Copyright 2018 Cristhian Alberto Gonzales Castillo <[email protected]>
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <gdf/gdf.h>
+
+#ifdef __cplusplus
+#define BEGIN_NAMESPACE_GDF_PARQUET                                           \
+    namespace gdf {                                                           \
+    namespace parquet {
+#define END_NAMESPACE_GDF_PARQUET                                             \
+    }                                                                         \
+    }
+#else
+#define BEGIN_NAMESPACE_GDF_PARQUET
+#define END_NAMESPACE_GDF_PARQUET
+#endif
+
+BEGIN_NAMESPACE_GDF_PARQUET
+
+extern "C" gdf_error
+read_parquet_file(const char *const  filename,
+                  gdf_column **const out_gdf_columns,
+                  std::size_t *const out_gdf_columns_length);
+
+END_NAMESPACE_GDF_PARQUET