apacheGH-46092: [C++] Add filesystem related options to Meson (apache#46101)

WillAyd · zanmato1984 · commit 003331740e46 · 2025-04-15T15:03:51.000-07:00
### Rationale for this change This continues adding support for the Meson build system by adding filesystem support ### What changes are included in this PR? This implements the filesystem directory and adds support for Azure. GCS and S3 are stubbed out but will raise an error when a user attempts to enable them, strictly because they look like larger integrations to get the dependency chain properly configured. ### Are these changes tested? Locally yes ### Are there any user-facing changes? No * GitHub Issue: apache#46092 Authored-by: Will Ayd <william.ayd@icloud.com> Signed-off-by: Sutou Kouhei <kou@clear-code.com>
diff --git a/cpp/cmake_modules/DefineOptions.cmake b/cpp/cmake_modules/DefineOptions.cmake
@@ -346,7 +346,8 @@ takes precedence over ccache if a storage backend is configured" ON)
                 ARROW_WITH_UTF8PROC)
 
   define_option(ARROW_GCS
-                "Build Arrow with GCS support (requires the GCloud SDK for C++)"
+                "Build Arrow with GCS support (requires the Google Cloud Platform "
+                "C++ Client Libraries)"
                 OFF
                 DEPENDS
                 ARROW_FILESYSTEM)
diff --git a/cpp/meson.build b/cpp/meson.build
@@ -58,8 +58,11 @@ endif
 
 needs_benchmarks = get_option('benchmarks')
 needs_csv = get_option('csv')
+needs_azure = get_option('azure')
+needs_gcs = get_option('gcs')
 needs_hdfs = get_option('hdfs')
-needs_filesystem = false or needs_hdfs
+needs_s3 = get_option('s3')
+needs_filesystem = get_option('filesystem') or needs_azure or needs_gcs or needs_hdfs or needs_s3
 needs_integration = get_option('integration')
 needs_tests = get_option('tests')
 needs_ipc = get_option('ipc') or needs_tests or needs_benchmarks
diff --git a/cpp/meson.options b/cpp/meson.options
@@ -15,6 +15,13 @@
 # specific language governing permissions and limitations
 # under the License.
 
+option(
+    'azure',
+    type: 'boolean',
+    description: 'Build Arrow with Azure support (requires the Azure SDK for C++)',
+    value: false,
+)
+
 option(
     'benchmarks',
     type: 'boolean',
@@ -29,6 +36,20 @@ option(
     value: false,
 )
 
+option(
+    'filesystem',
+    type: 'boolean',
+    description: 'Build the Arrow Filesystem Layer',
+    value: false,
+)
+
+option(
+    'gcs',
+    type: 'boolean',
+    description: 'Build Arrow with GCS support (requires the Google Cloud Platform C++ Client Libraries)',
+    value: false,
+)
+
 option(
     'hdfs',
     type: 'boolean',
@@ -73,6 +94,13 @@ option(
     description: 'Arbitrary string that identifies the kind of package (for informational purposes)',
 )
 
+option(
+    's3',
+    type: 'boolean',
+    description: 'Build Arrow with S3 support (requires the AWS SDK for C++)',
+    value: false,
+)
+
 option(
     'testing',
     type: 'boolean',
diff --git a/cpp/src/arrow/filesystem/meson.build b/cpp/src/arrow/filesystem/meson.build
@@ -0,0 +1,83 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+install_headers(
+    [
+        'api.h',
+        'azurefs.h',
+        'filesystem.h',
+        'filesystem_library.h',
+        'gcsfs.h',
+        'hdfs.h',
+        'localfs.h',
+        'mockfs.h',
+        'path_util.h',
+        's3fs.h',
+        's3_test_util.h',
+        'test_util.h',
+        'type_fwd.h',
+    ],
+    subdir: 'arrow/filesystem',
+)
+
+pkg.generate(
+    filebase: 'arrow-filesystem',
+    name: 'Apache Arrow Filesystem',
+    description: 'Filesystem API for accessing local and remote filesystems',
+    requires: ['arrow'],
+)
+
+test_cpp_arg = '-DARROW_FILESYSTEM_EXAMPLE_LIBPATH="@0@"'.format(
+    arrow_filesystem_example.full_path(),
+)
+exc = executable(
+    'arrow-filesystem-test',
+    sources: ['filesystem_test.cc', 'localfs_test.cc'],
+    dependencies: [arrow_test_dep],
+    cpp_args: test_cpp_arg,
+)
+test('arrow-filesystem-test', exc)
+
+exc = executable(
+    'arrow-filesystem-localfs-benchmark',
+    sources: ['localfs_benchmark.cc'],
+    dependencies: [arrow_benchmark_dep],
+)
+benchmark('arrow-filesystem-localfs-benchmark', exc)
+
+exc = executable(
+    'arrow-gcsfs-test',
+    sources: ['gcsfs_test.cc'],
+    dependencies: [arrow_test_dep, gcs_dep],
+)
+test('arrow-gcsfs-test', exc)
+
+exc = executable(
+    'arrow-azurefs-test',
+    sources: ['azurefs_test.cc'],
+    dependencies: [arrow_test_dep, azure_dep],
+)
+test('arrow-azurefs-test', exc)
+
+if needs_hdfs
+    exc = executable(
+        'arrow-hdfs-test',
+        sources: ['hdfs_test.cc'],
+        dependencies: [arrow_test_dep],
+    )
+    test('arrow-hdfs-test', exc)
+endif
diff --git a/cpp/src/arrow/meson.build b/cpp/src/arrow/meson.build
@@ -264,6 +264,71 @@ else
     rapidjson_dep = disabler()
 endif
 
+azure_dep = disabler()
+gcs_dep = disabler()
+s3_dep = disabler()
+if needs_filesystem
+    arrow_filesystem_srcs = [
+        'filesystem/filesystem.cc',
+        'filesystem/localfs.cc',
+        'filesystem/mockfs.cc',
+        'filesystem/path_util.cc',
+        'filesystem/util_internal.cc',
+    ]
+
+    arrow_filesystem_deps = []
+
+    if needs_azure
+        arrow_filesystem_srcs += ['filesystem/azurefs.cc']
+        cmake = import('cmake')
+        azure_opt = cmake.subproject_options()
+        azure_opt.add_cmake_defines(
+            {'BUILD_PERFORMANCE_TESTS': 'FALSE'},
+            {'BUILD_SAMPLES': 'FALSE'},
+            {'BUILD_TESTING': 'FALSE'},
+            {'BUILD_WINDOWS_UWP': 'TRUE'},
+            {'CMAKE_UNITY_BUILD': 'FALSE'},
+            {'DISABLE_AZURE_CORE_OPENTELEMETRY': 'TRUE'},
+            {'ENV{AZURE_SDK_DISABLE_AUTO_VCPKG}': 'TRUE'},
+            {'WARNINGS_AS_ERRORS': 'FALSE'},
+        )
+        azure_opt.append_compile_args('cpp', '-fPIC')
+        azure_proj = cmake.subproject('azure', options: azure_opt)
+
+        azure_dep = declare_dependency(
+            dependencies: [
+                azure_proj.dependency('azure-core'),
+                azure_proj.dependency('azure-identity'),
+                azure_proj.dependency('azure-storage-blobs'),
+                azure_proj.dependency('azure-storage-common'),
+                azure_proj.dependency('azure-storage-files-datalake'),
+            ],
+        )
+        arrow_filesystem_deps += [azure_dep]
+    endif
+
+    if needs_gcs
+        error('gcs filesystem support is not yet implemented in Meson')
+    endif
+
+    if needs_hdfs
+        arrow_filesystem_srcs += ['filesystem/hdfs.cc']
+    endif
+
+    if needs_s3
+        error('s3 filesystem support is not yet implemented in Meson')
+    endif
+
+    arrow_components += {
+        'arrow_filesystem': {
+            'sources': arrow_filesystem_srcs,
+            'dependencies': arrow_filesystem_deps,
+        },
+    }
+
+    arrow_testing_srcs += ['filesystem/test_util.cc']
+endif
+
 if needs_ipc
     arrow_ipc_srcs = [
         'ipc/dictionary.cc',
@@ -406,7 +471,7 @@ endif
 arrow_test_lib = static_library(
     'arrow_testing',
     sources: arrow_testing_srcs,
-    dependencies: [arrow_dep, filesystem_dep, gtest_main_dep],
+    dependencies: [arrow_dep, filesystem_dep, gmock_dep, gtest_main_dep],
 )
 
 if needs_tests
@@ -555,3 +620,7 @@ endif
 if needs_csv
     subdir('csv')
 endif
+
+if needs_filesystem
+    subdir('filesystem')
+endif
diff --git a/cpp/src/arrow/testing/meson.build b/cpp/src/arrow/testing/meson.build
@@ -54,9 +54,11 @@ foreach key, val : testing_tests
 endforeach
 
 if needs_tests and needs_filesystem
-    shared_module(
+    arrow_filesystem_example = shared_module(
         'arrow-filesystem-example',
         sources: ['examplefs.cc'],
         dependencies: [arrow_dep, gtest_dep],
     )
+else
+    arrow_filesystem_example = disabler()
 endif
diff --git a/cpp/subprojects/azure.wrap b/cpp/subprojects/azure.wrap
@@ -0,0 +1,23 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+[wrap-file]
+source_url = https://github.com/Azure/azure-sdk-for-cpp/archive/azure-identity_1.9.0.tar.gz
+source_filename = azure-sdk-for-cpp-azure-identity_1.9.0.tar.gz
+source_hash = 97065bfc971ac8df450853ce805f820f52b59457bd7556510186a1569502e4a1
+directory = azure-sdk-for-cpp-azure-identity_1.9.0
+method = cmake
