Infrastructure: allow passing stdin to subprocesses.

CMakeLists.txt

@@ -34,15 +34,10 @@ if(DEFINE_DISABLE_METRICS OR VCPKG_DISABLE_METRICS)
         "file vcpkg.disable_metrics next to the binary.")
 endif()
 
-set(LANGUAGES "CXX")
-if(VCPKG_BUILD_TLS12_DOWNLOADER)
-    list(APPEND LANGUAGES "C")
-endif()
-
 project(vcpkg
     DESCRIPTION "vcpkg helps you manage C and C++ libraries on Windows, Linux and MacOS."
     HOMEPAGE_URL "https://github.com/microsoft/vcpkg"
-    LANGUAGES ${LANGUAGES}
+    LANGUAGES C CXX
 )
 
 list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_LIST_DIR}/cmake")
@@ -449,9 +444,8 @@ if(VCPKG_ADD_SOURCELINK)
         REF "${VCPKG_VERSION}"
     )
 endif()
-if(VCPKG_PDB_SUFFIX)
-    set_property(TARGET vcpkg PROPERTY PDB_NAME "vcpkg${VCPKG_PDB_SUFFIX}")
-endif()
+
+set_property(TARGET vcpkg PROPERTY PDB_NAME "vcpkg${VCPKG_PDB_SUFFIX}")
 
 # === Target: generate-message-map ===
 set(GENERATE_MESSAGE_MAP_DEPENDENCIES vcpkg)
@@ -481,6 +475,7 @@ if (BUILD_TESTING)
         "${CMAKE_CURRENT_SOURCE_DIR}/src/vcpkg.manifest"
     )
     target_link_libraries(vcpkg-test PRIVATE vcpkglib)
+    set_property(TARGET vcpkg-test PROPERTY PDB_NAME "vcpkg-test${VCPKG_PDB_SUFFIX}")
     if(ANDROID)
         target_link_libraries(vcpkg-test PRIVATE log)
     endif()
@@ -499,27 +494,43 @@ if (BUILD_TESTING)
 endif()
 
 # === Target: vcpkg-fuzz ===
-
-file(GLOB VCPKG_FUZZ_SOURCES CONFIGURE_DEPENDS "src/vcpkg-fuzz/*.cpp")
+set(VCPKG_FUZZ_SOURCES "${CMAKE_CURRENT_SOURCE_DIR}/src/vcpkg-fuzz/main.cpp")
 if(VCPKG_BUILD_FUZZING)
     add_executable(vcpkg-fuzz ${VCPKG_FUZZ_SOURCES} "${CMAKE_CURRENT_SOURCE_DIR}/src/vcpkg.manifest")
     target_link_libraries(vcpkg-fuzz PRIVATE vcpkglib)
+    set_property(TARGET vcpkg-fuzz PROPERTY PDB_NAME "vcpkg-fuzz${VCPKG_PDB_SUFFIX}")
 endif()
 
 
 # === Target: tls12-download ===
-
 set(TLS12_DOWNLOAD_SOURCES "${CMAKE_CURRENT_SOURCE_DIR}/src/tls12-download.c")
 if(VCPKG_BUILD_TLS12_DOWNLOADER)
     add_executable(tls12-download ${TLS12_DOWNLOAD_SOURCES} "${CMAKE_CURRENT_SOURCE_DIR}/src/vcpkg.manifest")
     set_property(TARGET tls12-download PROPERTY MSVC_RUNTIME_LIBRARY "MultiThreaded")
     set_property(TARGET tls12-download APPEND PROPERTY LINK_OPTIONS "$<IF:$<CONFIG:Debug>,,/ENTRY:entry>")
     target_link_libraries(tls12-download winhttp wintrust shell32)
-    if(VCPKG_PDB_SUFFIX)
-        set_property(TARGET tls12-download PROPERTY PDB_NAME "tls12-download${VCPKG_PDB_SUFFIX}")
-    endif()
+    set_property(TARGET tls12-download PROPERTY PDB_NAME "tls12-download${VCPKG_PDB_SUFFIX}")
 endif()
 
+if (BUILD_TESTING)
+# === Target: closes-stdin ===
+
+set(CLOSES_STDIN_SOURCES "${CMAKE_CURRENT_SOURCE_DIR}/src/closes-stdin.c")
+add_executable(closes-stdin ${CLOSES_STDIN_SOURCES} "${CMAKE_CURRENT_SOURCE_DIR}/src/vcpkg.manifest")
+set_property(TARGET closes-stdin PROPERTY PDB_NAME "closes-stdin${VCPKG_PDB_SUFFIX}")
+
+# === Target: closes-stdout ===
+
+set(CLOSES_STDOUT_SOURCES "${CMAKE_CURRENT_SOURCE_DIR}/src/closes-stdout.c")
+add_executable(closes-stdout ${CLOSES_STDOUT_SOURCES} "${CMAKE_CURRENT_SOURCE_DIR}/src/vcpkg.manifest")
+set_property(TARGET closes-stdout PROPERTY PDB_NAME "closes-stdout${VCPKG_PDB_SUFFIX}")
+
+# === Target: reads-stdin ===
+
+set(READS_STDIN_SOURCES "${CMAKE_CURRENT_SOURCE_DIR}/src/reads-stdin.c")
+add_executable(reads-stdin ${READS_STDIN_SOURCES} "${CMAKE_CURRENT_SOURCE_DIR}/src/vcpkg.manifest")
+set_property(TARGET reads-stdin PROPERTY PDB_NAME "reads-stdin${VCPKG_PDB_SUFFIX}")
+endif()
 
 # === Target: format ===
 
@@ -540,6 +551,9 @@ if(CLANG_FORMAT)
 
         COMMAND "${CLANG_FORMAT}" -i -verbose ${VCPKG_FUZZ_SOURCES}
         COMMAND "${CLANG_FORMAT}" -i -verbose ${TLS12_DOWNLOAD_SOURCES}
+        COMMAND "${CLANG_FORMAT}" -i -verbose ${CLOSES_STDIN_SOURCES}
+        COMMAND "${CLANG_FORMAT}" -i -verbose ${CLOSES_STDOUT_SOURCES}
+        COMMAND "${CLANG_FORMAT}" -i -verbose ${READS_STDIN_SOURCES}
     )
 endif()
 

include/vcpkg/base/downloads.h

@@ -36,8 +36,7 @@ namespace vcpkg
                                     View<std::pair<std::string, Path>> url_pairs,
                                     View<std::string> headers);
 
-    bool send_snapshot_to_api(const Filesystem& fs,
-                              const std::string& github_token,
+    bool send_snapshot_to_api(const std::string& github_token,
                               const std::string& github_repository,
                               const Json::Object& snapshot);
     ExpectedL<int> put_file(const ReadOnlyFilesystem&,

include/vcpkg/base/files.h

@@ -330,6 +330,10 @@ namespace vcpkg
         std::vector<std::string> exts;
         bool operator()(const Path& target) const;
     };
+
+#if !defined(_WIN32)
+    void close_mark_invalid(int& fd) noexcept;
+#endif // ^^^ !_WIN32
 }
 
 VCPKG_FORMAT_AS(vcpkg::Path, vcpkg::StringView);

include/vcpkg/base/system-headers.h

@@ -21,17 +21,16 @@
 #if __APPLE__
 extern "C"
 {
-#endif
+#endif // __APPLE__
 
 #include <unistd.h>
 
 #if __APPLE__
 }
-#endif
-
-#endif
+#endif // __APPLE__
 
 #include <sys/types.h>
 // glibc defines major and minor in sys/types.h, and should not
 #undef major
 #undef minor
+#endif // ^^^ Unix

include/vcpkg/base/system.process.h

@@ -1,10 +1,10 @@
 #pragma once
 
 #include <vcpkg/base/fwd/files.h>
-#include <vcpkg/base/fwd/optional.h>
 #include <vcpkg/base/fwd/system.process.h>
 
 #include <vcpkg/base/expected.h>
+#include <vcpkg/base/optional.h>
 #include <vcpkg/base/path.h>
 #include <vcpkg/base/span.h>
 #include <vcpkg/base/stringview.h>
@@ -134,24 +134,27 @@ namespace vcpkg
                                                                 const WorkingDirectory& wd = default_working_directory,
                                                                 const Environment& env = default_environment,
                                                                 Encoding encoding = Encoding::Utf8,
-                                                                EchoInDebug echo_in_debug = EchoInDebug::Hide);
+                                                                EchoInDebug echo_in_debug = EchoInDebug::Hide,
+                                                                StringView stdin_content = {});
 
     std::vector<ExpectedL<ExitCodeAndOutput>> cmd_execute_and_capture_output_parallel(
         View<Command> cmd_lines,
         const WorkingDirectory& wd = default_working_directory,
         const Environment& env = default_environment);
 
     ExpectedL<int> cmd_execute_and_stream_lines(const Command& cmd_line,
-                                                std::function<void(StringView)> per_line_cb,
+                                                const std::function<void(StringView)>& per_line_cb,
                                                 const WorkingDirectory& wd = default_working_directory,
                                                 const Environment& env = default_environment,
-                                                Encoding encoding = Encoding::Utf8);
+                                                Encoding encoding = Encoding::Utf8,
+                                                StringView stdin_content = {});
 
     ExpectedL<int> cmd_execute_and_stream_data(const Command& cmd_line,
-                                               std::function<void(StringView)> data_cb,
+                                               const std::function<void(StringView)>& data_cb,
                                                const WorkingDirectory& wd = default_working_directory,
                                                const Environment& env = default_environment,
-                                               Encoding encoding = Encoding::Utf8);
+                                               Encoding encoding = Encoding::Utf8,
+                                               StringView stdin_content = {});
 
     uint64_t get_subproccess_stats();
 

src/closes-stdin.c

@@ -0,0 +1,7 @@
+#if defined(_WIN32)
+#include <Windows.h>
+int main() { CloseHandle(GetStdHandle(STD_INPUT_HANDLE)); }
+#else
+#include <unistd.h>
+int main(void) { close(STDIN_FILENO); }
+#endif

src/closes-stdout.c

@@ -0,0 +1,18 @@
+#include <stdio.h>
+#if defined(_WIN32)
+#include <Windows.h>
+#else // ^^^ _WIN32 // !_WIN32
+#include <unistd.h>
+#endif // ^^^ !_WIN32
+
+int main(void)
+{
+    const char content[] = "hello world";
+    fwrite(content, 1, sizeof(content) - 1, stdout);
+    fflush(stdout);
+#if defined(_WIN32)
+    CloseHandle(GetStdHandle(STD_OUTPUT_HANDLE));
+#else  // ^^^ _WIN32 // !_WIN32
+    close(STDOUT_FILENO);
+#endif // ^^^ !_WIN32
+}

src/reads-stdin.c

@@ -0,0 +1,38 @@
+#include <stdio.h>
+#include <string.h>
+
+// This program reads stdin and asserts that it is the expected value repeated
+// until stdin ends
+
+int main(int argc, char** argv)
+{
+    char buffer[20];
+    // The repeated string 'example' is intentionally a prime length to make hitting buffering edge
+    // cases more likely
+    const char expected[] = "exampleexampleexampleexamp";
+    size_t offset = 0; // always between 0 and 6
+    for (;;)
+    {
+        size_t read_amount = fread(buffer, 1, sizeof(buffer), stdin);
+        if (argc > 1)
+        {
+            puts(argv[1]);
+            fflush(stdout);
+        }
+        if (read_amount == 0)
+        {
+            if (feof(stdin))
+            {
+                puts("success");
+                return 0;
+            }
+            return 1;
+        }
+
+        if (memcmp(buffer, expected + offset, read_amount) != 0)
+        {
+            return 2;
+        }
+        offset = (offset + read_amount) % 7;
+    }
+}

src/vcpkg-test/system.process.cpp

@@ -0,0 +1,76 @@
+#include <catch2/catch.hpp>
+
+#include <vcpkg/base/strings.h>
+#include <vcpkg/base/system.process.h>
+
+using namespace vcpkg;
+
+TEST_CASE ("captures-output", "[system.process]")
+{
+    auto test_program = Path(get_exe_path_of_current_process().parent_path()) / "reads-stdin";
+    Command cmd{test_program};
+    cmd.string_arg("this is printed when something is read");
+    static constexpr std::size_t minimum_size = 1'000'000; // to exceed OS pipe buffer size
+    constexpr StringLiteral example = "example";
+    constexpr auto examples = (minimum_size / example.size()) + 1;
+    std::string input;
+    constexpr auto input_size = examples * example.size();
+    for (std::size_t idx = 0; idx < examples; ++idx)
+    {
+        input.append(example.data(), example.size());
+    }
+
+    std::string expected;
+    constexpr StringLiteral repeat = "this is printed when something is read";
+    constexpr auto repeats = (input_size / 20) + (input_size % 20 != 0) + 1;
+    for (std::size_t idx = 0; idx < repeats; ++idx)
+    {
+        expected.append(repeat.data(), repeat.size());
+#if defined(_WIN32)
+        expected.push_back('\r');
+#endif // ^^^ _WIN32
+        expected.push_back('\n');
+    }
+
+    expected.append("success");
+#if defined(_WIN32)
+    expected.push_back('\r');
+#endif // ^^^ _WIN32
+    expected.push_back('\n');
+
+    auto run = cmd_execute_and_capture_output(
+                   cmd, default_working_directory, default_environment, Encoding::Utf8, EchoInDebug::Hide, input)
+                   .value_or_exit(VCPKG_LINE_INFO);
+    REQUIRE(run.exit_code == 0);
+    REQUIRE(run.output == expected);
+}
+
+TEST_CASE ("no closes-stdin crash", "[system.process]")
+{
+    auto test_program = Path(get_exe_path_of_current_process().parent_path()) / "closes-stdin";
+    Command cmd{test_program};
+    auto run = cmd_execute_and_capture_output(cmd,
+                                              default_working_directory,
+                                              default_environment,
+                                              Encoding::Utf8,
+                                              EchoInDebug::Hide,
+                                              "this is some input that will be intentionally not read")
+                   .value_or_exit(VCPKG_LINE_INFO);
+    REQUIRE(run.exit_code == 0);
+    REQUIRE(run.output.empty());
+}
+
+TEST_CASE ("no closes-stdout crash", "[system.process]")
+{
+    auto test_program = Path(get_exe_path_of_current_process().parent_path()) / "closes-stdout";
+    Command cmd{test_program};
+    auto run = cmd_execute_and_capture_output(cmd,
+                                              default_working_directory,
+                                              default_environment,
+                                              Encoding::Utf8,
+                                              EchoInDebug::Hide,
+                                              "this is some input that will be read")
+                   .value_or_exit(VCPKG_LINE_INFO);
+    REQUIRE(run.exit_code == 0);
+    REQUIRE(run.output == "hello world");
+}

src/vcpkg/base/downloads.cpp

@@ -12,7 +12,6 @@
 #include <vcpkg/base/system.process.h>
 #include <vcpkg/base/system.proxy.h>
 #include <vcpkg/base/util.h>
-#include <vcpkg/base/uuid.h>
 
 #include <vcpkg/commands.version.h>
 #include <vcpkg/metrics.h>
@@ -522,8 +521,7 @@ namespace vcpkg
         return ret;
     }
 
-    bool send_snapshot_to_api(const Filesystem& fs,
-                              const std::string& github_token,
+    bool send_snapshot_to_api(const std::string& github_token,
                               const std::string& github_repository,
                               const Json::Object& snapshot)
     {
@@ -540,24 +538,24 @@ namespace vcpkg
         cmd.string_arg("-H").string_arg("X-GitHub-Api-Version: 2022-11-28");
         cmd.string_arg(
             Strings::concat("https://api.github.com/repos/", github_repository, "/dependency-graph/snapshots"));
-
-        const auto tmp_dir = fs.create_or_get_temp_directory(VCPKG_LINE_INFO);
-        const auto unique_file_name = fmt::format("dependency_snapshot_{}.json", generate_random_UUID());
-        const auto snapshot_file_path = tmp_dir / unique_file_name;
-        fs.write_contents(snapshot_file_path, Json::stringify(snapshot), VCPKG_LINE_INFO);
-
-        cmd.string_arg("-d").string_arg(fmt::format("@{}", snapshot_file_path));
+        cmd.string_arg("-d").string_arg("@-");
         int code = 0;
-        auto result = cmd_execute_and_stream_lines(cmd, [&code](StringView line) {
-            if (Strings::starts_with(line, guid_marker))
-            {
-                code = std::strtol(line.data() + guid_marker.size(), nullptr, 10);
-            }
-            else
-            {
-                Debug::println(line);
-            }
-        });
+        auto result = cmd_execute_and_stream_lines(
+            cmd,
+            [&code](StringView line) {
+                if (Strings::starts_with(line, guid_marker))
+                {
+                    code = std::strtol(line.data() + guid_marker.size(), nullptr, 10);
+                }
+                else
+                {
+                    Debug::println(line);
+                }
+            },
+            default_working_directory,
+            default_environment,
+            Encoding::Utf8,
+            Json::stringify(snapshot));
 
         auto r = result.get();
         if (r && *r == 0 && code >= 200 && code < 300)